Python異步爬蟲(chóng)實(shí)現(xiàn)原理與知識(shí)總結(jié)

import time 
#串形
 
def getPage(url):
    print("開(kāi)始爬取網(wǎng)站",url)
    time.sleep(2)#阻塞
    print("爬取完成?。?！",url)
 
 
urls = ['url1','url2','url3','url4','url5']
 
beginTime = time.time()#開(kāi)始計(jì)時(shí)
 
for url in urls:
    getPage(url)
 
endTime= time.time()#結(jié)束計(jì)時(shí)
print("完成時(shí)間%d"%(endTime - beginTime))

import time 
#使用線(xiàn)程池對(duì)象
from multiprocessing.dummy import Pool
 
def getPage(url):
    print("開(kāi)始爬取網(wǎng)站",url)
    time.sleep(2)#阻塞
    print("爬取完成！?。?,url)
 
 
urls = ['url1','url2','url3','url4','url5']
 
beginTime = time.time()#開(kāi)始計(jì)時(shí)
 
#準(zhǔn)備開(kāi)啟5個(gè)線(xiàn)程,并示例化對(duì)象
pool = Pool(5)
pool.map(getPage, urls)#urls是可迭代對(duì)象，里面每個(gè)參數(shù)都會(huì)給getPage方法處理
 
endTime= time.time()#結(jié)束計(jì)時(shí)
print("完成時(shí)間%d"%(endTime - beginTime))

#%%
import time 
#使用協(xié)程
import asyncio
 
 
async def getPage(url):  #定義了一個(gè)協(xié)程對(duì)象，python中函數(shù)也是對(duì)象
    print("開(kāi)始爬取網(wǎng)站",url)
    time.sleep(2)#阻塞
    print("爬取完成?。?！",url)
    
#async修飾的函數(shù)返回的對(duì)象    
c = getPage(11)
 
#創(chuàng)建事件對(duì)象
loop_event = asyncio.get_event_loop()
#注冊(cè)并啟動(dòng)looP
loop_event.run_until_complete(c)
 
#task對(duì)象使用，封裝協(xié)程對(duì)象c
'''
loop_event = asyncio.get_event_loop()
task = loop_event.create_task(c)
loop_event.run_until_complete(task)
'''
 
#Future對(duì)象使用，封裝協(xié)程對(duì)象c            用法和task差不多
'''
loop_event = asyncio.get_event_loop()
task       = asyncio.ensure_future(c)
loop_event.run_until_complete(task)
'''
 
#綁定回調(diào)使用
 
async def getPage2(url):  #定義了一個(gè)協(xié)程對(duì)象，python中函數(shù)也是對(duì)象
    print("開(kāi)始爬取網(wǎng)站",url)
    time.sleep(2)#阻塞
    print("爬取完成?。?！",url)
    return url
    
#async修飾的函數(shù)返回的對(duì)象    
c2 = getPage2(2)
 
def callback_func(task):
    print(task.result()) #task.result()返回任務(wù)對(duì)象中封裝的協(xié)程對(duì)象對(duì)應(yīng)函數(shù)的返回值
 
 
#綁定回調(diào)
loop_event = asyncio.get_event_loop()
task       = asyncio.ensure_future(c2)
 
task.add_done_callback(callback_func)  #真正綁定，
loop_event.run_until_complete(task)

import time 
#使用多任務(wù)協(xié)程
import asyncio
 
 
 
 
urls = ['url1','url2','url3','url4','url5']
 
 
 
async def getPage(url):  #定義了一個(gè)協(xié)程對(duì)象，python中函數(shù)也是對(duì)象
    print("開(kāi)始爬取網(wǎng)站",url)
    #在異步協(xié)程中如果出現(xiàn)同步模塊相關(guān)的代碼，那么無(wú)法實(shí)現(xiàn)異步
    #time.sleep(2)#阻塞
    await asyncio.sleep(2)#遇到阻塞操作必須手動(dòng)掛起
    print("爬取完成！??！",url)
    return url
    
 
beginTime = time.time()  
 
 
#任務(wù)列表，有多個(gè)任務(wù)
tasks = []
 
for url in urls:
    c = getPage(url)
    task = asyncio.ensure_future(c)#創(chuàng)建任務(wù)對(duì)象
    tasks.append(task)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))#不能直接放task，需要封裝進(jìn)入asyncio,wait()方法中
 
endTime = time.time()   
print("完成時(shí)間%d"%(endTime - beginTime)) 

async def getPage(url):  #定義了一個(gè)協(xié)程對(duì)象，python中函數(shù)也是對(duì)象
    print("開(kāi)始爬取網(wǎng)站",url)
    #在異步協(xié)程中如果出現(xiàn)同步模塊相關(guān)的代碼，那么無(wú)法實(shí)現(xiàn)異步
    #requests.get(url)#阻塞
    async with aiohttp.ClintSession() as session:
 
                     async with await  session.get(url) as response: #手動(dòng)掛起
 
                                       page_text =  await response.text() #.text()返回字符串，read()返回二進(jìn)制數(shù)據(jù)，注意不是content
    print("爬取完成?。。?,url)
    return page_text