免費(fèi)AI編程助手
本文主要介紹了利用 Python 來爬取當(dāng)當(dāng)、京東、亞馬遜平臺上的圖書信息代碼實(shí)例，具有一定借鑒價(jià)值，正在學(xué)習(xí)Python爬蟲 或正好有需要的朋友可以參考下。
注：
1.本程序采用 MSSQLserver 數(shù)據(jù)庫存儲，請運(yùn)行程序前手動修改程序開頭處的數(shù)據(jù)庫鏈接信息；
2.需要 bs4、requests、pymssql 庫支持；
3.支持多線程。
源碼
from bs4 import BeautifulSoup
import re,requests,pymysql,threading,os,traceback
try:
conn = pymysql.connect(host='127.0.0.1', port=3306, user='root', passwd='root', db='book',charset="utf8")
cursor = conn.cursor()
except:
print('\\\\n錯(cuò)誤：數(shù)據(jù)庫連接失敗')
#返回指定頁面的html信息
def getHTMLText(url):
try:
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'}
r = requests.get(url,headers = headers)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return ''
#返回指定url的Soup對象
def getSoupObject(url):
try:
html = getHTMLText(url)
soup = BeautifulSoup(html,'html.parser')
return soup
except:
return ''
#獲取該關(guān)鍵字在圖書網(wǎng)站上的總頁數(shù)
def getPageLength(webSiteName,url):
try:
soup = getSoupObject(url)
if webSiteName == 'DangDang':
a = soup('a',{'name':'bottom-page-turn'})
return a[-1].string
elif webSiteName == 'Amazon':
a = soup('span',{'class':'pagnDisabled'})
return a[-1].string
except:
print('\\\\n錯(cuò)誤：獲取{}總頁數(shù)時(shí)出錯(cuò)...'.format(webSiteName))
return -1
class DangDangThread(threading.Thread):
def __init__(self,keyword):
threading.Thread.__init__(self)
self.keyword = keyword
def run(self):
print('\\\\n提示：開始爬取當(dāng)當(dāng)網(wǎng)數(shù)據(jù)...')
count = 1
length = getPageLength('DangDang','http://search.dangdang.com/?key={}'.format(self.keyword))#總頁數(shù)
tableName = 'db_{}_dangdang'.format(self.keyword)
try:
print('\\\\n提示：正在創(chuàng)建DangDang表...')
cursor.execute('create table {} (id int ,title text,prNow text,prPre text,link text)'.format(tableName))
print('\\\\n提示：開始爬取當(dāng)當(dāng)網(wǎng)頁面...')
for i in range(1,int(length)):
url = 'http://search.dangdang.com/?key={}&page_index={}'.format(self.keyword,i)
soup = getSoupObject(url)
lis = soup('li',{'class':re.compile(r'line'),'id':re.compile(r'p')})
for li in lis:
a = li.find_all('a',{'name':'itemlist-title','dd_name':'單品標(biāo)題'})
pn = li.find_all('span',{'class': 'search_now_price'})
pp = li.find_all('span',{'class': 'search_pre_price'})
if not len(a) == 0:
link = a[0].attrs['href']
title = a[0].attrs['title'].strip()
else:
link = 'NULL'
title = 'NULL'
if not len(pn) == 0:
prNow = pn[0].string
else:
prNow = 'NULL'
if not len(pp) == 0:
prPre = pp[0].string
else:
prPre = 'NULL'
sql = "insert into {} (id,title,prNow,prPre,link) values ({},'{}','{}','{}','{}')".format(tableName,count,title,prNow,prPre,link)
cursor.execute(sql)
print('\\\\r提示：正在存入當(dāng)當(dāng)數(shù)據(jù),當(dāng)前處理id：{}'.format(count),end='')
count += 1
conn.commit()
except:
pass
class AmazonThread(threading.Thread):
def __init__(self,keyword):
threading.Thread.__init__(self)
self.keyword = keyword
def run(self):
print('\\\\n提示：開始爬取亞馬遜數(shù)據(jù)...')
count = 1
length = getPageLength('Amazon','https://www.amazon.cn/s/keywords={}'.format(self.keyword))#總頁數(shù)
tableName = 'db_{}_amazon'.format(self.keyword)
try:
print('\\\\n提示：正在創(chuàng)建Amazon表...')
cursor.execute('create table {} (id int ,title text,prNow text,link text)'.format(tableName))
print('\\\\n提示：開始爬取亞馬遜頁面...')
for i in range(1,int(length)):
url = 'https://www.amazon.cn/s/keywords={}&page={}'.format(self.keyword,i)
soup = getSoupObject(url)
lis = soup('li',{'id':re.compile(r'result_')})
for li in lis:
a = li.find_all('a',{'class':'a-link-normal s-access-detail-page a-text-normal'})
pn = li.find_all('span',{'class': 'a-size-base a-color-price s-price a-text-bold'})
if not len(a) == 0:
link = a[0].attrs['href']
title = a[0].attrs['title'].strip()
else:
link = 'NULL'
title = 'NULL'
if not len(pn) == 0:
prNow = pn[0].string
else:
prNow = 'NULL'
sql = "insert into {} (id,title,prNow,link) values ({},'{}','{}','{}')".format(tableName,count,title,prNow,link)
cursor.execute(sql)
print('\\\\r提示：正在存入亞馬遜數(shù)據(jù),當(dāng)前處理id：{}'.format(count),end='')
count += 1
conn.commit()
except:
pass
class JDThread(threading.Thread):
def __init__(self,keyword):
threading.Thread.__init__(self)
self.keyword = keyword
def run(self):
print('\\\\n提示：開始爬取京東數(shù)據(jù)...')
count = 1
tableName = 'db_{}_jd'.format(self.keyword)
try:
print('\\\\n提示：正在創(chuàng)建JD表...')
cursor.execute('create table {} (id int,title text,prNow text,link text)'.format(tableName))
print('\\\\n提示：開始爬取京東頁面...')
for i in range(1,100):
url = 'https://search.jd.com/Search?keyword={}&page={}'.format(self.keyword,i)
soup = getSoupObject(url)
lis = soup('li',{'class':'gl-item'})
for li in lis:
a = li.find_all('div',{'class':'p-name'})
pn = li.find_all('div',{'class': 'p-price'})[0].find_all('i')
if not len(a) == 0:
link = 'http:' + a[0].find_all('a')[0].attrs['href']
title = a[0].find_all('em')[0].get_text()
else:
link = 'NULL'
title = 'NULL'
if(len(link) > 128):
link = 'TooLong'
if not len(pn) == 0:
prNow = '￥'+ pn[0].string
else:
prNow = 'NULL'
sql = "insert into {} (id,title,prNow,link) values ({},'{}','{}','{}')".format(tableName,count,title,prNow,link)
cursor.execute(sql)
print('\\\\r提示：正在存入京東網(wǎng)數(shù)據(jù),當(dāng)前處理id：{}'.format(count),end='')
count += 1
conn.commit()
except :
pass
def closeDB():
global conn,cursor
conn.close()
cursor.close()
def main():
print('提示：使用本程序，請手動創(chuàng)建空數(shù)據(jù)庫：Book，并修改本程序開頭的數(shù)據(jù)庫連接語句')
keyword = input("\\\\n提示：請輸入要爬取的關(guān)鍵字：")
dangdangThread = DangDangThread(keyword)
amazonThread = AmazonThread(keyword)
jdThread = JDThread(keyword)
dangdangThread.start()
amazonThread.start()
jdThread.start()
dangdangThread.join()
amazonThread.join()
jdThread.join()
closeDB()
print('\\\\n爬取已經(jīng)結(jié)束，即將關(guān)閉....')
os.system('pause')
main() 示例截圖：
關(guān)鍵詞：Android 下的部分運(yùn)行結(jié)果（以導(dǎo)出至 Excel ）
總結(jié)
以上就是編程獅(w3cschool.cn) 為你整理的關(guān)于Python爬取當(dāng)當(dāng)、京東、亞馬遜圖書信息代碼實(shí)例的全部內(nèi)容，希望對大家有所幫助。
