这次运用数据来存储，并从中提取数据

成果展示

部分成果1.png

部分成果2.png

代码

import requests

import time

from bs4 import BeautifulSoup

from pymongo import MongoClient

client = MongoClient('localhost', 27017)

PhoneNum = client['PhoneNum']

url_list = PhoneNum['url_list']

phone_info = PhoneNum['phone_info']]

# spider 1

def get_phone_link(pages):

url = 'http://bj.58.com/shoujihao/pn{}/'.format(str(pages))

wb_data = requests.get(url)

time.sleep(1)

soup = BeautifulSoup(wb_data.text, 'lxml')

links = soup.select('li a.t')

numbers = soup.select('strong')

lk = soup.select('#infocont > span > b')

if '0' not in lk:

for number,link in zip(numbers,links):

data = {

'number': number.get_text(),

'PhoneLink' : link.get('href').split('?')[0]

}

print(data)

url_list.insert_one(data)

else:

pass

# spider 2

def get_item_info(url):

wb_data = requests.get(url)

time.sleep(1)

soup = BeautifulSoup(wb_data.text,'lxml')

titles = soup.select('div.col_sub.mainTitle > h1')

prices = soup.select('div.col_sub.sumary >ul > li > div.su_con > span')

for title, price in zip(titles,prices):

data = {

'title' :( title.get_text()).replace("\n","").replace(" ",""),

'price' : (price.get_text()).replace("\n","").replace(" ","")

}

print(data)

phone_info.insert_one(data)

for page in range(1, 200):

get_phone_link(page)

for info in url_list.find():

url = info["url"]

get_item_info(url)

总结

遇到需要采集大量的数据，最好的方式就是将功能分离开来。一次只执行一个动作