python爬虫 去哪网数据分析

#1.下载去哪网数据
import requests
import csv
import time
from bs4 import BeautifulSoup
#定义数据字典
subjects = {"游乐场","自然风光","文化古迹","城市观光"}
headers = {"User_Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36"}
#准备CSV文档
csv_file = open("data.csv","w+",encoding="utf-8-sig",newline="")
csv_writer = csv.writer(csv_file)
#将CSV文件的头写入到文件中
csv_writer.writerow(["景点名称","所在城市","景点类型","景点级别","景点地址","景点热度"])
#定义一个用来请求URL函数
def get_html_by_url(url):try:response = requests.get(url=url,headers=headers)if response.status_code == 200:return response.textexcept:print("爬虫数据错误！")
#解析HTML，并将数据插存入到CSV文件中
def parse_html(html,subject):#先判断HTML中是否有数据，则直接返回Noneif not html:return None#开始解析，创建BS类对象soupsoup = BeautifulSoup(html,"html.parser")#通过HTML代码的分析，我们发现所有的有效数据都在一个叫#所以我们要通过BS先查找search-list节点search_list = soup.find(id="search-list")#将search-list节点中所有的子节点获取到，形成一个列表items = search_list.find_all("div",class_="sight_item")#通过循环取出每一个景点中的信息数据for item in items:#获取景点名称name = item["data-sight-name"]#获取景点类别category = item["data-sight-category"]#获取景点城市city = item["data-districts"]#获取景点地址address = item["data-address"]#获取景点级别level = item.find("span",class_="level")level = level.text if level else ""#获取景点热度hot = item.find("span",class_="product_star_level")hot = hot.text if hot else ""#将数据写入到CSV文件中csv_writer.writerow([name,city,category,level,address,hot])
#开始执行爬虫
def start():for subject in subjects:for page in range(10):page+=1url = "https://piao.qunar.com/ticket/list.htm?keyword=热门景点&subject={}&page={}".format(subject,page)print(url)#获取指定url页面的html代码html = get_html_by_url(url)#解析htmlparse_html(html,subject)#每次下载数据之后，暂停3秒time.sleep(3)
if __name__ == "__main__":start()
#2.对数据进行清洗，将有效数据保存到CSV文件中
#3.通过Pandas对CSV数据进行分析