1.创建项目

在控制台通过scrapy startproject 创建项目

我们通过scrapy startproject haozu 创建爬虫项目

2.创建爬虫文件

在控制台 进入spiders 文件夹下 通过scrapy genspider <网站域名>

scrapy genspider haozu_xzl www.haozu.com 创建爬虫文件

3.在爬虫文件中 haozu_xzl.py写代码 python version=3.6.0

-- coding: utf-8 --

import scrapy

import requests

from lxml import html

etree =html.etree

from ..items import HaozuItem

import random

class HaozuXzlSpider(scrapy.Spider):

# scrapy crawl haozu_xzlname = 'haozu_xzl'# allowed_domains = ['www.haozu.com/sz/zuxiezilou/']start_urls = "http://www.haozu.com/sz/zuxiezilou/"province_list = ['bj', 'sh', 'gz', 'sz', 'cd', 'cq', 'cs','dl','fz','hz','hf','nj','jian','jn','km','nb','sy', 'su','sjz','tj','wh','wx','xa','zz']

def start_requests(self):user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2'headers = {'User-Agent': user_agent}for s in self.province_list:start_url = "http://www.haozu.com/{}/zuxiezilou/".format(s)# 包含yield语句的函数是一个生成器，每次产生一个值，函数被冻结，被唤醒后再次产生一个值yield scrapy.Request(url=start_url, headers=headers, method='GET', callback=self.parse,  meta={"headers": headers,"city":s})def parse(self, response):lists = response.body.decode('utf-8')selector = etree.HTML(lists)elem_list = selector.xpath('/html/body/p[2]/p[2]/p/dl[1]/dd/p[2]/p[1]/a')print(elem_list,type(elem_list))for elem in elem_list[1:-1]:try:district = str(elem.xpath("text()"))[1:-1].replace("'",'')# district.remove(district[0])# district.pop()print(district,type(district))district_href =str(elem.xpath("@href"))[1:-1].replace("'",'')# district_href.remove(district_href[0])print(district_href,type(district_href))elem_url ="http://www.haozu.com{}".format(district_href)print(elem_url)yield scrapy.Request(url=elem_url, headers=response.meta["headers"], method='GET', callback=self.detail_url, meta={"district": district,"url":elem_url,"headers":response.meta["headers"],"city":response.meta["city"]})except Exception as e:print(e)passdef detail_url(self, response):print("===================================================================")for i in range(1,50):# 组建urlre_url = "{}o{}/".format(response.meta["url"],i)print(re_url)try:response_elem = requests.get(re_url,headers=response.meta["headers"])seles= etree.HTML(response_elem.content)sele_list = seles.xpath("/html/body/p[3]/p[1]/ul[1]/li")for sele in sele_list:href = str(sele.xpath("./p[2]/h1/a/@href"))[1:-1].replace("'",'')print(href)href_url = "http://www.haozu.com{}".format(href)print(href_url)yield scrapy.Request(url=href_url, headers=response.meta["headers"], method='GET', callback=self.final_url, meta={"district": response.meta["district"],"city":response.meta["city"]})except Exception as e:print(e)passdef final_url(self,response):try:body = response.body.decode('utf-8')sele_body = etree.HTML(body)#获取价格 名称 地址item = HaozuItem()item["city"]= response.meta["city"]item['district']=response.meta["district"]item['addr'] = str(sele_body.xpath("/html/body/p[2]/p[2]/p/p/p[2]/span[1]/text()[2]"))[1:-1].replace("'",'')item['title'] = str(sele_body.xpath("/html/body/p[2]/p[2]/p/p/p[1]/h1/span/text()"))[1:-1].replace("'",'')price = str(sele_body.xpath("/html/body/p[2]/p[3]/p[2]/p[1]/span/text()"))[1:-1].replace("'",'')price_danwei=str(sele_body.xpath("/html/body/p[2]/p[3]/p[2]/p[1]/p/p/i/text()"))[1:-1].replace("'",'')print(price price_danwei)item['price']=price price_danweiyield itemexcept Exception as e:print(e)pass

4.修改items.py 文件

-- coding: utf-8 --

Define here the models for your scraped items

See documentation in:

https://doc.scrapy.org/en/latest/topics/items.html

import scrapy

class HaozuItem(scrapy.Item):

# define the fields for your item here like:# name = scrapy.Field()city = scrapy.Field()district =scrapy.Field()title = scrapy.Field()addr =scrapy.Field()price = scrapy.Field()

5修改settings.py

打开

ITEM_PIPELINES = {

'haozu.pipelines.HaozuPipeline': 300,

}

6 修改pipelines.py文件 这里可以自定义存储文件格式

-- coding: utf-8 --

Define your item pipelines here

Don't forget to add your pipeline to the ITEM_PIPELINES setting

See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import csv

class HaozuPipeline(object):

def process_item(self, item, spider):f = open('./xiezilou2.csv', 'a ',encoding='utf-8',newline='')write = csv.writer(f)write.writerow((item['city'],item['district'],item['addr'],item['title'],item['price']))print(item)return item

7.启动框架

在控制台 输入 scrapy crawl haozu_xzl 启动程序