Scrapy无法跳转到下一页

  • Z时代
  • 2024-03-09
  • 分类：IT

Scrapy无法跳转到下一页

-- coding: utf-8 --

import scrapy
from mySpider.items import MyspiderItem

class ItcastSpider(scrapy.Spider):

name = 'itcast'
allowed_domains = ['dsxxcx.zstu.edu.cn']
start_urls = ['http://dsxxcx.zstu.edu.cn/master/index.php?r=site/college&college=%E4%BF%A1%E6%81%AF%E5%AD%A6%E9%99%A2']
def parse(self, response):
    for each in response.xpath('//tr'):
        item = MyspiderItem()
        name = each.xpath("./td/a/text()").extract()
        title = each.xpath("./td[2]/text()").extract()
        info = each.xpath("./td[3]/text()").extract()
        item['name']=name[0]if name else None
        item['title']=title[0]if title else None
        item['info']=info[0]if info else None
        yield item
    next_page = response.xpath("//ul/li[@class='next']/a/@href").extract_first()
    if next_page is not None:
        next_page = response.urljoin(next_page)
        yield scrapy.Request(start_urls=next_page, callback=self.parse,dont_filter=True)

回答：

已经解决了，下一页网址爬下来不全，要把缺的那部分给他补上，补上域就好了

以上是 Scrapy无法跳转到下一页 的全部内容， 来源链接： utcz.com/p/937851.html

回到顶部