Python + selenium 爬虫,点击下一页后,页面依然显示本页的数据?

Python + selenium 爬虫,点击下一页后,页面依然显示本页的数据?

用代码里换页的部分,对页面中页码的输入框进行清空,输入页码,点击跳页的操作,没有任何报错,但是页面的数据始终是第一页的数据,请问要怎么处理?
代码:

import time

from datetime import datetime

from selenium import webdriver

# from selenium.webdriver import ChromeOptions

from selenium.webdriver.chrome.options import Options

CHROME_DRIVER = 'chromedriver.exe'

URL_ = 'https://red.library.sh.cn/searchInstance'

TXT_FILE = '文献.txt' # TODO

MAX_PAGE = 851 # TODO

MAX_ITEM = 8505 # TODO

class RedLib:

def __init__(self):

self.chrome_options = Options()

# self.chrome_options.add_argument('--headless')

self.chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])

self.chrome_options.add_experimental_option('useAutomationExtension', False)

self.chrome_options.add_argument("--disable-web-security")

prefs = {"profile.managed_default_content_settings.images": 2}

self.chrome_options.add_experimental_option("prefs", prefs)

self.driver = webdriver.Chrome(executable_path=CHROME_DRIVER, options=self.chrome_options)

self.driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {

"source": """

Object.defineProperty(navigator, 'webdriver', {

get: () => undefined

})

"""

})

def handle_info(self, start_page):

self.driver.get(URL_)

time.sleep(5)

for now_page in range(start_page, 6): # TODO MAX_PAGE + 1

print(f'\n当前正处理第{now_page}页,{now_page / MAX_PAGE * 100:.2f}%,{str(datetime.now()).split(" ")[1].split(".")[0]}')

time.sleep(2)

# 换页操作

self.driver.find_element_by_xpath('//input[@class="searchInput"]').clear()

self.driver.find_element_by_xpath('//input[@class="searchInput"]').send_keys(f'{now_page}')

self.driver.find_element_by_xpath('//div[@class="pagination"]/button').click()

time.sleep(2)

items = 5 if now_page == MAX_PAGE else 10

for item in range(1, items + 1):

info = self.driver.find_element_by_xpath(f'//div[@class="s_right"]/div[{item}]/div[@class="book"]').text

info_text = info.replace('\n', '&&').split('&&更多版本信息')[0]

print(item, info_text)

with open(TXT_FILE, 'a', encoding='utf-8') as tf:

tf.write(info_text)

tf.write('\n')

# self.driver.find_element_by_xpath('//button[@class="btn-next"]').click()

self.driver.quit()

if __name__ == '__main__':

start_page = 1

redlib = RedLib()

redlib.handle_info(start_page)


回答:

说明你没正确输入页码 或者 没有执行到 点击跳页, 先确定第一步 输入页码后拿一下页面数据,第二步 直接在原网页中f12控制台中输入跳页代码

以上是 Python + selenium 爬虫,点击下一页后,页面依然显示本页的数据? 的全部内容, 来源链接: utcz.com/p/937944.html

回到顶部