使用Selenium抓取Google地图完整评论：解决“更多”内容展开问题

from selenium import webdriver
from selenium.webdriver.chrome.service import Service as ChromeService
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException, StaleElementReferenceException
from selenium.webdriver.common.keys import Keys
import time

# 定义目标URL
# 请根据实际需要替换为您的Google地图商家评论页URL
TARGET_URL = 'https://www.google.com/maps/place/Henn+na+Hotel+Tokyo+Asakusa+Tawaramachi/@35.7081692,139.7888494,17z/data=!4m22!1m12!3m11!1s0x60188f36ab21f05b:0x9241dab287ff62c9!2sHenn+na+Hotel+Tokyo+Asakusa+Tawaramachi!5m2!4m1!1i2!8m2!3d35.7081692!4d139.7914243!9m1!1b1!16s%2Fg%2F11h0gzlhht!3m8!1s0x60188f36ab21f05b:0x9241dab287ff62c9!5m2!4m1!1i2!8m2!3d35.7081692!4d139.7914243!16s%2Fg%2F11h0gzlhht?entry=ttu'

# 配置Chrome选项
options = Options()
options.add_argument('--no-sandbox') # 禁用沙盒模式，在某些Linux环境下可能需要
options.add_argument('--disable-dev-shm-usage') # 禁用/dev/shm使用，在某些Docker/Linux环境下可能需要
# options.add_argument('--headless') # 启用无头模式，不显示浏览器界面，适合服务器环境

# 初始化WebDriver
driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=options)
driver.get(TARGET_URL)

# 等待页面加载完成，例如等待H1标题出现
try:
    WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, 'h1')))
    print("页面H1标题已加载。")
except TimeoutException:
    print("页面H1标题未在指定时间内加载。")

def accept_cookie_policy():
    """尝试点击Cookie政策接受按钮"""
    try:
        # 查找所有按钮，并根据文本内容判断是否为“Accept all”
        buttons = driver.find_elements(By.TAG_NAME, 'button')
        for button in buttons:
            if button.text == "Accept all":
                print("找到并点击 'Accept all' Cookie按钮。")
                button.click()
                time.sleep(2) # 等待弹窗消失
                return
    except (TimeoutException, NoSuchElementException):
        print("Cookie政策按钮未找到或无法点击。")

accept_cookie_policy()

# 查找并点击“评论”按钮
try:
    all_buttons = driver.find_elements(By.TAG_NAME, 'button')
    for button in all_buttons:
        if button.text == 'Reviews': # 根据页面语言，可能是“评论”
            print("找到并点击 'Reviews' 按钮。")
            button.click()
            time.sleep(3) # 等待评论区加载
            break
except NoSuchElementException:
    print("未找到 'Reviews' 按钮。")

# 找到评论区的可滚动面板
# 经验证，该XPath通常指向包含评论的滚动容器
SCROLLABLE_REVIEWS_PANEL_XPATH = "//div[contains(@class, 'm6QErb') and contains(@class, 'DxyBCb') and contains(@class, 'kA9KIf') and contains(@class, 'dS8AEf')]"
try:
    scrollable_reviews_panel = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.XPATH, SCROLLABLE_REVIEWS_PANEL_XPATH))
    )
    print("找到评论滚动面板。")
except TimeoutException:
    print("未在指定时间内找到评论滚动面板。请检查XPath。")
    driver.quit()
    exit()

last_height = driver.execute_script("return arguments[0].scrollHeight", scrollable_reviews_panel)
print("开始滚动加载所有评论...")
while True:
    # 滚动到评论面板的底部
    driver.execute_script("arguments[0].scrollTo(0, arguments[0].scrollHeight);", scrollable_reviews_panel)
    time.sleep(2) # 等待新评论加载
    new_height = driver.execute_script("return arguments[0].scrollHeight", scrollable_reviews_panel)
    if new_height == last_height:
        # 如果滚动高度不再变化，说明已加载所有评论
        print("所有评论已加载。")
        break
    last_height = new_height

# 展开所有“更多”按钮
print("开始展开所有 '更多' 评论...")
# 'jftiEf' 通常是单个评论卡片的类名
review_containers = driver.find_elements(By.CLASS_NAME, 'jftiEf')
for i, container in enumerate(review_containers):
    try:
        # 'w8nwRe' 通常是“更多”按钮的类名
        more_button = container.find_element(By.CLASS_NAME, "w8nwRe")
        if more_button.text == "More": # 确保是“More”按钮
            more_button.click()
            time.sleep(0.5) # 短暂延迟，等待评论展开
            # print(f"已展开第 {i+1} 条评论的 '更多' 内容。")
    except NoSuchElementException:
        # 当前评论没有“更多”按钮，跳过
        pass
    except StaleElementReferenceException: