Google地图评论数据抓取：Playwright问题与Selenium解决方案

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import re
import time

# 1. 初始化WebDriver
# 请确保已安装对应浏览器的WebDriver，并将其路径添加到系统PATH中
# 例如：Chrome WebDriver (chromedriver.exe)
driver = webdriver.Chrome()

# 2. 导航到Google地图搜索结果页
# 替换为实际的Google地图搜索URL
driver.get("https://www.google.com/maps/search/restaurants+near+me")

# 确保页面加载完成，可以等待某个主要元素出现
WebDriverWait(driver, 20).until(
    EC.presence_of_element_located((By.XPATH, '//div[@role="main"]'))
)

scraped_data = []

# 3. 循环处理每个商家列表项
# 注意：Google地图的列表是动态加载的，可能需要滚动才能加载所有列表项
# 这里简化为只处理当前可见的列表项。对于大规模抓取，需要实现滚动加载逻辑。
# 初始获取所有列表项，这个XPath需要根据实际页面结构调整
# 例如：每个商家列表项可能是一个带有特定role和jsaction属性的div
initial_listing_xpath = '//div[@role="article" and @jsaction]'

for i in range(10): # 假设我们想抓取前10个商家，或者直到没有更多商家
    try:
        # 避免Stale Element Reference：每次循环重新获取列表元素
        # 这很重要，因为点击详情页并返回后，原始的elements对象可能失效
        WebDriverWait(driver, 10).until(
            EC.presence_of_all_elements_located((By.XPATH, initial_listing_xpath))
        )
        listing_elements = driver.find_elements(By.XPATH, initial_listing_xpath)

        if i >= len(listing_elements):
            print(f"已处理完所有可见的 {len(listing_elements)} 个商家。")
            break # 如果没有更多列表项，则退出循环

        current_listing = listing_elements[i]

        # 滚动到当前列表项，确保其可见并可点击
        driver.execute_script("arguments[0].scrollIntoView(true);", current_listing)
        time.sleep(0.5) # 给滚动一点时间

        # 点击列表项以打开详细信息面板
        current_listing.click()

        # 4. 等待详细信息面板加载
        # 识别详细信息面板中独有的元素，例如商家名称
        detail_panel_name_xpath = '//div[contains(@class, "qBF1Pd fontHeadlineSmall ")]'
        WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.XPATH, detail_panel_name_xpath))
        )
        time.sleep(1) # 额外等待，确保所有内容渲染完毕

        business_info = {}

        # 5. 从详细信息面板中提取数据
        # 这些XPath应该定位详细信息面板中的元素，而不是依赖原始列表索引
        try:
            business_info['name'] = driver.find_element(By.XPATH, detail_panel_name_xpath).text
        except:
            business_info['name'] = "N/A"

        try:
            business_info['address'] = driver.find_element(By.XPATH, '//button[@data-item-id="address"]//div[contains(@class, "fontBodyMedium")]').text
        except:
            business_info['address'] = "N/A"

        try:
            business_info['website'] = driver.find_element(By.XPATH, '//a[@data-item-id="authority"]//div[contains(@class, "fontBodyMedium")]').text
        except:
            business_info['website'] = "N/A"

        try:
            business_info['phone_number'] = driver.find_element(By.XPATH, '//button[contains(@data-item-id, "phone:tel:")]//div[contains(@class, "fontBodyMedium")]').text
        except:
            business_info['phone_number'] = "N/A"

        try:
            # 这个XPath可能需要根据实际页面调整，它看起来是针对特定布局的
            business_info['category'] = driver.find_element(By.XPATH, '//*[@id="QA0Szd"]/div/div/div[1]/div[3]/div/div[1]/div/div/div[2]/div[2]/div/div[1]/div[2]/div/div[2]/span/span/button').text
        except:
            business_info['category'] = "N/A"

        # 提取评论数据：关键步骤
        try:
            # 定位详细信息面板中的评论元素，它通常有一个role="img"和aria-label属性
            reviews_span_element = WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.XPATH, '//span[@role="img" and contains(@aria-label, "stars")]'))
            )
            reviews_label = reviews_span_element.get_attribute("aria-label")
            print(f"Reviews Label: {reviews_label}")

            # 使用正则表达式解析评论标签
            match = re.match(r'([\d.]+) stars ([\d,]+) Reviews', reviews_label)
            if match:
                business_info['reviews_average'] = float(match.group(1))
                business_info['reviews_count'] = int(re.sub(',', '', match.group(2)))
            else:
                business_info['reviews_average'] = None
                business_info['reviews_count'] = None
        except Exception as e:
            print(f"无法获取评论数据 for {business_info.get('name', 'N/A')}: {e}")
            business_info['reviews_average'] = None
            business_info['reviews_count'] = None

        scraped_data.append(business_info)

        # 6. 返回到列表视图
        # 寻找并点击返回按钮，这个XPath需要根据实际页面调整
        # Google地图的返回按钮通常有一个aria-label="返回"或类似属性
        try:
            back_button_xpath = '//button[@aria-label="返回" or @data-tooltip-id="返回"]' # 示例XPath
            back_button = WebDriverWait(driver, 10).until(
                EC.element_to_be_clickable((By.XPATH, back_button_xpath))
            )
            back_button.click()
            # 返回后，等待列表重新出现，确保页面状态正确
            WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.XPATH, initial_listing_xpath))
            )
            time.sleep(1) # 额外等待，确保列表完全渲染
        except Exception as e:
            print(f"无法