Google 地图评论数据抓取：提升稳定性和准确性

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import re
import time

class Business:
    def __init__(self):
        self.name = "N/A"
        self.address = "N/A"
        self.website = "N/A"
        self.phone_number = "N/A"
        self.category = "N/A"
        self.reviews_average = None
        self.reviews_count = None

def scrape_google_maps_reviews(search_url):
    # 配置 ChromeDriver 服务
    # 请确保您的 ChromeDriver 版本与 Chrome 浏览器版本兼容
    service = Service(executable_path='/path/to/chromedriver') # 替换为您的 ChromeDriver 路径
    options = webdriver.ChromeOptions()
    # options.add_argument('--headless') # 可选：无头模式，不显示浏览器界面
    options.add_argument('--disable-gpu')
    options.add_argument('--no-sandbox')
    options.add_argument('--disable-dev-shm-usage')
    options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")

    driver = webdriver.Chrome(service=service, options=options)
    driver.get(search_url)

    # 显式等待搜索结果加载
    WebDriverWait(driver, 20).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, 'div[role="feed"]'))
    )

    businesses_data = []

    # 模拟滚动以加载更多商家
    # 找到包含商家列表的滚动区域，通常是 role="feed" 的 div
    scrollable_div_xpath = '//*[@id="QA0Szd"]/div/div/div[1]/div[2]/div/div[1]/div/div/div[2]' # 示例XPath，可能需要根据实际页面调整
    try:
        scrollable_div = WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.XPATH, scrollable_div_xpath))
        )
        last_height = driver.execute_script("return arguments[0].scrollHeight", scrollable_div)
        while True:
            driver.execute_script("arguments[0].scrollTop = arguments[0].scrollHeight", scrollable_div)
            time.sleep(2) # 等待新内容加载
            new_height = driver.execute_script("return arguments[0].scrollHeight", scrollable_div)
            if new_height == last_height:
                break
            last_height = new_height
    except Exception as e:
        print(f"滚动加载失败或无滚动区域: {e}")

    # 获取所有商家列表项
    # 注意：这里获取的是列表中的元素，后续点击后，详情页的元素需要重新定位
    listing_elements = driver.find_elements(By.CSS_SELECTOR, 'div[role="feed"] > div > a') 
    print(f"找到 {len(listing_elements)} 个商家列表项。")

    for i in range(len(listing_elements)):
        try:
            # 重新获取列表项，因为点击后页面可能刷新或元素引用失效
            # 也可以尝试存储元素的唯一标识符（如 href），然后通过 href 重新定位
            # 但最简单可靠的方式是每次循环重新获取所有可见列表项，然后点击第i个
            # 注意：这里需要确保点击后，浏览器能正确返回列表视图，或者详情页是侧边栏
            # Google Maps 详情页通常是侧边栏，所以可以直接点击

            # 重新定位当前可见的列表项
            current_listing_elements = driver.find_elements(By.CSS_SELECTOR, 'div[role="feed"] > div > a')
            if i >= len(current_listing_elements):
                print(f"列表项 {i+1} 不再可见，跳过。")
                continue

            listing_to_click = current_listing_elements[i]
            # 获取商家的名称或链接，用于日志和验证
            listing_name = listing_to_click.find_element(By.CSS_SELECTOR, '.qBF1Pd').text if listing_to_click.find_elements(By.CSS_SELECTOR, '.qBF1Pd') else "未知名称"
            print(f"\n尝试点击商家: {listing_name}")

            listing_to_click.click()

            # 显式等待详情页加载完成，例如等待商家名称或评论区域出现
            WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.CSS_SELECTOR, 'div.qBF1Pd.fontHeadlineSmall')) # 商家名称
            )
            WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.CSS_SELECTOR, 'button[data-item-id="reviews"]')) # 评论按钮
            )
            time.sleep(1) # 短暂等待，确保所有动态内容渲染完毕

            business = Business()

            # 在详情页内定位元素并抓取信息
            try:
                business.name = driver.find_element(By.CSS_SELECTOR, 'div.qBF1Pd.fontHeadlineSmall').text
            except:
                pass
            try:
                business.address = driver.find_element(By.CSS_SELECTOR, 'button[data-item-id="address"] div.fontBodyMedium').text
            except:
                pass
            try:
                business.website = driver.find_element(By.CSS_SELECTOR, 'a[data-item-id="authority"] div.fontBodyMedium').text
            except:
                pass
            try:
                business.phone_number = driver.find_element(By.CSS_SELECTOR, 'button[data-item-id^="phone:tel:"] div.fontBodyMedium').text
            except:
                pass
            try:
                # 类别通常在名称下方，可能需要更精确的定位
                category_element = driver.find_element(By.XPATH, '//*[@id="QA0Szd"]/div/div/div[1]/div[3]/div/div[1]/div/div/div[2]/div[2]/div/div[1]/div[2]/div/div[2]/span/span/button')
                business.category = category_element.text
            except:
                pass

            # 尝试滚动详情面板以确保评论元素可见（如果需要）
            # 通常详情面板是可滚动的，评论可能在下方
            detail_panel_xpath = '//*[@id="QA0Szd"]/div/div/div[1]/div[3]/div/div[1]' # 详情面板的示例XPath
            try:
                detail_panel = driver.find_element(By.XPATH, detail_panel_xpath)
                driver.execute_script("arguments[0].scrollTop = arguments[0].scrollHeight", detail_panel)
                time.sleep(1) # 等待滚动完成
            except Exception as e:
                print(f"详情面板滚动失败: {e}")

            # 定位评论元素 (注意：这里不再使用 index，而是直接定位详情页内的评论区域)
            # Google Maps 评论通常在一个带有 role="img" 的 span 中，且其父元素可能是评论按钮
            reviews_span_xpath_in_detail = '//button[contains(@aria-label, "stars")]/span[@role="img"]'
            reviews_element = driver.find_elements(By.XPATH, reviews_span_xpath_in_detail)

            if reviews_element:
                reviews_label = reviews_element[0].get_attribute("aria-label")
                print(f"Reviews Label for {business.name}: {reviews_label}")

                # 使用正则表达式处理评论标签
                match = re.match(r'([\d.]+) stars ([\d,]+) Reviews', reviews_label)
                if match:
                    business.reviews_average = float(match.group(1))
                    business.reviews_count = int(re.sub(',', '', match.group(2)))
                else:
                    print(f"无法解析评论标签: {reviews_label}")
            else:
                print(f"未找到 {business.name} 的评论信息。")

            businesses_data.append(business)

            # 返回到列表视图 (如果详情页是独立页面，则需要 driver.back())
            # 对于 Google Maps 侧边栏详情，通常不需要额外操作，直接点击下一个列表项即可
            # 但为了确保，可以尝试点击一个返回按钮或者等待列表重新可见
            # 如果列表项是动态加载的，每次循环重新获取 listing_elements 是必要的

            # 简单等待，确保页面状态稳定，为下一次点击做准备
            time.sleep(1) 

        except Exception as e:
            print(f"处理第 {i+1} 个商家时发生错误: {e}")
            # 发生错误时，尝试返回列表或刷新页面，然后继续
            # driver.refresh() # 谨慎使用，可能导致当前列表丢失
            time.sleep(2) # 稍作等待，避免连续错误
            continue # 继续下一个商家

    driver.quit()
    return businesses_data

# 示例用法
if __name__ == "__main__":
    search_query = "restaurants in New York"
    # 注意：Google Maps 的 URL 结构可能很复杂，这里只是一个示例
    # 实际应用中，您可能需要先通过搜索框输入查询，然后获取结果页URL
    # 或者直接构建一个包含查询参数的URL
    google_maps_url = f"https://www.google.com/maps/search/{search_query.replace(' ', '+')}"

    scraped_data = scrape_google_maps_reviews(google_maps_url)

    print("\n--- 抓取结果 ---")
    for biz in scraped_data:
        print(f"名称: {biz.name}")
        print(f"地址: {biz.address}")
        print(f"网站: {biz.website}")
        print(f"电话: {biz.phone_number}")
        print(f"类别: {biz.category}")
        print(f"平均评分: {biz.reviews_average}")
        print(f"评论数量: {biz.reviews_count}")
        print("-" * 20)

    print(f"总共抓取了 {len(scraped_data)} 条商家数据。")