使用BeautifulSoup高效抓取多URL数据：避免覆盖与处理分页

for ending in endings:
    url = base_url + ending
    r = requests.get(url)
    soup = BeautifulSoup(r.content, 'html.parser')
    results = soup.find('div', id='container') # 每次循环都会重新赋值
# ...
titles = results.find_all('h2', class_='text_header') # 每次循环都会重新赋值
# ...
for title in titles:
    print(title, end="\n"*2) # 仅打印当前（最后一次循环）的titles

import requests
from bs4 import BeautifulSoup
import pandas as pd # 导入Pandas用于数据整理
import time # 导入time模块用于设置请求间隔

# 定义基础URL和航空公司后缀
base_url = 'https://www.airlinequality.com/airline-reviews/'
endings = ['american-airlines', 'delta-air-lines', 'united-airlines',
           'southwest-airlines', 'alaska-airlines']

# 初始化一个空列表，用于存储所有抓取到的评论数据
all_reviews_data = []

# 遍历每个航空公司的URL后缀
for ending in endings:
    url = base_url + ending
    print(f"正在抓取 {ending} ({url}) 的评论数据...")

    try:
        # 发送HTTP GET请求，并设置超时时间
        r = requests.get(url, timeout=10)
        # 检查请求是否成功（状态码200）
        r.raise_for_status() 
    except requests.exceptions.RequestException as e:
        print(f"请求 {url} 失败: {e}，跳过当前航空公司。")
        time.sleep(1) # 请求失败也稍作等待
        continue # 跳过当前URL，继续处理下一个

    # 使用BeautifulSoup解析网页内容
    soup = BeautifulSoup(r.content, 'html.parser')

    # 使用CSS选择器更精确地定位评论区块
    # 'article[itemprop="review"]' 通常能定位到每个独立的评论容器
    reviews = soup.select('article[itemprop="review"]')

    if not reviews:
        print(f"在 {url} 未找到评论。")
        time.sleep(1)
        continue

    for review_item in reviews:
        # 提取评论标题
        title_element = review_item.h2
        title = title_element.text.strip() if title_element else "N/A"

        # 提取评论星级评分
        rating_element = review_item.select_one('span[itemprop="ratingValue"]')
        rating = rating_element.text.strip() if rating_element else "N/A"

        # 将提取到的数据以字典形式添加到列表中
        all_reviews_data.append({
            'airline': ending,
            'title': title,
            'rating': rating
        })

    time.sleep(2) # 设置请求间隔，避免对服务器造成过大压力

print("\n所有航空公司评论数据抓取完成。")

# 将列表转换为Pandas DataFrame，便于后续的数据分析、筛选和导出
df = pd.DataFrame(all_reviews_data)
print("部分抓取结果：")
print(df.head())
print(f"\n共抓取到 {len(df)} 条评论数据。")

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time

base_url = 'https://www.airlinequality.com' # 注意这里是网站的根域名
endings = ['american-airlines', 'delta-air-lines', 'united-airlines',
           'southwest-airlines', 'alaska-airlines']

all_reviews_data_paginated = []

# 遍历每个航空公司
for ending in endings:
    # 构建当前航空公司的第一个页面的URL
    # 注意：网站分页URL结构可能不同，需根据实际情况调整
    current_page_url = f'{base_url}/airline-reviews/{ending}/page/1/?sortby=post_date%3ADesc&pagesize=100'
    page_num = 1
    print(f"\n开始抓取航空公司: {ending} 的多页评论数据...")

    while True:
        print(f"  正在抓取 {ending} 的第 {page_num} 页: {current_page_url}")

        try:
            r = requests.get(current_page_url, timeout=15)
            r.raise_for_status()
        except requests.exceptions.RequestException as e:
            print(f"  请求 {current_page_url} 失败: {e}，跳过当前航空公司剩余页面。")
            break # 当前页面失败，跳出内层循环，尝试下一个航空公司

        soup = BeautifulSoup(r.content, 'html.parser')

        reviews = soup.select('article[itemprop="review"]')
        if not reviews:
            print(f"  在 {ending} 第 {page_num} 页未找到评论，可能已到达最后一页或页面结构有变。")
            break # 如果当前页面没有评论，则认为已到末尾

        for review_item in reviews:
            title_element = review_item.h2
            rating_element = review_item.select_one('span[itemprop="ratingValue"]')

            title = title_element.text.strip() if title_element else "N/A"
            rating = rating_element.text.strip() if rating_element else "N/A"

            all_reviews_data_paginated.append({
                'airline': ending,
                'title': title,
                'rating': rating,
                'page': page_num # 记录数据来源页码
            })

        # 查找“下一页”链接
        # 示例中提供的是 'article.comp_reviews-pagination ul li