优化Python语言评估器：加速英文单词检测性能

async def count_non_english_words(self, words):
    english_words = await self.load_english_words()
    # 核心瓶颈：对于每个输入单词，遍历整个英文词典进行前缀匹配
    return sum(1 for word in words if not any(english_word.startswith(word) for english_word in english_words))

import re
from collections import Counter

class LanguageEvaluator:
    def __init__(self, english_words_file='words.txt', min_word_len=4, min_non_english_count=4):
        self.min_word_len = min_word_len
        self.file_path = english_words_file
        self.min_non_english_count = min_non_english_count
        self.english_words = set()
        self.english_prefix_regexp = None # 新增：用于存储编译后的正则表达式

    async def load_english_words(self):
        if not self.english_words:
            with open(self.file_path, 'r', encoding='utf-8') as file:
                self.english_words = {word.strip().lower() for word in file}
            # 优化：构建并编译正则表达式
            # 使用re.escape确保单词中的特殊字符被正确转义
            # 使用'^(' 和 ')' 包裹，确保匹配的是整个单词的前缀
            self.english_prefix_regexp = re.compile('^(' + '|'.join(re.escape(w) for w in self.english_words) + ')')
        return self.english_words

    def is_english_word(self, word):
        # 使用编译好的正则表达式进行匹配
        return self.english_prefix_regexp.search(word) is not None

    async def count_non_english_words(self, words):
        await self.load_english_words() # 确保正则表达式已编译
        # 使用优化的is_english_word方法
        return sum(not self.is_english_word(word) for word in words)

import re
from collections import Counter

class LanguageEvaluator:
    def __init__(self, english_words_file='words.txt', min_word_len=4, min_non_english_count=4):
        self.min_word_len = min_word_len
        self.file_path = english_words_file
        self.min_non_english_count = min_non_english_count
        self.english_words = set()
        self.english_prefix_regexp = None # 用于存储编译后的正则表达式

    async def load_english_words(self):
        """
        异步加载英文单词列表，并编译成正则表达式以供后续高效匹配。
        """
        if not self.english_words:
            with open(self.file_path, 'r', encoding='utf-8') as file:
                self.english_words = {word.strip().lower() for word in file}
            # 构建并编译正则表达式。
            # re.escape确保单词中的特殊字符被正确转义。
            # '^(' 和 ')' 包裹，使得正则表达式匹配输入单词是否以字典中的某个单词为前缀。
            self.english_prefix_regexp = re.compile('^(' + '|'.join(re.escape(w) for w in self.english_words) + ')')
        return self.english_words

    async def preprocess_text(self, text):
        """
        预处理文本，提取符合条件的单词。
        """
        words = re.findall(r'\b\w+\b', text.lower())
        return [word for word in words if len(word) >= self.min_word_len and not word.startswith('@') and not re.match(r'^https?://', word)]

    def is_english_word(self, word):
        """
        判断一个单词是否为英文（即是否以字典中的某个单词为前缀）。
        此方法使用预编译的正则表达式，效率极高。
        """
        if self.english_prefix_regexp is None:
            # 如果正则表达式尚未加载，则抛出错误或触发加载
            raise RuntimeError("English words dictionary and regex not loaded. Call load_english_words first.")
        return self.english_prefix_regexp.search(word) is not None

    async def count_non_english_words(self, words):
        """
        计算非英文单词的数量。
        此方法利用优化的is_english_word，大幅提升性能。
        """
        await self.load_english_words() # 确保字典和正则表达式已加载
        return sum(not self.is_english_word(word) for word in words)

    async def is_english_custom(self, text):
        """
        评估给定文本是否主要为英文。
        """
        words_in_text = await self.preprocess_text(text)
        non_english_count = await self.count_non_english_words(words_in_text)
        print(f"Non-English words count: {non_english_count}")
        return non_english_count <= self.min_non_english_count

    async def count_duplicate_words(self, text):
        """
        计算文本中的重复单词数量。
        """
        words = await self.preprocess_text(text)
        word_counts = Counter(words)
        duplicate_count = sum(
            count - 1 for count in word_counts.values() if count > 1)
        return duplicate_count

# 示例使用 (假设words.txt存在)
# async def main():
#     evaluator = LanguageEvaluator(english_words_file='words.txt')
#     test_text = "This is an example message with many words, including some non-english ones like bonjour and grazie." * 10
#     start_time = time.time()
#     is_eng = await evaluator.is_english_custom(test_text)
#     end_time = time.time()
#     print(f"Is English: {is_eng}, Time taken: {end_time - start_time:.2f} seconds")

# if __name__ == "__main__":
#     import asyncio
#     import time
#     # 创建一个简单的words.txt文件用于测试
#     with open('words.txt', 'w', encoding='utf-8') as f:
#         f.write("apple\nbanana\norange\ngrape\nhello\nworld\npython\nprogramming\n")
#     asyncio.run(main())