Python文本语言评估优化：利用正则表达式加速大规模词典匹配

import re
from collections import Counter

class LanguageEvaluator:
    def __init__(self, english_words_file='words.txt', min_word_len=4, min_non_english_count=4):
        self.min_word_len = min_word_len
        self.file_path = english_words_file
        self.min_non_english_count = min_non_english_count
        self.english_words = set()
        self.english_prefix_regexp = None # 用于优化方案

    async def load_english_words(self):
        if not self.english_words:
            with open(self.file_path, 'r', encoding='utf-8') as file:
                self.english_words = {word.strip().lower() for word in file}
        return self.english_words

    async def preprocess_text(self, text):
        words = re.findall(r'\b\w+\b', text.lower())
        return [word for word in words if len(word) >= self.min_word_len and not word.startswith('@') and not re.match(r'^https?://', word)]

    async def count_non_english_words(self, words):
        english_words = await self.load_english_words()
        # 原始的低效逻辑
        return sum(1 for word in words if not any(english_word.startswith(word) for english_word in english_words))

    # ... 其他方法（is_english_custom, count_duplicate_words）

not any(english_word.startswith(word) for english_word in english_words)

import re
from collections import Counter

class LanguageEvaluator:
    def __init__(self, english_words_file='words.txt', min_word_len=4, min_non_english_count=4):
        self.min_word_len = min_word_len
        self.file_path = english_words_file
        self.min_non_english_count = min_non_english_count
        self.english_words = set()
        self.english_prefix_regexp = None # 用于存储编译后的正则表达式

    async def load_english_words(self):
        if not self.english_words:
            with open(self.file_path, 'r', encoding='utf-8') as file:
                self.english_words = {word.strip().lower() for word in file}
            # 优化：将所有英语词汇编译成一个正则表达式
            # 注意：re.escape() 用于转义特殊字符，防止它们被解释为正则表达式元字符
            # ^(word1|word2|...) 匹配以任意一个英语单词开头
            self.english_prefix_regexp = re.compile('^(' + '|'.join(re.escape(w) for w in self.english_words) + ')')
        return self.english_words

    def is_english_word(self, word):
        """
        检查一个词是否以任何一个英语词典中的词开头。
        """
        # 使用编译好的正则表达式进行匹配
        return self.english_prefix_regexp.search(word) is not None

    async def preprocess_text(self, text):
        words = re.findall(r'\b\w+\b', text.lower())
        return [word for word in words if len(word) >= self.min_word_len and not word.startswith('@') and not re.match(r'^https?://', word)]

    async def count_non_english_words(self, words):
        await self.load_english_words() # 确保词典和正则表达式已加载
        # 优化：使用正则表达式进行高效匹配
        return sum(not self.is_english_word(word) for word in words)

    async def is_english_custom(self, text):
        words_in_text = await self.preprocess_text(text)
        non_english_count = await self.count_non_english_words(words_in_text)
        print(f"Non-English words count: {non_english_count}")
        return non_english_count <= self.min_non_english_count

    async def count_duplicate_words(self, text):
        words = await self.preprocess_text(text)
        word_counts = Counter(words)
        duplicate_count = sum(
            count - 1 for count in word_counts.values() if count > 1)
        return duplicate_count