Numpy数组与Python列表存储大小深度解析：优化与误区

import random
import pickle
import numpy as np

# 假设 all_games 包含一些独特的7元素列表
# 例如：all_games = [[float(i) for i in range(7)] for _ in range(100)]
# 如果 all_games 元素数量远小于 SAMPLE * DRAW，则重复引用的可能性很高

def sample_games_list_pickle(all_games, file_name='sampled_list.pkl'):
    DRAW = 10000
    SAMPLE = 10000
    # 这里的 random.choice(all_games) 可能会重复选择 all_games 中的同一个子列表对象
    sampled_data = [[random.choice(all_games) for _ in range(DRAW)] for _ in range(SAMPLE)]

    with open(file_name, 'wb') as file:
        pickle.dump(sampled_data, file)
    print(f"Pickled list saved to {file_name}")

# 示例调用 (需要先定义 all_games)
# all_games_example = [[float(j) for j in range(7)] for _ in range(100)] # 假设只有100个独特的子列表
# sample_games_list_pickle(all_games_example, 'sampled_list_optimized.pkl')

import random
import pickle

def sample_games_list_pickle_with_copy(all_games, file_name='sampled_list_copied.pkl'):
    DRAW = 10000
    SAMPLE = 10000
    # 显式复制子列表，强制 pickle 存储所有副本
    sampled_data = [[random.choice(all_games).copy() for _ in range(DRAW)] for _ in range(SAMPLE)]

    with open(file_name, 'wb') as file:
        pickle.dump(sampled_data, file)
    print(f"Pickled list with copies saved to {file_name}")

# 示例调用 (需要先定义 all_games)
# all_games_example = [[float(j) for j in range(7)] for _ in range(100)]
# sample_games_list_pickle_with_copy(all_games_example, 'sampled_list_copied.pkl')

import numpy as np

def sample_games_numpy_compressed(all_games_list, file_name='sampled_data_compressed.npz'):
    # 将原始Python列表转换为Numpy数组
    all_games_np = np.array(all_games_list, dtype=np.float16)
    DRAW = 10000
    SAMPLE = 10000

    rng = np.random.default_rng() # 推荐使用新的随机数生成器
    # 生成随机索引
    sampled_indices = rng.choice(all_games_np.shape[0], size=(SAMPLE, DRAW), replace=True)
    # 根据索引获取采样数据
    sampled_data = all_games_np[sampled_indices]

    # 使用 savez_compressed 保存压缩后的Numpy数组
    np.savez_compressed(file_name, data=sampled_data)
    print(f"Compressed Numpy array saved to {file_name}")

# 示例调用 (需要先定义 all_games_list)
# all_games_list_example = [[float(j) for j in range(7)] for _ in range(100000)] # 假设有10万个独特的子列表
# sample_games_numpy_compressed(all_games_list_example, 'sampled_data_compressed.npz')