# 環境設置和庫導入
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from collections import Counter
import math
from scipy.special import entr
import warnings
warnings.filterwarnings('ignore')

# 設置中文字體（如果遇到顯示問題可以註釋掉）
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
def information_content(probability):
    """
    計算單個事件的信息量
    
    Parameters:
    probability: 事件發生的概率，範圍[0,1]
    
    Returns:
    information: 信息量（比特）
    """
    if probability == 0:
        return float('inf')  # 不可能事件的信息量為無窮大
    return -math.log2(probability)

# 示例計算
print("=== 信息量計算示例 ===")
events = [
    ("太陽從東邊升起", 0.999),
    ("今天會下雨", 0.3),
    ("贏得彩票", 0.0000001),
    ("拋硬幣正面朝上", 0.5)
]

for event_name, prob in events:
    info = information_content(prob)
    print(f"事件: {event_name:20} 概率: {prob:8.6f} 信息量: {info:8.2f} bits")

# 可視化信息量與概率的關係
probabilities = np.linspace(0.001, 1.0, 1000)
information_values = [information_content(p) for p in probabilities]

plt.figure(figsize=(10, 6))
plt.plot(probabilities, information_values, 'b-', linewidth=2, alpha=0.8)
plt.xlabel('概率 P(x)')
plt.ylabel('信息量 I(x) (bits)')
plt.title('信息量與概率的關係\nI(x) = -log₂P(x)')
plt.grid(True, alpha=0.3)

# 標記關鍵點
key_points = [0.1, 0.5, 0.9]
for p in key_points:
    info = information_content(p)
    plt.plot(p, info, 'ro', markersize=8)
    plt.annotate(f'P={p}, I={info:.2f}', (p, info), 
                xytext=(10, 10), textcoords='offset points',
                bbox=dict(boxstyle='round,pad=0.3', facecolor='yellow', alpha=0.7))

plt.tight_layout()
plt.show()

# 環境設置和庫導入
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from collections import Counter
import math
from scipy.special import entr
import warnings
warnings.filterwarnings('ignore')

# 設置中文字體（如果遇到顯示問題可以註釋掉）
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False

def entropy(probabilities):
    """
    計算概率分佈的信息熵
    
    Parameters:
    probabilities: 概率列表，應該和為1
    
    Returns:
    entropy_value: 信息熵值（比特）
    """
    # 輸入驗證
    probabilities = np.array(probabilities)
    if not np.allclose(np.sum(probabilities), 1.0):
        raise ValueError("概率之和必須為1")
    if np.any(probabilities < 0):
        raise ValueError("概率不能為負")
    
    entropy_val = 0.0
    for p in probabilities:
        if p > 0:  # 避免log(0)的情況
            entropy_val -= p * math.log2(p)
    return entropy_val

def entropy_from_labels(labels):
    """
    從標籤數據直接計算熵
    
    Parameters:
    labels: 類別標籤數組
    
    Returns:
    entropy_value: 信息熵值
    """
    counter = Counter(labels)
    total = len(labels)
    probabilities = [count / total for count in counter.values()]
    return entropy(probabilities)

print("=== 信息熵計算示例 ===")

# 不同的概率分佈示例
distributions = {
    "完全確定": [1.0],
    "高度傾斜": [0.9, 0.1],
    "中等傾斜": [0.7, 0.3],
    "公平硬幣": [0.5, 0.5],
    "公平骰子": [1/6, 1/6, 1/6, 1/6, 1/6, 1/6],
    "三類別均勻": [1/3, 1/3, 1/3]
}

print(f"{'分佈類型':<15} {'概率分佈':<40} {'熵值':<8} {'最大可能熵':<12}")
print("-" * 85)

for dist_name, probs in distributions.items():
    h = entropy(probs)
    max_possible = math.log2(len(probs)) if len(probs) > 0 else 0
    efficiency = (h / max_possible * 100) if max_possible > 0 else 0
    
    prob_str = str([round(p, 3) for p in probs])
    print(f"{dist_name:<15} {prob_str:<40} {h:<8.4f} {max_possible:<12.4f}")

# 二分類熵曲線可視化
p_values = np.linspace(0.001, 0.999, 1000)
binary_entropies = [entropy([p, 1-p]) for p in p_values]

plt.figure(figsize=(12, 6))
plt.plot(p_values, binary_entropies, 'r-', linewidth=3, alpha=0.8, label='二分類熵')
plt.axvline(x=0.5, color='blue', linestyle='--', alpha=0.7, 
           label='最大熵點 (p=0.5)')
plt.axhline(y=1.0, color='green', linestyle='--', alpha=0.7, 
           label='最大熵值 (1 bit)')

plt.xlabel('類別1的概率 (p)')
plt.ylabel('熵 H(X) (bits)')
plt.title('二分類信息熵曲線\nH(X) = -[p·log₂(p) + (1-p)·log₂(1-p)]')
plt.legend()
plt.grid(True, alpha=0.3)

# 添加一些關鍵點的標註
key_probabilities = [0.1, 0.25, 0.5, 0.75, 0.9]
for p in key_probabilities:
    h_val = entropy([p, 1-p])
    plt.plot(p, h_val, 'ko', markersize=6)
    plt.annotate(f'p={p}\nH={h_val:.3f}', (p, h_val),
                xytext=(20, 20), textcoords='offset points',
                arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0'),
                bbox=dict(boxstyle='round,pad=0.3', facecolor='lightblue', alpha=0.7))

plt.tight_layout()
plt.show()

# 環境設置和庫導入
import numpy as np
import matplotlib.pyplot as plt
import math
from scipy.stats import entropy
import warnings
warnings.filterwarnings('ignore')

# 設置中文字體（如果遇到顯示問題可以註釋掉）
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False

def model_uncertainty_analysis(predictions):
    """
    分析模型預測的不確定性
    
    Parameters:
    predictions: 模型輸出的概率分佈列表
    
    Returns:
    average_entropy: 平均熵值
    uncertainty_ratio: 不確定性比例
    """
    entropies = [entropy(pred) for pred in predictions]
    avg_entropy = np.mean(entropies)
    max_entropy = math.log2(len(predictions[0]))  # 假設所有預測維度相同
    uncertainty_ratio = avg_entropy / max_entropy
    
    return avg_entropy, uncertainty_ratio

# 模擬模型預測
sample_predictions = [
    [0.9, 0.1],    # 確定性高的預測
    [0.6, 0.4],    # 中等確定性
    [0.5, 0.5],    # 完全不確定
    [0.8, 0.2],    # 較高確定性
    [0.7, 0.3]     # 中等確定性
]

avg_ent, uncertainty = model_uncertainty_analysis(sample_predictions)
print(f"模型預測分析:")
print(f"  平均熵值: {avg_ent:.4f} bits")
print(f"  不確定性比例: {uncertainty:.2%}")

# 環境設置和庫導入
import numpy as np
import matplotlib.pyplot as plt
import math
from scipy.stats import entropy
from collections import Counter
import warnings
warnings.filterwarnings('ignore')

def text_complexity_analysis(text):
    """
    分析文本的複雜度（基於字符級熵）
    
    Parameters:
    text: 輸入文本
    
    Returns:
    complexity: 文本複雜度（熵值）
    """
    char_counts = Counter(text)
    total_chars = len(text)
    probabilities = [count/total_chars for count in char_counts.values()]
    return entropy(probabilities)

# 示例文本分析
sample_texts = [
    "aaaaabbbbbcccccdddddeeeee",  # 低複雜度
    "hello world this is a test",  # 中等複雜度
    "the quick brown fox jumps over the lazy dog",  # 較高複雜度
    "abc123def456ghi789jkl0",     # 高複雜度
]

print("\n=== 文本複雜度分析 ===")
for text in sample_texts:
    comp = text_complexity_analysis(text)
    print(f"文本: {text[:30]:<30} 複雜度: {comp:.4f} bits/字符")

# 環境設置和庫導入
import numpy as np
import matplotlib.pyplot as plt
import math
from scipy.stats import entropy
from collections import Counter
import warnings
warnings.filterwarnings('ignore')

# 設置中文字體（如果遇到顯示問題可以註釋掉）
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False

def joint_entropy(joint_prob_matrix):
    """
    計算兩個隨機變量的聯合熵
    
    Parameters:
    joint_prob_matrix: 聯合概率矩陣，P(X,Y)
    
    Returns:
    joint_entropy_value: 聯合熵值
    """
    joint_prob_matrix = np.array(joint_prob_matrix)
    
    # 驗證輸入
    if not np.allclose(np.sum(joint_prob_matrix), 1.0):
        raise ValueError("聯合概率矩陣之和必須為1")
    if np.any(joint_prob_matrix < 0):
        raise ValueError("概率不能為負")
    
    entropy_val = 0.0
    for i in range(joint_prob_matrix.shape[0]):
        for j in range(joint_prob_matrix.shape[1]):
            p = joint_prob_matrix[i, j]
            if p > 0:
                entropy_val -= p * math.log2(p)
    return entropy_val

print("=== 聯合熵計算示例 ===")

# 示例1：天氣和活動的聯合分佈
# 行：天氣（0=晴天，1=雨天），列：活動（0=室內，1=室外）
print("示例1：天氣和活動的聯合分佈")

# 情況A：獨立變量
joint_independent = np.array([
    [0.3, 0.2],  # 晴天：室內0.3，室外0.2
    [0.3, 0.2]   # 雨天：室內0.3，室外0.2
])

# 情況B：依賴變量
joint_dependent = np.array([
    [0.4, 0.1],  # 晴天：室內0.4，室外0.1
    [0.1, 0.4]   # 雨天：室內0.1，室外0.4
])

cases = [("獨立變量", joint_independent), ("依賴變量", joint_dependent)]

for case_name, joint_probs in cases:
    h_joint = joint_entropy(joint_probs)
    h_weather = entropy(np.sum(joint_probs, axis=1))  # 天氣的邊際熵
    h_activity = entropy(np.sum(joint_probs, axis=0)) # 活動的邊際熵
    
    print(f"\n{case_name}:")
    print(f"  聯合概率分佈:")
    print(f"       活動=0   活動=1")
    print(f"  天氣=0  {joint_probs[0,0]:.2f}     {joint_probs[0,1]:.2f}")
    print(f"  天氣=1  {joint_probs[1,0]:.2f}     {joint_probs[1,1]:.2f}")
    print(f"  聯合熵 H(天氣,活動): {h_joint:.4f}")
    print(f"  天氣熵 H(天氣): {h_weather:.4f}")
    print(f"  活動熵 H(活動): {h_activity:.4f}")
    print(f"  邊際熵之和: {h_weather + h_activity:.4f}")
    print(f"  關係驗證: H(X,Y) ≤ H(X) + H(Y): {h_joint <= h_weather + h_activity}")

# 可視化聯合分佈
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))

# 獨立情況
im1 = ax1.imshow(joint_independent, cmap='Blues', aspect='auto')
ax1.set_xticks([0, 1])
ax1.set_yticks([0, 1])
ax1.set_xticklabels(['室內', '室外'])
ax1.set_yticklabels(['晴天', '雨天'])
ax1.set_title('獨立變量聯合分佈\nH(X,Y) = H(X) + H(Y)')
plt.colorbar(im1, ax=ax1)

# 添加數值標註
for i in range(2):
    for j in range(2):
        ax1.text(j, i, f'{joint_independent[i, j]:.2f}', 
                ha='center', va='center', fontsize=12, color='red')

# 依賴情況
im2 = ax2.imshow(joint_dependent, cmap='Blues', aspect='auto')
ax2.set_xticks([0, 1])
ax2.set_yticks([0, 1])
ax2.set_xticklabels(['室內', '室外'])
ax2.set_yticklabels(['晴天', '雨天'])
ax2.set_title('依賴變量聯合分佈\nH(X,Y) < H(X) + H(Y)')
plt.colorbar(im2, ax=ax2)

# 添加數值標註
for i in range(2):
    for j in range(2):
        ax2.text(j, i, f'{joint_dependent[i, j]:.2f}', 
                ha='center', va='center', fontsize=12, color='red')

plt.tight_layout()
plt.show()

# 環境設置和庫導入
import numpy as np
import matplotlib.pyplot as plt
import math
from scipy.stats import entropy
from collections import Counter
import warnings
warnings.filterwarnings('ignore')

# 設置中文字體（如果遇到顯示問題可以註釋掉）
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False

def joint_entropy(joint_prob_matrix):
    """
    計算兩個隨機變量的聯合熵
    
    Parameters:
    joint_prob_matrix: 聯合概率矩陣，P(X,Y)
    
    Returns:
    joint_entropy_value: 聯合熵值
    """
    joint_prob_matrix = np.array(joint_prob_matrix)
    
    # 驗證輸入
    if not np.allclose(np.sum(joint_prob_matrix), 1.0):
        raise ValueError("聯合概率矩陣之和必須為1")
    if np.any(joint_prob_matrix < 0):
        raise ValueError("概率不能為負")
    
    entropy_val = 0.0
    for i in range(joint_prob_matrix.shape[0]):
        for j in range(joint_prob_matrix.shape[1]):
            p = joint_prob_matrix[i, j]
            if p > 0:
                entropy_val -= p * math.log2(p)
    return entropy_val

def conditional_entropy(joint_probs):
    """
    計算條件熵 H(Y|X)
    
    Parameters:
    joint_probs: 聯合概率矩陣 P(X,Y)
    
    Returns:
    conditional_entropy_value: 條件熵值
    """
    joint_probs = np.array(joint_probs)
    
    # 計算邊際分佈 P(X)
    p_x = np.sum(joint_probs, axis=1)
    
    cond_entropy = 0.0
    for i in range(joint_probs.shape[0]):
        if p_x[i] > 0:
            # 計算條件分佈 P(Y|X=x_i)
            p_y_given_x = joint_probs[i, :] / p_x[i]
            # 計算條件分佈的熵
            h_y_given_x = entropy(p_y_given_x)
            # 加權平均
            cond_entropy += p_x[i] * h_y_given_x
    
    return cond_entropy

print("=== 條件熵計算示例 ===")

# 使用之前的依賴變量例子
joint_probs = np.array([
    [0.4, 0.1],  # 晴天：室內0.4，室外0.1
    [0.1, 0.4]   # 雨天：室內0.1，室外0.4
])

h_activity_given_weather = conditional_entropy(joint_probs)
h_joint = joint_entropy(joint_probs)
h_weather = entropy(np.sum(joint_probs, axis=1))
h_activity = entropy(np.sum(joint_probs, axis=0))

print("依賴變量案例詳細分析:")
print(f"  天氣熵 H(Weather): {h_weather:.4f}")
print(f"  活動熵 H(Activity): {h_activity:.4f}")
print(f"  聯合熵 H(Weather, Activity): {h_joint:.4f}")
print(f"  條件熵 H(Activity|Weather): {h_activity_given_weather:.4f}")

# 驗證鏈式法則
print(f"\n鏈式法則驗證:")
print(f"  H(Weather) + H(Activity|Weather) = {h_weather:.4f} + {h_activity_given_weather:.4f} = {h_weather + h_activity_given_weather:.4f}")
print(f"  H(Weather, Activity) = {h_joint:.4f}")
print(f"  差值: {abs((h_weather + h_activity_given_weather) - h_joint):.8f}")

# 與獨立情況對比
print(f"\n與獨立情況對比:")
joint_indep = np.array([
    [0.25, 0.25],
    [0.25, 0.25]
])
h_activity_given_weather_indep = conditional_entropy(joint_indep)
print(f"  獨立情況 - H(Activity|Weather): {h_activity_given_weather_indep:.4f}")
print(f"  依賴情況 - H(Activity|Weather): {h_activity_given_weather:.4f}")
print(f"  知道天氣在依賴情況下減少更多不確定性")

# 不確定性減少比例
reduction_dep = h_activity - h_activity_given_weather
reduction_indep = h_activity - h_activity_given_weather_indep
print(f"\n不確定性減少量:")
print(f"  依賴情況減少: {reduction_dep:.4f} bits ({reduction_dep/h_activity*100:.1f}%)")
print(f"  獨立情況減少: {reduction_indep:.4f} bits ({reduction_indep/h_activity*100:.1f}%)")

# 環境設置和庫導入
import numpy as np
import matplotlib.pyplot as plt
import math
from scipy.stats import entropy
from collections import Counter
import warnings
warnings.filterwarnings('ignore')

# 設置中文字體（如果遇到顯示問題可以註釋掉）
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False

def joint_entropy(joint_prob_matrix):
    """
    計算兩個隨機變量的聯合熵
    
    Parameters:
    joint_prob_matrix: 聯合概率矩陣，P(X,Y)
    
    Returns:
    joint_entropy_value: 聯合熵值
    """
    joint_prob_matrix = np.array(joint_prob_matrix)
    
    # 驗證輸入
    if not np.allclose(np.sum(joint_prob_matrix), 1.0):
        raise ValueError("聯合概率矩陣之和必須為1")
    if np.any(joint_prob_matrix < 0):
        raise ValueError("概率不能為負")
    
    entropy_val = 0.0
    for i in range(joint_prob_matrix.shape[0]):
        for j in range(joint_prob_matrix.shape[1]):
            p = joint_prob_matrix[i, j]
            if p > 0:
                entropy_val -= p * math.log2(p)
    return entropy_val

def mutual_information(joint_probs):
    """
    計算兩個隨機變量的互信息
    
    Parameters:
    joint_probs: 聯合概率矩陣 P(X,Y)
    
    Returns:
    mi: 互信息值
    mi_alt: 另一種計算方法的結果（用於驗證）
    """
    joint_probs = np.array(joint_probs)
    
    # 方法1：使用熵的定義
    p_x = np.sum(joint_probs, axis=1)  # P(X)
    p_y = np.sum(joint_probs, axis=0)  # P(Y)
    
    h_x = entropy(p_x)
    h_y = entropy(p_y)
    h_xy = joint_entropy(joint_probs)
    
    mi = h_x + h_y - h_xy
    
    # 方法2：直接計算
    mi_alt = 0.0
    for i in range(joint_probs.shape[0]):
        for j in range(joint_probs.shape[1]):
            p_xy = joint_probs[i, j]
            if p_xy > 0:
                mi_alt += p_xy * math.log2(p_xy / (p_x[i] * p_y[j]))
    
    return mi, mi_alt

print("=== 互信息計算示例 ===")

# 不同依賴程度的案例
cases = [
    ("強依賴", np.array([[0.4, 0.1], [0.1, 0.4]])),
    ("中等依賴", np.array([[0.35, 0.15], [0.15, 0.35]])),
    ("弱依賴", np.array([[0.3, 0.2], [0.2, 0.3]])),
    ("獨立", np.array([[0.25, 0.25], [0.25, 0.25]]))
]

print(f"{'案例':<10} {'H(X)':<8} {'H(Y)':<8} {'H(X,Y)':<8} {'I(X;Y)':<8} {'歸一化MI':<10}")
print("-" * 65)

for case_name, joint_probs in cases:
    mi, mi_alt = mutual_information(joint_probs)
    h_x = entropy(np.sum(joint_probs, axis=1))
    h_y = entropy(np.sum(joint_probs, axis=0))
    h_xy = joint_entropy(joint_probs)
    
    # 歸一化互信息（0到1之間）
    normalized_mi = mi / min(h_x, h_y) if min(h_x, h_y) > 0 else 0
    
    print(f"{case_name:<10} {h_x:<8.4f} {h_y:<8.4f} {h_xy:<8.4f} {mi:<8.4f} {normalized_mi:<10.4f}")

# 驗證兩種計算方法的一致性
print(f"\n計算方法驗證:")
for case_name, joint_probs in cases[:2]:  # 只驗證前兩個案例
    mi, mi_alt = mutual_information(joint_probs)
    print(f"{case_name}: 方法1={mi:.6f}, 方法2={mi_alt:.6f}, 差值={abs(mi-mi_alt):.8f}")

# 互信息可視化
fig, axes = plt.subplots(2, 2, figsize=(12, 10))
axes = axes.ravel()

for idx, (case_name, joint_probs) in enumerate(cases):
    mi, _ = mutual_information(joint_probs)
    h_x = entropy(np.sum(joint_probs, axis=1))
    h_y = entropy(np.sum(joint_probs, axis=0))
    
    # 繪製韋恩圖風格的表示
    circle1 = plt.Circle((0.3, 0.5), 0.2, color='blue', alpha=0.3, label='H(X)')
    circle2 = plt.Circle((0.7, 0.5), 0.2, color='red', alpha=0.3, label='H(Y)')
    
    axes[idx].add_patch(circle1)
    axes[idx].add_patch(circle2)
    
    # 設置圖形屬性
    axes[idx].set_xlim(0, 1)
    axes[idx].set_ylim(0, 1)
    axes[idx].set_aspect('equal')
    axes[idx].set_title(f'{case_name}\nI(X;Y) = {mi:.4f}')
    axes[idx].text(0.5, 0.9, f'H(X)={h_x:.3f}', ha='center', fontsize=10)
    axes[idx].text(0.5, 0.1, f'H(Y)={h_y:.3f}', ha='center', fontsize=10)
    axes[idx].text(0.5, 0.5, f'MI={mi:.3f}', ha='center', fontsize=12, 
                  bbox=dict(boxstyle='round', facecolor='yellow', alpha=0.7))
    
    # 隱藏座標軸
    axes[idx].set_xticks([])
    axes[idx].set_yticks([])

plt.tight_layout()
plt.show()

# 環境設置和庫導入
import numpy as np
import math
from scipy.stats import entropy
import warnings
warnings.filterwarnings('ignore')

def kl_divergence(p, q):
    """
    計算KL散度 D_KL(P || Q)
    
    Parameters:
    p: 真實分佈P
    q: 近似分佈Q
    
    Returns:
    kl_divergence_value: KL散度值
    """
    p = np.array(p)
    q = np.array(q)
    
    # 輸入驗證
    if len(p) != len(q):
        raise ValueError("分佈P和Q必須有相同的長度")
    if not np.allclose(np.sum(p), 1.0) or not np.allclose(np.sum(q), 1.0):
        raise ValueError("分佈P和Q必須和為1")
    if np.any(p < 0) or np.any(q < 0):
        raise ValueError("概率不能為負")
    
    divergence = 0.0
    for i in range(len(p)):
        if p[i] > 0:
            if q[i] > 0:
                divergence += p[i] * math.log2(p[i] / q[i])
            else:
                return float('inf')  # 如果Q在P有概率的地方為0，散度為無窮大
    return divergence

print("=== KL散度計算示例 ===")

# 真實分佈（例如真實的詞頻分佈）
p_true = np.array([0.5, 0.3, 0.15, 0.05])

# 不同的模型近似
approximations = [
    ("優秀近似", np.array([0.48, 0.31, 0.14, 0.07])),
    ("中等近似", np.array([0.6, 0.25, 0.1, 0.05])),
    ("較差近似", np.array([0.7, 0.2, 0.08, 0.02])),
    ("錯誤近似", np.array([0.1, 0.1, 0.4, 0.4])),
    ("零概率問題", np.array([0.5, 0.3, 0.2, 0.0]))  # 在P>0的地方Q=0
]

print(f"真實分佈 P: {p_true}")
print("\n不同近似的KL散度:")
print(f"{'近似類型':<12} {'分佈Q':<30} {'D_KL(P||Q)':<12} {'評估':<10}")
print("-" * 75)

for name, q in approximations:
    try:
        kl = kl_divergence(p_true, q)
        if kl < 0.1:
            assessment = "優秀"
        elif kl < 0.5:
            assessment = "良好"
        elif kl < 1.0:
            assessment = "一般"
        else:
            assessment = "較差"
    except:
        kl = float('inf')
        assessment = "無效"
    
    q_str = str([round(val, 3) for val in q])
    print(f"{name:<12} {q_str:<30} {kl:<12.4f} {assessment:<10}")

# 不對稱性演示
print(f"\n=== KL散度不對稱性演示 ===")
p = np.array([0.7, 0.3])
q = np.array([0.4, 0.6])

kl_pq = kl_divergence(p, q)
kl_qp = kl_divergence(q, p)

print(f"分佈 P: {p}")
print(f"分佈 Q: {q}")
print(f"D_KL(P||Q) = {kl_pq:.4f}")
print(f"D_KL(Q||P) = {kl_qp:.4f}")
print(f"不對稱性: D_KL(P||Q) ≠ D_KL(Q||P)")

# KL散度與模型訓練的關係
print(f"\n=== KL散度在模型訓練中的應用 ===")
print("在機器學習中，我們通常最小化 D_KL(P_data || P_model)")
print("這等價於最大似然估計")
print("KL散度作為正則項可以防止模型過度偏離先驗分佈")

# 環境設置和庫導入
import numpy as np
import math
from scipy.stats import entropy
import warnings
warnings.filterwarnings('ignore')

def kl_divergence(p, q):
    """
    計算KL散度 D_KL(P || Q)
    
    Parameters:
    p: 真實分佈P
    q: 近似分佈Q
    
    Returns:
    kl_divergence_value: KL散度值
    """
    p = np.array(p)
    q = np.array(q)
    
    # 輸入驗證
    if len(p) != len(q):
        raise ValueError("分佈P和Q必須有相同的長度")
    if not np.allclose(np.sum(p), 1.0) or not np.allclose(np.sum(q), 1.0):
        raise ValueError("分佈P和Q必須和為1")
    if np.any(p < 0) or np.any(q < 0):
        raise ValueError("概率不能為負")
    
    divergence = 0.0
    for i in range(len(p)):
        if p[i] > 0:
            if q[i] > 0:
                divergence += p[i] * math.log2(p[i] / q[i])
            else:
                return float('inf')  # 如果Q在P有概率的地方為0，散度為無窮大
    return divergence

def cross_entropy(p, q):
    """
    計算交叉熵 H(P, Q)
    
    Parameters:
    p: 真實分佈P
    q: 預測分佈Q
    
    Returns:
    cross_entropy_value: 交叉熵值
    """
    p = np.array(p)
    q = np.array(q)
    
    # 輸入驗證
    if len(p) != len(q):
        raise ValueError("分佈P和Q必須有相同的長度")
    if not np.allclose(np.sum(p), 1.0) or not np.allclose(np.sum(q), 1.0):
        raise ValueError("分佈P和Q必須和為1")
    if np.any(p < 0) or np.any(q < 0):
        raise ValueError("概率不能為負")
    
    ce = 0.0
    for i in range(len(p)):
        if p[i] > 0:
            if q[i] > 0:
                ce -= p[i] * math.log2(q[i])
            else:
                return float('inf')  # 如果Q在P有概率的地方為0，交叉熵為無窮大
    return ce

print("=== 交叉熵計算示例 ===")

# 真實分佈和不同的預測分佈
p_true = np.array([0.6, 0.3, 0.1])

predictions = [
    ("完美預測", np.array([0.6, 0.3, 0.1])),
    ("良好預測", np.array([0.5, 0.4, 0.1])),
    ("一般預測", np.array([0.7, 0.2, 0.1])),
    ("較差預測", np.array([0.8, 0.1, 0.1])),
    ("錯誤預測", np.array([0.1, 0.1, 0.8]))
]

print(f"真實分佈 P: {p_true}")
print(f"{'預測類型':<12} {'預測分佈Q':<25} {'交叉熵':<10} {'KL散度':<10} {'真實熵':<10}")
print("-" * 80)

for name, q in predictions:
    ce = cross_entropy(p_true, q)
    h_p = entropy(p_true)
    kl = kl_divergence(p_true, q)
    
    print(f"{name:<12} {str([round(val, 3) for val in q]):<25} {ce:<10.4f} {kl:<10.4f} {h_p:<10.4f}")

# 驗證交叉熵與KL散度的關係
print(f"\n=== 交叉熵與KL散度關係驗證 ===")
p_test = np.array([0.5, 0.3, 0.2])
q_test = np.array([0.4, 0.4, 0.2])

ce_val = cross_entropy(p_test, q_test)
h_p_val = entropy(p_test)
kl_val = kl_divergence(p_test, q_test)

print(f"真實分佈 P: {p_test}")
print(f"預測分佈 Q: {q_test}")
print(f"交叉熵 H(P,Q): {ce_val:.4f}")
print(f"真實熵 H(P): {h_p_val:.4f}")
print(f"KL散度 D_KL(P||Q): {kl_val:.4f}")
print(f"關係驗證: H(P) + D_KL(P||Q) = {h_p_val:.4f} + {kl_val:.4f} = {h_p_val + kl_val:.4f}")
print(f"與交叉熵相等: {abs((h_p_val + kl_val) - ce_val) < 1e-10}")

# 二分類交叉熵示例
print(f"\n=== 二分類交叉熵示例 ===")
def binary_cross_entropy(y_true, y_pred):
    """
    計算二分類交叉熵
    
    Parameters:
    y_true: 真實標籤 (0或1)
    y_pred: 預測概率 (0到1之間)
    
    Returns:
    bce: 平均交叉熵
    """
    y_true = np.array(y_true)
    y_pred = np.array(y_pred)
    
    # 避免log(0)的情況
    y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
    
    bce = -np.mean(y_true * np.log2(y_pred) + (1 - y_true) * np.log2(1 - y_pred))
    return bce

# 示例數據
y_true = np.array([1, 0, 1, 1, 0])
y_pred_good = np.array([0.9, 0.2, 0.8, 0.7, 0.3])
y_pred_poor = np.array([0.6, 0.5, 0.6, 0.6, 0.4])

bce_good = binary_cross_entropy(y_true, y_pred_good)
bce_poor = binary_cross_entropy(y_true, y_pred_poor)

print(f"真實標籤: {y_true}")
print(f"良好預測: {y_pred_good} → 交叉熵: {bce_good:.4f}")
print(f"較差預測: {y_pred_poor} → 交叉熵: {bce_poor:.4f}")

# 環境設置和庫導入
import numpy as np
import math
from scipy.stats import entropy
import warnings
warnings.filterwarnings('ignore')

def kl_divergence(p, q):
    """
    計算KL散度 D_KL(P || Q)
    
    Parameters:
    p: 真實分佈P
    q: 近似分佈Q
    
    Returns:
    kl_divergence_value: KL散度值
    """
    p = np.array(p)
    q = np.array(q)
    
    # 輸入驗證
    if len(p) != len(q):
        raise ValueError("分佈P和Q必須有相同的長度")
    if not np.allclose(np.sum(p), 1.0) or not np.allclose(np.sum(q), 1.0):
        raise ValueError("分佈P和Q必須和為1")
    if np.any(p < 0) or np.any(q < 0):
        raise ValueError("概率不能為負")
    
    divergence = 0.0
    for i in range(len(p)):
        if p[i] > 0:
            if q[i] > 0:
                divergence += p[i] * math.log2(p[i] / q[i])
            else:
                return float('inf')  # 如果Q在P有概率的地方為0，散度為無窮大
    return divergence

def cross_entropy(p, q):
    """
    計算交叉熵 H(P, Q)
    
    Parameters:
    p: 真實分佈P
    q: 預測分佈Q
    
    Returns:
    cross_entropy_value: 交叉熵值
    """
    p = np.array(p)
    q = np.array(q)
    
    # 輸入驗證
    if len(p) != len(q):
        raise ValueError("分佈P和Q必須有相同的長度")
    if not np.allclose(np.sum(p), 1.0) or not np.allclose(np.sum(q), 1.0):
        raise ValueError("分佈P和Q必須和為1")
    if np.any(p < 0) or np.any(q < 0):
        raise ValueError("概率不能為負")
    
    ce = 0.0
    for i in range(len(p)):
        if p[i] > 0:
            if q[i] > 0:
                ce -= p[i] * math.log2(q[i])
            else:
                return float('inf')  # 如果Q在P有概率的地方為0，交叉熵為無窮大
    return ce

class LanguageModelTrainer:
    """簡化的語言模型訓練器，演示交叉熵的應用"""
    
    def __init__(self, vocab_size=1000):
        self.vocab_size = vocab_size
        
    def calculate_batch_loss(self, true_distributions, predicted_distributions):
        """
        計算批量的交叉熵損失
        
        Parameters:
        true_distributions: 真實分佈列表
        predicted_distributions: 預測分佈列表
        
        Returns:
        average_loss: 平均交叉熵損失
        loss_breakdown: 每個樣本的損失詳情
        """
        batch_size = len(true_distributions)
        losses = []
        
        for i in range(batch_size):
            ce_loss = cross_entropy(true_distributions[i], predicted_distributions[i])
            losses.append(ce_loss)
        
        average_loss = np.mean(losses)
        
        loss_breakdown = {
            'average_loss': average_loss,
            'min_loss': np.min(losses),
            'max_loss': np.max(losses),
            'std_loss': np.std(losses),
            'losses': losses
        }
        
        return average_loss, loss_breakdown
    
    def analyze_training_progress(self, epoch_losses):
        """
        分析訓練進度
        
        Parameters:
        epoch_losses: 每個epoch的損失列表
        
        Returns:
        analysis: 訓練分析結果
        """
        analysis = {
            'final_loss': epoch_losses[-1],
            'best_loss': np.min(epoch_losses),
            'improvement': epoch_losses[0] - epoch_losses[-1],
            'convergence_rate': self._calculate_convergence_rate(epoch_losses)
        }
        return analysis
    
    def _calculate_convergence_rate(self, losses):
        """計算收斂速率"""
        if len(losses) < 2:
            return 0
        improvements = []
        for i in range(1, len(losses)):
            improvement = losses[i-1] - losses[i]
            improvements.append(improvement)
        return np.mean(improvements)

# 模擬訓練過程
print("=== 語言模型訓練模擬 ===")
trainer = LanguageModelTrainer(vocab_size=1000)

# 模擬訓練數據（真實分佈和預測分佈）
batch_size = 5
vocab_size = 10

# 生成模擬數據
true_dists = []
pred_dists = []

for i in range(batch_size):
    # 真實分佈（one-hot或接近one-hot）
    true_dist = np.zeros(vocab_size)
    true_class = np.random.randint(0, vocab_size)
    true_dist[true_class] = 1.0
    true_dists.append(true_dist)
    
    # 預測分佈（模型輸出）
    pred_dist = np.random.dirichlet(np.ones(vocab_size) * 0.1)  # 狄利克雷分佈
    pred_dists.append(pred_dist)

# 計算損失
avg_loss, loss_details = trainer.calculate_batch_loss(true_dists, pred_dists)

print(f"批量訓練結果:")
print(f"  平均損失: {avg_loss:.4f} bits")
print(f"  最小損失: {loss_details['min_loss']:.4f} bits")
print(f"  最大損失: {loss_details['max_loss']:.4f} bits")
print(f"  損失標準差: {loss_details['std_loss']:.4f} bits")

# 模擬訓練過程
print(f"\n=== 訓練過程模擬 ===")
epochs = 20
simulated_losses = []

# 模擬損失下降（指數衰減）
initial_loss = 8.0
for epoch in range(epochs):
    # 模擬損失下降
    loss = initial_loss * np.exp(-epoch / 5) + np.random.normal(0, 0.1)
    simulated_losses.append(loss)
    print(f"Epoch {epoch+1:2d}: 損失 = {loss:.4f} bits")

# 分析訓練進度
analysis = trainer.analyze_training_progress(simulated_losses)
print(f"\n訓練分析:")
print(f"  最終損失: {analysis['final_loss']:.4f} bits")
print(f"  最佳損失: {analysis['best_loss']:.4f} bits")
print(f"  總改進: {analysis['improvement']:.4f} bits")
print(f"  平均每epoch改進: {analysis['convergence_rate']:.4f} bits")

# 環境設置和庫導入
import numpy as np
import math
from scipy.stats import entropy
from collections import Counter
import warnings
warnings.filterwarnings('ignore')

def entropy_from_labels(labels):
    """
    從標籤數據直接計算熵
    
    Parameters:
    labels: 類別標籤數組
    
    Returns:
    entropy_value: 信息熵值
    """
    counter = Counter(labels)
    total = len(labels)
    probabilities = [count / total for count in counter.values()]
    return entropy(probabilities)

def information_gain(feature_values, labels):
    """
    計算特徵的信息增益
    
    Parameters:
    feature_values: 特徵值數組
    labels: 目標標籤數組
    
    Returns:
    information_gain_value: 信息增益值
    split_info: 分割的詳細信息
    """
    # 原始熵 H(Y)
    original_entropy = entropy_from_labels(labels)
    
    # 條件熵 H(Y|X)
    unique_features = np.unique(feature_values)
    conditional_entropy_val = 0.0
    
    split_info = {
        'original_entropy': original_entropy,
        'feature_values': unique_features,
        'splits': []
    }
    
    for feature_val in unique_features:
        mask = feature_values == feature_val
        subset_labels = labels[mask]
        
        if len(subset_labels) > 0:
            weight = len(subset_labels) / len(labels)
            subset_entropy = entropy_from_labels(subset_labels)
            conditional_entropy_val += weight * subset_entropy
            
            # 記錄分割信息
            split_info['splits'].append({
                'feature_value': feature_val,
                'weight': weight,
                'subset_entropy': subset_entropy,
                'subset_size': len(subset_labels),
                'label_distribution': dict(Counter(subset_labels))
            })
    
    ig = original_entropy - conditional_entropy_val
    split_info['conditional_entropy'] = conditional_entropy_val
    split_info['information_gain'] = ig
    
    return ig, split_info

print("=== 信息增益計算示例 ===")

# 學生成績預測數據集
# 特徵：學習時間 (0=少, 1=多)
# 目標：成績 (0=不及格, 1=及格)

study_time = np.array([0, 0, 0, 0, 1, 1, 1, 1])
grades = np.array([0, 0, 1, 0, 1, 1, 1, 1])

print("數據集:")
print(f"學習時間: {study_time} (0=少, 1=多)")
print(f"成績:     {grades} (0=不及格, 1=及格)")

# 計算信息增益
ig, split_info = information_gain(study_time, grades)

print(f"\n信息增益分析:")
print(f"原始熵 H(成績): {split_info['original_entropy']:.4f} bits")
print(f"條件熵 H(成績|學習時間): {split_info['conditional_entropy']:.4f} bits")
print(f"信息增益 IG(成績, 學習時間): {ig:.4f} bits")
print(f"不確定性減少比例: {ig/split_info['original_entropy']*100:.1f}%")

# 詳細分割信息
print(f"\n詳細分割信息:")
for split in split_info['splits']:
    study_desc = "學習時間少" if split['feature_value'] == 0 else "學習時間多"
    print(f"  {study_desc}:")
    print(f"    權重: {split['weight']:.2f}")
    print(f"    子集大小: {split['subset_size']}")
    print(f"    子集熵: {split['subset_entropy']:.4f}")
    print(f"    標籤分佈: {split['label_distribution']}")

# 多特徵比較
print(f"\n=== 多特徵信息增益比較 ===")

# 添加更多特徵
temperature = np.array([1, 1, 0, 0, 0, 0, 1, 1])  # 0=冷, 1=暖
random_feature = np.array([0, 1, 0, 1, 0, 1, 0, 1])  # 隨機特徵

features = [
    ("學習時間", study_time),
    ("温度", temperature),
    ("隨機特徵", random_feature)
]

print("特徵比較:")
print(f"{'特徵':<10} {'信息增益':<12} {'歸一化增益':<12} {'評估':<10}")
print("-" * 50)

for feature_name, feature_vals in features:
    ig_val, _ = information_gain(feature_vals, grades)
    normalized_ig = ig_val / entropy_from_labels(grades)
    
    if normalized_ig > 0.7:
        assessment = "優秀"
    elif normalized_ig > 0.3:
        assessment = "良好"
    elif normalized_ig > 0.1:
        assessment = "一般"
    else:
        assessment = "無用"
    
    print(f"{feature_name:<10} {ig_val:<12.4f} {normalized_ig:<12.4f} {assessment:<10}")

# 決策樹節點選擇演示
print(f"\n=== 決策樹節點選擇演示 ===")
best_feature = max(features, key=lambda x: information_gain(x[1], grades)[0])
print(f"最佳分割特徵: '{best_feature[0]}'")
print(f"信息增益: {information_gain(best_feature[1], grades)[0]:.4f} bits")

# 環境設置和庫導入
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from collections import Counter
import math
from scipy.special import entr
import warnings
warnings.filterwarnings('ignore')

# 設置中文字體（如果遇到顯示問題可以註釋掉）
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False

def entropy(probabilities):
    """
    計算概率分佈的信息熵
    
    Parameters:
    probabilities: 概率列表，應該和為1
    
    Returns:
    entropy_value: 信息熵值（比特）
    """
    # 輸入驗證
    probabilities = np.array(probabilities)
    if not np.allclose(np.sum(probabilities), 1.0):
        raise ValueError("概率之和必須為1")
    if np.any(probabilities < 0):
        raise ValueError("概率不能為負")
    
    entropy_val = 0.0
    for p in probabilities:
        if p > 0:  # 避免log(0)的情況
            entropy_val -= p * math.log2(p)
    return entropy_val

def cross_entropy(p, q):
    """
    計算交叉熵 H(P, Q)
    
    Parameters:
    p: 真實分佈P
    q: 預測分佈Q
    
    Returns:
    cross_entropy_value: 交叉熵值
    """
    p = np.array(p)
    q = np.array(q)
    
    # 輸入驗證
    if len(p) != len(q):
        raise ValueError("分佈P和Q必須有相同的長度")
    if not np.allclose(np.sum(p), 1.0) or not np.allclose(np.sum(q), 1.0):
        raise ValueError("分佈P和Q必須和為1")
    if np.any(p < 0) or np.any(q < 0):
        raise ValueError("概率不能為負")
    
    ce = 0.0
    for i in range(len(p)):
        if p[i] > 0:
            if q[i] > 0:
                ce -= p[i] * math.log2(q[i])
            else:
                return float('inf')  # 如果Q在P有概率的地方為0，交叉熵為無窮大
    return ce

class LanguageModelAnalyzer:
    """語言模型分析器，使用信息論工具"""
    
    def __init__(self, vocab_size=50000):
        self.vocab_size = vocab_size
        
    def calculate_perplexity(self, cross_entropy):
        """
        計算困惑度
        
        Parameters:
        cross_entropy: 交叉熵值
        
        Returns:
        perplexity: 困惑度
        """
        return 2 ** cross_entropy
    
    def analyze_model_uncertainty(self, predictions):
        """
        分析模型預測的不確定性
        
        Parameters:
        predictions: 模型輸出的概率分佈列表
        
        Returns:
        analysis: 不確定性分析結果
        """
        entropies = [entropy(pred) for pred in predictions]
        cross_entropies = [cross_entropy([1.0] + [0.0]*(len(pred)-1), pred) for pred in predictions]  # 假設完美預測
        
        analysis = {
            'average_entropy': np.mean(entropies),
            'average_cross_entropy': np.mean(cross_entropies),
            'perplexity': self.calculate_perplexity(np.mean(cross_entropies)),
            'entropy_std': np.std(entropies),
            'max_entropy': np.max(entropies),
            'min_entropy': np.min(entropies),
            'confidence_scores': [1 - e / math.log2(len(pred)) for e, pred in zip(entropies, predictions)]  # 置信度
        }
        return analysis
    
    def temperature_sampling_analysis(self, base_probs, temperatures):
        """
        分析温度採樣對分佈的影響
        
        Parameters:
        base_probs: 基礎概率分佈
        temperatures: 温度值列表
        
        Returns:
        results: 不同温度下的分析結果
        """
        results = {}
        
        for temp in temperatures:
            # 應用温度採樣
            if temp == 0:
                # 貪婪採樣（實際中temp不會為0）
                scaled_probs = base_probs
            else:
                scaled_probs = self.apply_temperature(base_probs, temp)
            
            results[temp] = {
                'distribution': scaled_probs,
                'entropy': entropy(scaled_probs),
                'max_prob': np.max(scaled_probs),
                'effective_tokens': self.calculate_effective_tokens(scaled_probs)
            }
        
        return results
    
    def apply_temperature(self, probs, temperature):
        """應用温度到概率分佈"""
        scaled_probs = np.log(probs) / temperature
        scaled_probs = np.exp(scaled_probs - np.max(scaled_probs))  # 數值穩定性
        return scaled_probs / np.sum(scaled_probs)
    
    def calculate_effective_tokens(self, probs):
        """計算有效token數（基於熵）"""
        h = entropy(probs)
        return 2 ** h

# 大語言模型分析示例
print("=== 大語言模型信息論分析 ===")
analyzer = LanguageModelAnalyzer(vocab_size=50000)

# 模擬語言模型輸出（不同確定性的預測）
sample_predictions = [
    # 高確定性預測
    [0.9, 0.05, 0.03, 0.01, 0.01] + [0.0] * 15,
    # 中等確定性預測  
    [0.4, 0.3, 0.2, 0.05, 0.05] + [0.0] * 15,
    # 低確定性預測
    [0.2, 0.15, 0.15, 0.1, 0.1, 0.08, 0.07, 0.05, 0.05, 0.05] + [0.0] * 10,
    # 均勻分佈（高不確定性）
    [0.05] * 20
]

# 確保概率和為1
sample_predictions = [p / np.sum(p) for p in sample_predictions]

# 分析模型不確定性
uncertainty_analysis = analyzer.analyze_model_uncertainty(sample_predictions)

print("模型不確定性分析:")
print(f"  平均熵: {uncertainty_analysis['average_entropy']:.4f} bits")
print(f"  平均交叉熵: {uncertainty_analysis['average_cross_entropy']:.4f} bits")
print(f"  困惑度: {uncertainty_analysis['perplexity']:.2f}")
print(f"  熵標準差: {uncertainty_analysis['entropy_std']:.4f}")
print(f"  最大熵: {uncertainty_analysis['max_entropy']:.4f}")
print(f"  最小熵: {uncertainty_analysis['min_entropy']:.4f}")
print(f"  平均置信度: {np.mean(uncertainty_analysis['confidence_scores']):.3f}")

# 温度採樣分析
print(f"\n=== 温度採樣分析 ===")
base_distribution = np.array([0.6, 0.2, 0.1, 0.05, 0.03, 0.02])
temperatures = [0.1, 0.5, 1.0, 1.5, 2.0]

temp_results = analyzer.temperature_sampling_analysis(base_distribution, temperatures)

print("温度對概率分佈的影響:")
print(f"{'温度':<8} {'熵':<8} {'最大概率':<12} {'有效token數':<15}")
print("-" * 50)

for temp in temperatures:
    result = temp_results[temp]
    print(f"{temp:<8} {result['entropy']:<8.4f} {result['max_prob']:<12.4f} {result['effective_tokens']:<15.2f}")

# 可視化温度採樣的影響
plt.figure(figsize=(15, 5))

for i, temp in enumerate(temperatures[:3]):  # 只顯示前3個温度
    plt.subplot(1, 3, i+1)
    result = temp_results[temp]
    
    plt.bar(range(len(result['distribution'])), result['distribution'])
    plt.title(f'温度 = {temp}\n熵 = {result["entropy"]:.3f}')
    plt.xlabel('Token索引')
    plt.ylabel('概率')
    plt.ylim(0, 1)

plt.tight_layout()
plt.show()