# 檢查系統與Python版本
cat /etc/os-release
python3 --version

# 檢查PyTorch及torch_npu
python -c "import torch; print(f'PyTorch版本: {torch.__version__}')"
python -c "import torch_npu; print(f'torch_npu版本: {torch_npu.__version__}')"

# 沒有的話安裝，先執行pip install --upgrade pip
pip install torch torchvision torchaudio
pip install torch-npu

看到
PyTorch版本: 2.4.0
torch_npu版本: 2.4.0.post4
説明正常可用

python -c "import torch; import torch_npu; print(torch.npu.is_available())"

pip install transformers accelerate -i https://pypi.tuna.tsinghua.edu.cn/simple

import os
os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'  #在GitCode的昇騰環境中，直接訪問HuggingFace經常會超時，所以使用國內鏡像

import torch
import torch_npu  # 切記！
from transformers import AutoModelForCausalLM, AutoTokenizer
import time

# 配置
MODEL_NAME = "NousResearch/Llama-2-7b-hf"
DEVICE = "npu:0"

print("開始加載模型...")
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
model = AutoModelForCausalLM.from_pretrained(
    MODEL_NAME,
    torch_dtype=torch.float16,  # 使用FP16節省顯存
    low_cpu_mem_usage=True
)

print("將模型移至NPU...")
model = model.to(DEVICE)
model.eval（)  # 設置為評估模式

# 第三個“坑”：輸入張量遷移
prompt = "The capital of France is"
# 錯誤寫法：inputs = tokenizer(prompt, return_tensors="pt").npu() -> 報錯！
# 正確寫法：
inputs = tokenizer(prompt, return_tensors="pt").to(DEVICE)

# 推理
with torch.no_grad():
    start_time = time.time()
    outputs = model.generate(**inputs, max_new_tokens=50)
    end_time = time.time()

generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(f"生成內容: {generated_text}")
print(f"推理耗時: {end_time - start_time:.2f} 秒")

import os
os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'  #在GitCode的昇騰環境中，直接訪問HuggingFace經常會超時，所以使用國內鏡像


import torch
import torch_npu
import time
import json
from transformers import AutoModelForCausalLM, AutoTokenizer

# 配置
MODEL_NAME = "NousResearch/Llama-2-7b-hf"
DEVICE = "npu:0"
WARMUP_RUNS = 3
TEST_RUNS = 5

def load_model():
    print("加載模型與分詞器...")
    tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
    model = AutoModelForCausalLM.from_pretrained(
        MODEL_NAME,
        torch_dtype=torch.float16,
        low_cpu_mem_usage=True
    ).to(DEVICE)
    model.eval（)
    return model, tokenizer

def benchmark(prompt, model, tokenizer, max_new_tokens=100):
    inputs = tokenizer(prompt, return_tensors="pt").to(DEVICE)
    
    # 預熱
    print("預熱運行...")
    for _ in range(WARMUP_RUNS):
        with torch.no_grad():
            _ = model.generate(**inputs, max_new_tokens=max_new_tokens)
    
    # 正式測試
    print("開始性能測試...")
    latencies = []
    for i in range(TEST_RUNS):
        torch.npu.synchronize()  # 同步，確保計時準確
        start = time.time()
        
        with torch.no_grad():
            _ = model.generate(**inputs, max_new_tokens=max_new_tokens)
            
        torch.npu.synchronize()
        end = time.time()
        latency = end - start
        latencies.append(latency)
        print(f"  第{i+1}次耗時: {latency:.2f}s")

    avg_latency = sum(latencies) / len(latencies)
    throughput = max_new_tokens / avg_latency
    return throughput, avg_latency

if __name__ == "__main__":
    model, tokenizer = load_model()
    
    test_cases = [
        {"場景": "英文生成", "提示": "The future of artificial intelligence is", "長度": 100},
        {"場景": "中文問答", "提示": "請用簡單的話解釋量子計算：", "長度": 100},
        {"場景": "代碼生成", "提示": "Write a Python function to reverse a string:", "長度": 150},
    ]
    
    print("\n" + "="*50)
    print("性能測試結果")
    print("="*50)
    for case in test_cases:
        throughput, avg_latency = benchmark(case["提示"], model, tokenizer, case["長度"])
        print(f"- {case['場景']}:")
        print(f"  平均延遲: {avg_latency:.2f}s")
        print(f"  吞吐量: {throughput:.2f} tokens/s")
    print("="*50)

from transformers import BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=True)
model = AutoModelForCausalLM.from_pretrained(
    MODEL_NAME,
    quantization_config=quantization_config,
    device_map="auto"
)

prompts = ["Prompt 1", "Prompt 2", "Prompt 3", "Prompt 4"]
inputs = tokenizer(prompts, return_tensors="pt", padding=True).to(DEVICE)
outputs = model.generate(**inputs, max_new_tokens=100)