數據集結構示例

數據集結構示例

pip install datasets

from datasets import load_dataset

# 加載數據集
ds = load_dataset("UCSC-VLAA/MedTrinity-25M", "25M_demo", cache_dir="cache")

數據集下載進度顯示

# 查看訓練集的前1個樣本
print(ds['train'][:1])

{
    'image': [<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=512x512 at 0x15DD6D06530>], 
    'id': ['8031efe0-1b5c-11ef-8929-000066532cad'], 
    'caption': ['The image is a non-contrasted computed tomography (CT) scan of the brain, showing the cerebral structures without any medical devices present. The region of interest, located centrally and in the middle of the image, exhibits an area of altered density, which is indicative of a brain hemorrhage. This area is distinct from the surrounding brain tissue, suggesting a possible hematoma or bleeding within the brain parenchyma. The location and characteristics of this abnormality may suggest a relationship with the surrounding brain tissue, potentially causing a mass effect or contributing to increased intracranial pressure.']
}

# 可視化image內容
from PIL import Image
import matplotlib.pyplot as plt

image = ds['train'][0]['image']  # 獲取第一張圖像

plt.imshow(image)
plt.axis('off')  # 不顯示座標軸
plt.show()

[
  {
    "messages": [
      {
        "content": "<image>他們是誰？",
        "role": "user"
      },
      {
        "content": "他們是拜仁慕尼黑的凱恩和格雷茨卡。",
        "role": "assistant"
      },
      {
        "content": "他們在做什麼？",
        "role": "user"
      },
      {
        "content": "他們在足球場上慶祝。",
        "role": "assistant"
      }
    ],
    "images": [
      "mllm_demo_data/1.jpg"
    ]
  }
]

from datasets import load_dataset
import os
import json
from PIL import Image

def save_images_and_json(ds, output_dir="mllm_data"):
    """
    將數據集中的圖像和對應的 JSON 信息保存到指定目錄。

    參數:
    ds: 數據集對象，包含圖像和標題。
    output_dir: 輸出目錄，默認為 "mllm_data"。
    """
    # 創建輸出目錄
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    # 創建一個列表來存儲所有的消息和圖像信息
    all_data = []

    # 遍歷數據集中的每個項目
    for item in ds:
        img_path = f"{output_dir}/{item['id']}.jpg"  # 圖像保存路徑
        image = item["image"]  # 假設這裏是一個 PIL 圖像對象

        # 將圖像對象保存為文件
        image.save(img_path)  # 使用 PIL 的 save 方法

        # 添加消息和圖像信息到列表中
        all_data.append(
            {
                "messages": [
                    {
                        "content": "<image>圖片中的診斷結果是怎樣?",
                        "role": "user",
                    },
                    {
                        "content": item["caption"],  # 從數據集中獲取的標題
                        "role": "assistant",
                    },
                ],
                "images": [img_path],  # 圖像文件路徑
            }
        )

    # 創建 JSON 文件
    json_file_path = f"{output_dir}/mllm_data.json"
    with open(json_file_path, "w", encoding='utf-8') as f:
        json.dump(all_data, f, ensure_ascii=False)  # 確保中文字符正常顯示

if __name__ == "__main__":
    # 加載數據集
    ds = load_dataset("UCSC-VLAA/MedTrinity-25M", "25M_demo", cache_dir="cache")

    # 保存數據集中的圖像和 JSON 信息
    save_images_and_json(ds['train'])

git lfs install
# 下載模型
git clone https://www.modelscope.cn/Qwen/Qwen2-VL-2B-Instruct.git

# 創建python3.10版本虛擬環境
conda create --name train_env python=3.10

# 激活環境
conda activate train_env

# 安裝依賴包
pip install streamlit torch torchvision

# 安裝Qwen2建議的transformers版本
pip install git+https://github.com/huggingface/transformers

# 原始代碼
require_version("transformers>=4.41.2,<=4.45.2", "To fix: pip install transformers>=4.41.2,<=4.45.2")

# 修改後代碼
require_version("transformers>=4.41.2,<=4.47.0", "To fix: pip install transformers>=4.41.2,<=4.47.0")

llamafactory-cli webui

# 卸載舊的 accelerate
pip uninstall accelerate

# 安裝新的 accelerate
pip install accelerate==0.34.0

"mllm_med": {
    "file_name": "mllm_data/mllm_data.json",
    "formatting": "sharegpt",
    "columns": {
        "messages": "messages",
        "images": "images"
    },
    "tags": {
        "role_tag": "role",
        "content_tag": "content",
        "user_tag": "user",
        "assistant_tag": "assistant"
    }
}

llamafactory-cli train \
    --do_train True \
    --model_name_or_path models/Qwen2-VL-2B-Instruct \
    --preprocessing_num_workers 16 \
    --finetuning_type lora \
    --template qwen2_vl \
    --flash_attn auto \
    --dataset_dir data \
    --dataset mllm_med \
    --cutoff_len 1024 \
    --learning_rate 5e-05 \
    --num_train_epochs 3.0 \
    --max_samples 100000 \
    --per_device_train_batch_size 2 \
    --gradient_accumulation_steps 8 \
    --lr_scheduler_type cosine \
    --max_grad_norm 1.0 \
    --logging_steps 5 \
    --save_steps 3000 \
    --warmup_steps 0 \
    --optim adamw_torch \
    --packing False \
    --report_to none \
    --output_dir saves/Qwen2-VL-2B/full/Qwen2-VL-sft-demo1 \
    --bf16 True \
    --plot_loss True \
    --ddp_timeout 180000000 \
    --include_num_input_tokens_seen True \
    --lora_rank 8 \
    --lora_alpha 16 \
    --lora_dropout 0 \
    --lora_target all