OpenELM

----------------
ubuntu 22.04
python 3.10
cuda 12.1
pytorch 2.1.0
----------------

python -m pip install --upgrade pip
# 更换 pypi 源加速库的安装
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple

pip install modelscope==1.16.1
pip install transformers==4.42.4
pip install fastapi==0.111.1
pip install uvicorn==0.30.3
pip install SentencePiece==0.2.0
pip install accelerate==0.33.0

modelscope download --model shakechen/Llama-2-7b-hf  --local_dir /root/autodl-tmp/Llama-2-7b-hf --exclude ".bin" "*.safetensors" "configuration.json" 
modelscope download --model LLM-Research/OpenELM-3B-Instruct --local_dir /root/autodl-tmp/OpenELM-3B-Instruct

from fastapi import FastAPI, Request
from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
import uvicorn
import json
import datetime
import torch

# 设置设备参数
DEVICE = "cuda"  # 使用CUDA
DEVICE_ID = "0"  # CUDA设备ID，如果未设置则为空
CUDA_DEVICE = f"{DEVICE}:{DEVICE_ID}" if DEVICE_ID else DEVICE  # 组合CUDA设备信息

# 清理GPU内存函数
def torch_gc():
    if torch.cuda.is_available():  # 检查是否可用CUDA
        with torch.cuda.device(CUDA_DEVICE):  # 指定CUDA设备
            torch.cuda.empty_cache()  # 清空CUDA缓存
            torch.cuda.ipc_collect()  # 收集CUDA内存碎片

# 创建FastAPI应用
app = FastAPI()

# 处理POST请求的端点
@app.post("/")
async def create_item(request: Request):
    global model, tokenizer  # 声明全局变量以便在函数内部使用模型和分词器
    json_post_raw = await request.json()  # 获取POST请求的JSON数据
    json_post = json.dumps(json_post_raw)  # 将JSON数据转换为字符串
    json_post_list = json.loads(json_post)  # 将字符串转换为Python对象
    prompt = json_post_list.get('prompt')  # 获取请求中的提示

    # 调用模型进行对话生成
    model_inputs = tokenizer(prompt, add_special_tokens=True, return_tensors="pt")['input_ids'].cuda()
    generated_ids = model.generate(model_inputs, max_length=384)
    response = tokenizer.decode(generated_ids[0], skip_special_tokens=True)
    now = datetime.datetime.now()  # 获取当前时间
    time = now.strftime("%Y-%m-%d %H:%M:%S")  # 格式化时间为字符串
    # 构建响应JSON
    answer = {
        "response": response,
        "status": 200,
        "time": time
    }
    # 构建日志信息
    log = "[" + time + "] " + '", prompt:"' + prompt + '", response:"' + repr(response) + '"'
    print(log)  # 打印日志
    torch_gc()  # 执行GPU内存清理
    return answer  # 返回响应

# 主函数入口
if __name__ == '__main__':
    # 加载预训练的分词器和模型
    model_path = '/root/autodl-tmp/OpenELM-3B-Instruct'
    tokenizer_path = '/root/autodl-tmp/Llama-2-7b-hf'
    tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=False, trust_remote_code=True)
    model = AutoModelForCausalLM.from_pretrained(model_path, device_map="auto", torch_dtype=torch.bfloat16, trust_remote_code=True)

    # 启动FastAPI应用
    # 用6006端口可以将autodl的端口映射到本地，从而在本地使用api
    uvicorn.run(app, host='0.0.0.0', port=6006, workers=1)  # 在指定端口和主机上启动应用

cd /root/autodl-tmp
python api.py

python /root/api.py

curl -X POST "http://127.0.0.1:6006" \
     -H 'Content-Type: application/json' \
     -d '{"prompt": "Once upon a time there was"}'

import requests
import json

def get_completion(prompt):
    headers = {'Content-Type': 'application/json'}
    data = {"prompt": prompt}
    response = requests.post(url='http://127.0.0.1:6006', headers=headers, data=json.dumps(data))
    print(json.dumps(response.json()))
    return response.json()['response']

if __name__ == '__main__':
    response = get_completion('Once upon a time there was')

{"response": "Once upon a time there was a little girl named Rosie. Rosie loved to play dress-up, and her favorite costume was a princess dress. Rosie's mommy and daddy dressed her up in her princess dress every chance they got. Rosie loved her princess dress so much, she even wore it to bed! Rosie's mommy and daddy loved Rosie very much, and they wanted Rosie to have the best life possible.\n\nOne day Rosie's mommy and daddy took Rosie to visit their friends, the Johnsons. Rosie's mommy and daddy told Rosie all about their friends, and Rosie couldn't wait to meet them. Rosie's mommy and daddy dropped Rosie off at the Johnsons' house, and Rosie ran inside to find her princess dress waiting for her. Rosie put on her princess dress and tiptoed upstairs to meet her friends.\n\nWhen Rosie walked into the Johnsons' living room, she saw her friends sitting on the couch, dressed in jeans and t-shirts. Rosie smiled and ran over to give her friends a hug. Rosie's friends were so happy to see her dressed up like a princess! They asked Rosie lots of questions about her princess dress, and Rosie told them all about her mommy and daddy's special tradition. Rosie's friends loved hearing all about her princess dress, and they promised to wear their princess dresses whenever Rosie visited them.\n\nAfter playing dress-up for a while, Rosie's friends asked Rosie if she wanted to play outside. Rosie loved playing with her friends, but she also loved playing princess dress-up.", "status": 200, "time": "2024-08-24 21:28:34"}

----------------
ubuntu 22.04
python 3.10
cuda 12.1
pytorch 2.1.0
----------------

python -m pip install --upgrade pip
# 更换 pypi 源加速库的安装
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple

pip install modelscope==1.16.1
pip install transformers==4.42.4
pip install datasets==2.20.0
pip install peft==0.11.1
pip install fastapi==0.111.1
pip install uvicorn==0.30.3
pip install SentencePiece==0.2.0
pip install accelerate==0.33.0

modelscope download --model shakechen/Llama-2-7b-hf  --local_dir /root/autodl-tmp/Llama-2-7b-hf --exclude ".bin" "*.safetensors" "configuration.json" 
modelscope download --model LLM-Research/OpenELM-3B-Instruct --local_dir /root/autodl-tmp/OpenELM-3B-Instruct

{
    "instruction":"回答以下用户问题，仅输出答案。",
    "input":"1+1等于几?",
    "output":"2"
}

cd /root/autodl-tmp
git clone https://mirror.ghproxy.com/https://github.com/open-chinese/alpaca-chinese-dataset

def process_func(example):
    MAX_LENGTH = 384
    
    instruction = tokenizer(f"{example['en_instruction'] + example['en_input']}<sep>", add_special_tokens=True)
    response = tokenizer(f"{example['en_output']}", add_special_tokens=False)

    input_ids = instruction["input_ids"] + response["input_ids"] + [tokenizer.pad_token_id]
    attention_mask = instruction["attention_mask"] + response["attention_mask"] + [1]
    labels = [-100] * len(instruction["input_ids"]) + response["input_ids"] + [tokenizer.pad_token_id]  

    
    if len(input_ids) > MAX_LENGTH:  # 做一个截断
        input_ids = input_ids[:MAX_LENGTH]
        attention_mask = attention_mask[:MAX_LENGTH]
        labels = labels[:MAX_LENGTH]
    return {
        "input_ids": input_ids,
        "attention_mask": attention_mask,
        "labels": labels
    }

from peft import LoraConfig, TaskType, get_peft_model
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, DataCollatorForSeq2Seq, TrainingArguments, Trainer, GenerationConfig

tokenizer = AutoTokenizer.from_pretrained(
    '/root/autodl-tmp/Llama-2-7b-hf',
    trust_remote_code=True
)
tokenizer.pad_token = tokenizer.eos_token
model = AutoModelForCausalLM.from_pretrained(
    '/root/autodl-tmp/OpenELM-3B-Instruct',
    device_map="auto",
    torch_dtype=torch.bfloat16,
    trust_remote_code=True
)

config = LoraConfig(
    task_type=TaskType.CAUSAL_LM, 
    target_modules=['token_embeddings', "qkv_proj", "out_proj", "proj_1", "proj_2"],
    inference_mode=False,
    r=32, # Lora 秩
    lora_alpha=32, # Lora alaph，具体作用参见 Lora 原理
    lora_dropout=0.1 # Dropout 比例
)

args = TrainingArguments(
    output_dir="autodl-tmp/output/openelm_3B_lora",
    per_device_train_batch_size=8,
    gradient_accumulation_steps=4,
    logging_steps=100,
    num_train_epochs=0.87,  # 为了快速掩饰，我们训练到约1200个iter作为测试，建议设为10个epochs
    save_steps=600,
    learning_rate=1e-4,
    save_on_each_node=True,
    gradient_checkpointing=True
)

trainer = Trainer(
    model=peft_model,
    args=args,
    train_dataset=tokenized_id,
    data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True),
)
trainer.train()

from peft import LoraConfig, TaskType, get_peft_model, PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

mode_path = '/root/autodl-tmp/OpenELM-3B-Instruct'
lora_path = '/root/autodl-tmp/output/openelm_3B_lora/checkpoint-1200' # 这里改称你的 lora 输出对应 checkpoint 地址

# 加载tokenizer
tokenizer = AutoTokenizer.from_pretrained('/root/autodl-tmp/Llama-2-7b-hf', trust_remote_code=True)
# 加载模型
model = AutoModelForCausalLM.from_pretrained(mode_path, device_map="auto",torch_dtype=torch.bfloat16, trust_remote_code=True).eval()

# 加载lora权重
model = PeftModel.from_pretrained(model, model_id=lora_path)

prompt = "How to be a good learner?<sep>"
instruction = tokenizer(prompt, add_special_tokens=True, return_tensors="pt")

generated_ids = model.generate(instruction['input_ids'].cuda(), max_length=384)
print(tokenizer.decode(generated_ids[0], skip_special_tokens=True).split('<sep>')[-1])

'''
To be a good learner, it is important to be motivated, organized, and have a positive attitude. 
Motivation is key to learning, as it helps to keep you focused and engaged. Organization is 
important to ensure that you have the materials you need and that you are able to stay on track. 
Finally, a positive attitude is essential to staying motivated and to help you stay focused on 
the task at hand.
'''