基于diffusers训练lora，AI换装

阿里云登录 – 欢迎登录阿里云，安全稳定的云计算服务平台欢迎登录阿里云，全球领先的云计算及人工智能科技公司，阿里云为200多个国家和地区的企业、开发者和政府机构提供云计算基础服务及解决方案。阿里云云计算、安全、大数据、人工智能、企业应用、物联网等云计算服务。https://pai.console.aliyun.com/?regionId=cn-shanghai&spm=5176.12818093.ProductAndResource–ali–widget-product-recent.dre0.2f5b16d01JhuDy&workspaceId=322809#/dsw-gallery-workspace?category=all&pageNum=1在阿里云PAI平台上拍

torch1.13-gpu-py310-cu117  V100 16显存

1.diffusions安装

1.1 下载diffussers库并安装依赖

! git clone https://github.com/huggingface/diffusers
! cd diffusers && git checkout e126a82cc5d9afbeb9b476455de24dd3e7dd358a
! cd diffusers && pip install .

1.2 验证是够成功安装

import diffusers

1.3 配置accelerate

! mkdir -p /root/.cache/huggingface/accelerate/
! wget -c http://pai-vision-data-sh.oss-cn-shanghai.aliyuncs.com/aigc-data/accelerate/default_config.yaml -O /root/.cache/huggingface/accelerate/default_config.yaml

1.4 安装文生图算法相关依赖

! cd diffusers/examples/text_to_image && pip install -r requirements.txt

1.5 下载stable-diffusion-webui

import os

! apt update
! apt install -y aria2

def aria2(url, filename, d):
    !aria2c --console-log-level=error -c -x 16 -s 16 {url} -o {filename} -d {d}
    
url_prefix = {
    "cn-shanghai": "http://pai-vision-data-sh.oss-cn-shanghai-internal.aliyuncs.com",
    "cn-hangzhou": "http://pai-vision-data-hz2.oss-cn-hangzhou-internal.aliyuncs.com",
    "cn-shenzhen": "http://pai-vision-data-sz.oss-cn-shenzhen-internal.aliyuncs.com",
    "cn-beijing": "http://pai-vision-data-bj.oss-cn-beijing-internal.aliyuncs.com", 
}

dsw_region = os.environ.get("dsw_region")
prefix = url_prefix[dsw_region] if dsw_region in url_prefix else "http://pai-vision-data-sh.oss-cn-shanghai.aliyuncs.com"


! git clone https://gitcode.net/mirrors/AUTOMATIC1111/stable-diffusion-webui.git
%cd stable-diffusion-webui
! git checkout a9fed7c364061ae6efb37f797b6b522cb3cf7aa2
repositories_url = f"{prefix}/aigc-data/code/repositories.tar.gz"
aria2(repositories_url, repositories_url.split("/")[-1], "./")
! tar -xf repositories.tar.gz
%cd extensions
! git clone https://gitcode.net/mirrors/DominikDoom/a1111-sd-webui-tagcomplete.git
! git clone https://gitcode.net/ranting8323/stable-diffusion-webui-localization-zh_CN
%cd .. 
! wget -c http://pai-vision-data-sh.oss-cn-shanghai.aliyuncs.com/aigc-data/webui_config/config_tryon.json -O config.json
%cd ..

2.stable diffusion+lora模型finetune

2.1 准备数据集以及训练代码

! wget http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/EasyCV/datasets/try_on/cloth_train_example.tar.gz && tar -xvf cloth_train_example.tar.gz
! wget http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/EasyCV/datasets/try_on/train_text_to_image_lora.py

2.2 查看示例

from PIL import Image
display(Image.open("cloth_train_example/train/20230407174450.jpg"))

2.3 下载预训练模型病转成diffusers格式，可在hf上下载

safety_checker_url = f"{prefix}/aigc-data/hug_model/models--CompVis--stable-diffusion-safety-checker.tar.gz"
aria2(safety_checker_url, safety_checker_url.split("/")[-1], "./")
! tar -xf models--CompVis--stable-diffusion-safety-checker.tar.gz -C /root/.cache/huggingface/hub/

clip_url = f"{prefix}/aigc-data/hug_model/models--openai--clip-vit-large-patch14.tar.gz"
aria2(clip_url, clip_url.split("/")[-1], "./")
! tar -xf models--openai--clip-vit-large-patch14.tar.gz -C /root/.cache/huggingface/hub/

model_url = f"{prefix}/aigc-data/sd_models/chilloutmix_NiPrunedFp32Fix.safetensors"
aria2(model_url, model_url.split("/")[-1], "stable-diffusion-webui/models/Stable-diffusion/")

! python diffusers/scripts/convert_original_stable_diffusion_to_diffusers.py \
--checkpoint_path=stable-diffusion-webui/models/Stable-diffusion/chilloutmix_NiPrunedFp32Fix.safetensors \
--dump_path=chilloutmix-ni --from_safetensors

2.4 模型训练

! export MODEL_NAME="chilloutmix-ni" && \
export DATASET_NAME="cloth_train_example" && \
accelerate launch --mixed_precision="fp16" train_text_to_image_lora.py \
  --pretrained_model_name_or_path=$MODEL_NAME \
  --dataset_name=$DATASET_NAME --caption_column="text" \
  --width=640 --height=768 --random_flip \
  --train_batch_size=1 \
  --num_train_epochs=200 --checkpointing_steps=5000 \
  --learning_rate=1e-04 --lr_scheduler="constant" --lr_warmup_steps=0 \
  --seed=42 \
  --output_dir="cloth-model-lora" \
  --validation_prompt="cloth1" --validation_epochs=100

2.5 准备webui所需的模型文件

2.5.1 将lora模型转化成webui支持格式并拷贝至webui所在目录

! wget -c http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/EasyCV/datasets/convert-to-safetensors.py
! python convert-to-safetensors.py --file='cloth-model-lora/pytorch_lora_weights.bin'
! mkdir stable-diffusion-webui/models/Lora
! cp cloth-model-lora/pytorch_lora_weights_converted.safetensors stable-diffusion-webui/models/Lora/cloth_lora_weights.safetensors

2.5.2 准备额外文件，可以在hf上下载

detection_url = f"{prefix}/aigc-data/codeformer/detection_Resnet50_Final.pth"
aria2(detection_url, detection_url.split("/")[-1], "stable-diffusion-webui/repositories/CodeFormer/weights/facelib/")
parse_url = f"{prefix}/aigc-data/codeformer/parsing_parsenet.pth"
aria2(parse_url, parse_url.split("/")[-1], "stable-diffusion-webui/repositories/CodeFormer/weights/facelib/")
codeformer_url = f"{prefix}/aigc-data/codeformer/codeformer-v0.1.0.pth"
aria2(codeformer_url, codeformer_url.split("/")[-1], "stable-diffusion-webui/models/Codeformer/")

embedding_url = f"{prefix}/aigc-data/embedding/ng_deepnegative_v1_75t.pt"
aria2(embedding_url, embedding_url.split("/")[-1], "stable-diffusion-webui/embeddings/")

model_lora_url = f"{prefix}/aigc-data/lora/koreanDollLikeness_v10.safetensors"
aria2(model_lora_url, model_lora_url.split("/")[-1], "stable-diffusion-webui/models/Lora/")

3.启动webui

! cd stable-diffusion-webui && python -m venv --system-site-packages --symlinks venv
! cd stable-diffusion-webui && \
  sed -i 's/can_run_as_root=0/can_run_as_root=1/g' webui.sh && \
  ./webui.sh --no-download-sd-model --xformers

4.参数

正向prompt： cloth1,<lora:koreanDollLikeness_v10:0.4>, (extremely detailed CG unity 8k wallpaper),(RAW photo, best quality), (realistic, photo-realistic:1.2), a close up portrait photo, 1girl, shopping mall rooftop cafe, outdoor, smile, (high detailed skin:1.4), puffy eyes, gorgeous hair, air bangs, brown black hair, soft lighting, high quality, <lora:cloth_lora_weights:1>
负向prompt： ng_deepnegative_v1_75t,paintings, sketches, (worst quality:2), (low quality:2), (normal quality:2), lowres, ((monochrome)), (grayscale:1.2), skin spots, acnes, skin blemishes, age spot, glans,extra fingers,fewer fingers,(watermark:1.2),(letters:1.2),(nsfw:1.2),teeth
采样方法： Eular a
采样步数： 50
宽高： 640，768
随机种子： 1400244389
CFG scale 7
使用面部修复