Views
No views yet
1git clone https://github.com/antgroup/ditto-talkinghead
2cd ditto-talkingheadconda environment:1conda env create -f environment.yaml
2conda activate dittopytorch, cuda and cudnn, you only need to install a few packages using pip:1pip install \
2 tensorrt==8.6.1 \
3 librosa \
4 tqdm \
5 filetype \
6 imageio \
7 opencv_python_headless \
8 scikit-image \
9 cython \
10 cuda-python \
11 imageio-ffmpeg \
12 colored \
13 polygraphy \
14 numpy==2.0.1checkpoints dir:1git lfs install
2git clone https://huggingface.co/digital-avatar/ditto-talkinghead checkpointscheckpoints should be like:1./checkpoints/
2├── ditto_cfg
3│ ├── v0.4_hubert_cfg_trt.pkl
4│ └── v0.4_hubert_cfg_trt_online.pkl
5├── ditto_onnx
6│ ├── appearance_extractor.onnx
7│ ├── blaze_face.onnx
8│ ├── decoder.onnx
9│ ├── face_mesh.onnx
10│ ├── hubert.onnx
11│ ├── insightface_det.onnx
12│ ├── landmark106.onnx
13│ ├── landmark203.onnx
14│ ├── libgrid_sample_3d_plugin.so
15│ ├── lmdm_v0.4_hubert.onnx
16│ ├── motion_extractor.onnx
17│ ├── stitch_network.onnx
18│ └── warp_network.onnx
19└── ditto_trt_Ampere_Plus
20 ├── appearance_extractor_fp16.engine
21 ├── blaze_face_fp16.engine
22 ├── decoder_fp16.engine
23 ├── face_mesh_fp16.engine
24 ├── hubert_fp32.engine
25 ├── insightface_det_fp16.engine
26 ├── landmark106_fp16.engine
27 ├── landmark203_fp16.engine
28 ├── lmdm_v0.4_hubert_fp32.engine
29 ├── motion_extractor_fp32.engine
30 ├── stitch_network_fp16.engine
31 └── warp_network_fp16.engineditto_cfg/v0.4_hubert_cfg_trt_online.pkl is online configditto_cfg/v0.4_hubert_cfg_trt.pkl is offline configinference.py:1python inference.py \
2 --data_root "<path-to-trt-model>" \
3 --cfg_pkl "<path-to-cfg-pkl>" \
4 --audio_path "<path-to-input-audio>" \
5 --source_path "<path-to-input-image>" \
6 --output_path "<path-to-output-mp4>" 1python inference.py \
2 --data_root "./checkpoints/ditto_trt_Ampere_Plus" \
3 --cfg_pkl "./checkpoints/ditto_cfg/v0.4_hubert_cfg_trt.pkl" \
4 --audio_path "./example/audio.wav" \
5 --source_path "./example/image.png" \
6 --output_path "./tmp/result.mp4" hardware-compatibility-level=Ampere_Plus (checkpoints/ditto_trt_Ampere_Plus/). If your GPU does not support it, please execute the cvt_onnx_to_trt.py script to convert from the general onnx model (checkpoints/ditto_onnx/) to the tensorRT model.python scripts/cvt_onnx_to_trt.py --onnx_dir "./checkpoints/ditto_onnx" --trt_dir "./checkpoints/ditto_trt_custom"inference.py with --data_root=./checkpoints/ditto_trt_custom.checkpoints should be like:1./checkpoints/
2├── ditto_cfg
3│ ├── ...
4│ └── v0.4_hubert_cfg_pytorch.pkl
5├── ...
6└── ditto_pytorch
7 ├── aux_models
8 │ ├── 2d106det.onnx
9 │ ├── det_10g.onnx
10 │ ├── face_landmarker.task
11 │ ├── hubert_streaming_fix_kv.onnx
12 │ └── landmark203.onnx
13 └── models
14 ├── appearance_extractor.pth
15 ├── decoder.pth
16 ├── lmdm_v0.4_hubert.pth
17 ├── motion_extractor.pth
18 ├── stitch_network.pth
19 └── warp_network.pth1python inference.py \
2 --data_root "./checkpoints/ditto_pytorch" \
3 --cfg_pkl "./checkpoints/ditto_cfg/v0.4_hubert_cfg_pytorch.pkl" \
4 --audio_path "./example/audio.wav" \
5 --source_path "./example/image.png" \
6 --output_path "./tmp/result.mp4" 1@article{li2024ditto,
2 title={Ditto: Motion-Space Diffusion for Controllable Realtime Talking Head Synthesis},
3 author={Li, Tianqi and Zheng, Ruobing and Yang, Minghui and Chen, Jingdong and Yang, Ming},
4 journal={arXiv preprint arXiv:2411.19509},
5 year={2024}
6}