Views
No views yet
sparse-checkout --add.1!GIT_LFS_SPARSE_CHECKOUT=1 git clone git@hf.co:anonderpling/civitai_mirror # this is my command so I can push changes, you'll need to use the https://hf.co/ instead of git@hf.co:
2!cd civitai_mirror
3!git sparse-checkout set embeddings # embeddings are small, so it's easy enough to just pull all of them
4!git sparse-checkout add models/VAE # there's only a few VAEs, and they're generally needed, so grab all those too...
5!git sparse-checkout add models/Stable-diffusion/illuminati* models/Stable-diffusion/revAnimated* # add some stable diffusion models I intend to work with in this session
6!apt install aria2 # make sure aria2c is installed
7# let's break the following command down into parts, since there's multiple commands on one line
8# find models embeddings --type f --size -2 # find files in models and embeddings directories smaller than 2 kilobytes (these are the lfs pointers that were checked out)
9# | while read a; do #lets build an aria2c input file
10# echo "https://huggingface.co/anonderpling/civitai_mirror/resolve/main/${a}"; # tell aria2c where to find the file
11# echo " out=${a}"; # tell aria2c where to place said file
12# rm "${a}"; remove the existing file, because I'm too lazy to look up the option to have aria2c overwrite it (plus if you stop in the middle, you can tell at a glance what else is needed)
13# done | tee aria2.in.txt # end the loop, but watch to make sure theres nothing accidentally included by wildcards that shouldnt have been...downloads could take a while (and fill the disk) if I accidentally put a space before the *
14# aria2 -x16 --split=16 -i aria2.in.txt # download all the files as fast as possible
15!find models embeddings --type f --size -2 | while read a; do echo "https://huggingface.co/anonderpling/civitai_mirror/resolve/main/${a}"; echo " out=${a}"; rm "${a}"; done | tee aria2.in.txt
16!aria2 -x16 --split=16 -i aria2.in.txt 1# enable the git lfs filters
2!pip install huggingface_hub
3!huggingface-cli lfs_enable_largefiles .
4# yup, not telling. I'm an *anonymous* derpling, after all
5!git config --local user.name 'not telling'
6# really couldnt care less if this is accurate...maybe I'll start randomizing it...
7!git config --local user.email 'anonderpling@users.noreply.huggingface.co'
8# create an rsa key with no password
9!ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_hf.co -P ''
10# a clickable link in jupyter/colab/paperspace
11print('https://hf.co/settings/keys')
12# give the public key so it can be easily copied to huggingface
13!cat ~/.ssh/id_hf.co.pub
14# track files 1mb+ with lfs manually (huggingface filter deals with models automatically, but large previews will give you errors)
15!find -type f -size +999k -not \( -name '*.safetensors' -o -name '*.ckpt' -o -name '*.pt' \) -exec git lfs track '{}' +
16# IMPORTANT: make sure you add the git ssh key above before uploading
17!sleep 1m # gives you time to do so
18# upload your files now. do make sure you dont upload files that didnt download properly (interrupted aria2c, lfs pointers, etc)
19!git add .gitattributes models embeddings
20!git commit -m "add a message..."
21!git push1inurl=api.upload_folder( # https://huggingface.co/docs/huggingface_hub/v0.14.1/en/package_reference/hf_api#huggingface_hub.HfApi.upload_folder
2 token="hf_token", # only needed if you didn't already use huggingface_hub.login() previously.
3 path_or_fileobj="/content/stable-diffusion-webui/models/Stable-diffusion/Rev-Animated.safetensors", # location of the file you want to upload
4 path_in_repo="models/Stable-diffusion/Rev-Animated.safetensors",
5 repo_id="mirroring/civitai_mirror",
6 commit_message="Uploading Rev Animated", # optional.
7 commit_description="I want to upload Rev Animated because it's a special file for me.\nPlease accept my PR. I don't want to host it on my own HF repo!" # optional. Required (here) for PRs.
8 create_pr=True # optional. needed if you're not a special person allowed to add new files to the repo (ie, if you just want us to mirror something; make sure to fill out the description/message above, as well)
9)
10print("Uploaded files. Check them out at "+inurl) 1inurl=api.upload_folder( # https://huggingface.co/docs/huggingface_hub/v0.14.1/en/package_reference/hf_api#huggingface_hub.HfApi.upload_folder
2 token="hf_token", # only needed if you didn't already use huggingface_hub.login() previously.
3 folder_path="/stable-diffusion-webui/models/",
4 path_in_repo="models",
5 repo_id="mirroring/civitai_mirror",
6 allow_patterns="*.safetensors", # optional. Only upload certain files.
7 ignore_patterns=["*.tmp","tmp/*","*.jpg"], # optional. Ignore certain files.
8 commit_message="Uploading my own models",
9 commit_description="I want to upload these models because..."
10 create_pr=True # needed if you're not a special person allowed to add new files to the repo (ie, if you just want us to mirror something)
11)
12print("Uploaded files. Check them out at "+inurl)