Views
No views yet
pip install aiohttp
python crawler.py seeds.txt output_dir/ --workers 100
1curl -sL https://tranco-list.eu/top-1m.csv.zip -o tranco.zip && unzip tranco.zip 2awk -F, '{print "https://"$2"/"}' top-1m.csv > seeds.txt
