diff --git a/scripts/scan_torrents.sh b/scripts/scan_torrents.sh index 205bf4d..97b5b4e 100755 --- a/scripts/scan_torrents.sh +++ b/scripts/scan_torrents.sh @@ -6,6 +6,8 @@ cd .. torrents_csv="`pwd`/torrents.csv" scanned_out="`pwd`/infohashes_scanned.txt" +touch $scanned_out + help="Run ./scan_torrents.sh [TORRENTS_DIR] \nor goto https://gitlab.com/dessalines/torrents.csv for more help" if [ "$1" == "-h" ] || [ -z "$1" ]; then echo -e $help @@ -26,82 +28,77 @@ fi # Loop over all torrents pushd $torrents_dir -for torrent_file in *.torrent; do +# for torrent_file in *.torrent; do +find `pwd` -name "*.torrent" | sort -n | grep -vFf $scanned_out | while read torrent_file ; do file_infohash=$(basename $torrent_file | cut -d'.' -f 1) - if rg -Nq $file_infohash $scanned_out; then - echo "$file_infohash already scanned" + # Scrape it + health_text=$(torrent-tracker-health --torrent $torrent_file --timeout 1000 --addTrackers={udp://tracker.coppersurfer.tk:6969/announce,udp://tracker.internetwarriors.net:1337/announce,udp://tracker.opentrackr.org:1337/announce,udp://exodus.desync.com:6969/announce,udp://explodie.org:6969/announce}) + + echo -e "$health_text" + + infohash=$(jq -r '.hash' <<< $health_text) + name=$(jq -r '.name' <<< $health_text) + size_bytes=$(jq -r '.length' <<< $health_text) + seeders=$(jq -r '.seeds' <<< $health_text) + leechers=$(jq -r '.peers' <<< $health_text) + completed=$(jq -r '.completed' <<< $health_text) + date_string=$(jq -r '.created' <<< $health_text) + if [ "$date_string" == "null" ]; then + echo "Date was null, setting to now" + created_date=$(date +%s) else + created_date=$(date -d "${date_string}" +"%s") + fi + scraped_date=$(date +%s) - # Scrape it - health_text=$(torrent-tracker-health --torrent $torrent_file --timeout 1000 --addTrackers={udp://tracker.coppersurfer.tk:6969/announce,udp://tracker.internetwarriors.net:1337/announce,udp://tracker.opentrackr.org:1337/announce,udp://exodus.desync.com:6969/announce,udp://explodie.org:6969/announce}) + # Construct add line + add_line="$infohash;$name;$size_bytes;$created_date;$seeders;$leechers;$completed;$scraped_date" + # echo -e $add_line - echo -e "$health_text" + if (( $seeders > 0 )); then - infohash=$(jq -r '.hash' <<< $health_text) - name=$(jq -r '.name' <<< $health_text) - size_bytes=$(jq -r '.length' <<< $health_text) - seeders=$(jq -r '.seeds' <<< $health_text) - leechers=$(jq -r '.peers' <<< $health_text) - completed=$(jq -r '.completed' <<< $health_text) - date_string=$(jq -r '.created' <<< $health_text) - if [ "$date_string" == "null" ]; then - echo "Date was null, setting to now" - created_date=$(date +%s) - else - created_date=$(date -d "${date_string}" +"%s") - fi - scraped_date=$(date +%s) + found_line=$(rg -n $infohash $torrents_csv) + found_seeders=$(echo -e $found_line | cut -d';' -f 5) - # Construct add line - add_line="$infohash;$name;$size_bytes;$created_date;$seeders;$leechers;$completed;$scraped_date" - # echo -e $add_line + # Only re-add if the infohash doesn't exist, or + if [ ! -z "$found_line" ]; then - if (( $seeders > 0 )); then + # Seeder counts are different + if [ "$found_seeders" != "$seeders" ]; then - found_line=$(rg -n $infohash $torrents_csv) - found_seeders=$(echo -e $found_line | cut -d';' -f 5) + # Delete the original infohash line + rg -Nv "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv - # Only re-add if the infohash doesn't exist, or - if [ ! -z "$found_line" ]; then - - # Seeder counts are different - if [ "$found_seeders" != "$seeders" ]; then - - # Delete the original infohash line - rg -Nv "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv - - # Append the add lines to the torrents.csv file - echo -e "\n$add_line" >> $torrents_csv - # truncate -s -1 $torrents_csv # Removing last newline - echo -e "Updating Seeders: $torrent_file | $name | $infohash | $seeders" - else - echo -e "Not adding $name, had identical seeders" - fi - else # Append the add lines to the torrents.csv file echo -e "\n$add_line" >> $torrents_csv # truncate -s -1 $torrents_csv # Removing last newline - echo -e "New Torrent: $torrent_file | $name | $infohash | $seeders" - fi - - else - # Deleting the line if it existed - if [ ! -z "$infohash" ]; then - # This removes lines that have no seeders - echo -e "$name has no seeders" - if rg -Nq $infohash $torrents_csv; then - echo "Removing $name from $torrents_csv" - rg -Nv "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv - fi + echo -e "Updating Seeders: $torrent_file | $name | $infohash | $seeders" + else + echo -e "Not adding $name, had identical seeders" fi + else + # Append the add lines to the torrents.csv file + echo -e "\n$add_line" >> $torrents_csv + # truncate -s -1 $torrents_csv # Removing last newline + echo -e "New Torrent: $torrent_file | $name | $infohash | $seeders" fi - echo $file_infohash >> $scanned_out - + else + # Deleting the line if it existed + if [ ! -z "$infohash" ]; then + # This removes lines that have no seeders + echo -e "$name has no seeders" + if rg -Nq $infohash $torrents_csv; then + echo "Removing $name from $torrents_csv" + rg -Nv "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv + fi + fi fi + echo $file_infohash >> $scanned_out + done popd