rob
/
torrents.csv
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixing scan torrents.

This commit is contained in:
Dessalines 2018-12-21 13:31:21 -08:00
parent 9234811618
commit 08a3aa9095
1 changed files with 54 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
scripts/scan_torrents.sh
View File

@ -6,6 +6,8 @@ cd ..
torrents_csv="`pwd`/torrents.csv" torrents_csv="`pwd`/torrents.csv"
scanned_out="`pwd`/infohashes_scanned.txt" scanned_out="`pwd`/infohashes_scanned.txt"
touch $scanned_out
help="Run ./scan_torrents.sh [TORRENTS_DIR] \nor goto https://gitlab.com/dessalines/torrents.csv for more help" help="Run ./scan_torrents.sh [TORRENTS_DIR] \nor goto https://gitlab.com/dessalines/torrents.csv for more help"
if [ "$1" == "-h" ] || [ -z "$1" ]; then if [ "$1" == "-h" ] || [ -z "$1" ]; then
echo -e $help echo -e $help
@ -26,82 +28,77 @@ fi
# Loop over all torrents # Loop over all torrents
pushd $torrents_dir pushd $torrents_dir
for torrent_file in *.torrent; do # for torrent_file in *.torrent; do
find `pwd` -name "*.torrent" | sort -n | grep -vFf $scanned_out | while read torrent_file ; do
file_infohash=$(basename $torrent_file | cut -d'.' -f 1) file_infohash=$(basename $torrent_file | cut -d'.' -f 1)
if rg -Nq $file_infohash $scanned_out; then # Scrape it
echo "$file_infohash already scanned" health_text=$(torrent-tracker-health --torrent $torrent_file --timeout 1000 --addTrackers={udp://tracker.coppersurfer.tk:6969/announce,udp://tracker.internetwarriors.net:1337/announce,udp://tracker.opentrackr.org:1337/announce,udp://exodus.desync.com:6969/announce,udp://explodie.org:6969/announce})
echo -e "$health_text"
infohash=$(jq -r '.hash' <<< $health_text)
name=$(jq -r '.name' <<< $health_text)
size_bytes=$(jq -r '.length' <<< $health_text)
seeders=$(jq -r '.seeds' <<< $health_text)
leechers=$(jq -r '.peers' <<< $health_text)
completed=$(jq -r '.completed' <<< $health_text)
date_string=$(jq -r '.created' <<< $health_text)
if [ "$date_string" == "null" ]; then
echo "Date was null, setting to now"
created_date=$(date +%s)
else else
created_date=$(date -d "${date_string}" +"%s")
fi
scraped_date=$(date +%s)
# Scrape it # Construct add line
health_text=$(torrent-tracker-health --torrent $torrent_file --timeout 1000 --addTrackers={udp://tracker.coppersurfer.tk:6969/announce,udp://tracker.internetwarriors.net:1337/announce,udp://tracker.opentrackr.org:1337/announce,udp://exodus.desync.com:6969/announce,udp://explodie.org:6969/announce}) add_line="$infohash;$name;$size_bytes;$created_date;$seeders;$leechers;$completed;$scraped_date"
# echo -e $add_line
echo -e "$health_text" if (( $seeders > 0 )); then
infohash=$(jq -r '.hash' <<< $health_text) found_line=$(rg -n $infohash $torrents_csv)
name=$(jq -r '.name' <<< $health_text) found_seeders=$(echo -e $found_line | cut -d';' -f 5)
size_bytes=$(jq -r '.length' <<< $health_text)
seeders=$(jq -r '.seeds' <<< $health_text)
leechers=$(jq -r '.peers' <<< $health_text)
completed=$(jq -r '.completed' <<< $health_text)
date_string=$(jq -r '.created' <<< $health_text)
if [ "$date_string" == "null" ]; then
echo "Date was null, setting to now"
created_date=$(date +%s)
else
created_date=$(date -d "${date_string}" +"%s")
fi
scraped_date=$(date +%s)
# Construct add line # Only re-add if the infohash doesn't exist, or
add_line="$infohash;$name;$size_bytes;$created_date;$seeders;$leechers;$completed;$scraped_date" if [ ! -z "$found_line" ]; then
# echo -e $add_line
if (( $seeders > 0 )); then # Seeder counts are different
if [ "$found_seeders" != "$seeders" ]; then
found_line=$(rg -n $infohash $torrents_csv) # Delete the original infohash line
found_seeders=$(echo -e $found_line | cut -d';' -f 5) rg -Nv "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv
# Only re-add if the infohash doesn't exist, or
if [ ! -z "$found_line" ]; then
# Seeder counts are different
if [ "$found_seeders" != "$seeders" ]; then
# Delete the original infohash line
rg -Nv "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv
# Append the add lines to the torrents.csv file
echo -e "\n$add_line" >> $torrents_csv
# truncate -s -1 $torrents_csv # Removing last newline
echo -e "Updating Seeders: $torrent_file | $name | $infohash | $seeders"
else
echo -e "Not adding $name, had identical seeders"
fi
else
# Append the add lines to the torrents.csv file # Append the add lines to the torrents.csv file
echo -e "\n$add_line" >> $torrents_csv echo -e "\n$add_line" >> $torrents_csv
# truncate -s -1 $torrents_csv # Removing last newline # truncate -s -1 $torrents_csv # Removing last newline
echo -e "New Torrent: $torrent_file | $name | $infohash | $seeders" echo -e "Updating Seeders: $torrent_file | $name | $infohash | $seeders"
fi else
echo -e "Not adding $name, had identical seeders"
else
# Deleting the line if it existed
if [ ! -z "$infohash" ]; then
# This removes lines that have no seeders
echo -e "$name has no seeders"
if rg -Nq $infohash $torrents_csv; then
echo "Removing $name from $torrents_csv"
rg -Nv "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv
fi
fi fi
else
# Append the add lines to the torrents.csv file
echo -e "\n$add_line" >> $torrents_csv
# truncate -s -1 $torrents_csv # Removing last newline
echo -e "New Torrent: $torrent_file | $name | $infohash | $seeders"
fi fi
echo $file_infohash >> $scanned_out else
# Deleting the line if it existed
if [ ! -z "$infohash" ]; then
# This removes lines that have no seeders
echo -e "$name has no seeders"
if rg -Nq $infohash $torrents_csv; then
echo "Removing $name from $torrents_csv"
rg -Nv "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv
fi
fi
fi fi
echo $file_infohash >> $scanned_out
done done
popd popd