Fixing scanning.
This commit is contained in:
Dessalines
parent
ad17496f28
commit
8a2d934048
|
|
@ -4,6 +4,7 @@
|
||||||
# Help line
|
# Help line
|
||||||
cd ..
|
cd ..
|
||||||
torrents_csv="`pwd`/torrents.csv"
|
torrents_csv="`pwd`/torrents.csv"
|
||||||
|
scanned_out="`pwd`/infohashes_scanned.txt"
|
||||||
|
|
||||||
help="Run ./scan_torrents.sh [TORRENTS_DIR] \nor goto https://gitlab.com/dessalines/torrents.csv for more help"
|
help="Run ./scan_torrents.sh [TORRENTS_DIR] \nor goto https://gitlab.com/dessalines/torrents.csv for more help"
|
||||||
if [ "$1" == "-h" ] || [ -z "$1" ]; then
|
if [ "$1" == "-h" ] || [ -z "$1" ]; then
|
||||||
|
|
@ -27,60 +28,70 @@ fi
|
||||||
pushd $torrents_dir
|
pushd $torrents_dir
|
||||||
for torrent_file in *.torrent; do
|
for torrent_file in *.torrent; do
|
||||||
|
|
||||||
# Scrape it
|
file_infohash=$(basename $torrent_file | cut -d'.' -f 1)
|
||||||
health_text=$(torrent-tracker-health --torrent $torrent_file --timeout 1000 --addTrackers={udp://tracker.coppersurfer.tk:6969/announce,udp://tracker.internetwarriors.net:1337/announce,udp://tracker.opentrackr.org:1337/announce,udp://exodus.desync.com:6969/announce,udp://explodie.org:6969/announce})
|
|
||||||
|
|
||||||
# echo $health_text
|
if rg -Nq $file_infohash $scanned_out; then
|
||||||
|
echo "$file_infohash already scanned"
|
||||||
|
else
|
||||||
|
|
||||||
infohash=$(jq -r '.hash' <<< $health_text)
|
# Scrape it
|
||||||
name=$(jq -r '.name' <<< $health_text)
|
health_text=$(torrent-tracker-health --torrent $torrent_file --timeout 1000 --addTrackers={udp://tracker.coppersurfer.tk:6969/announce,udp://tracker.internetwarriors.net:1337/announce,udp://tracker.opentrackr.org:1337/announce,udp://exodus.desync.com:6969/announce,udp://explodie.org:6969/announce})
|
||||||
size_bytes=$(jq -r '.length' <<< $health_text)
|
|
||||||
seeders=$(jq -r '.seeds' <<< $health_text)
|
|
||||||
leechers=$(jq -r '.peers' <<< $health_text)
|
|
||||||
completed=$(jq -r '.completed' <<< $health_text)
|
|
||||||
date_string=$(jq -r '.created' <<< $health_text)
|
|
||||||
created_date=$(date -d "${date_string}" +"%s")
|
|
||||||
scraped_date=$(date +%s)
|
|
||||||
|
|
||||||
# Construct add line
|
echo -e "$health_text"
|
||||||
add_line="$infohash;$name;$size_bytes;$created_date;$seeders;$leechers;$completed;$scraped_date"
|
|
||||||
# echo -e $add_line
|
|
||||||
|
|
||||||
if (( $seeders > 0 )); then
|
|
||||||
|
|
||||||
found_line=$(rg -n $infohash $torrents_csv)
|
infohash=$(jq -r '.hash' <<< $health_text)
|
||||||
found_seeders=$(echo -e $found_line | cut -d';' -f 5)
|
name=$(jq -r '.name' <<< $health_text)
|
||||||
|
size_bytes=$(jq -r '.length' <<< $health_text)
|
||||||
|
seeders=$(jq -r '.seeds' <<< $health_text)
|
||||||
|
leechers=$(jq -r '.peers' <<< $health_text)
|
||||||
|
completed=$(jq -r '.completed' <<< $health_text)
|
||||||
|
date_string=$(jq -r '.created' <<< $health_text)
|
||||||
|
created_date=$(date -d "${date_string}" +"%s")
|
||||||
|
scraped_date=$(date +%s)
|
||||||
|
|
||||||
# Only re-add if the infohash doesn't exist, or
|
# Construct add line
|
||||||
if [ ! -z "$found_line" ]; then
|
add_line="$infohash;$name;$size_bytes;$created_date;$seeders;$leechers;$completed;$scraped_date"
|
||||||
|
# echo -e $add_line
|
||||||
|
|
||||||
# Seeder counts are different
|
if (( $seeders > 0 )); then
|
||||||
if [ "$found_seeders" != "$seeders" ]; then
|
|
||||||
|
|
||||||
# Delete the original infohash line
|
found_line=$(rg -n $infohash $torrents_csv)
|
||||||
grep -v "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv
|
found_seeders=$(echo -e $found_line | cut -d';' -f 5)
|
||||||
|
|
||||||
|
# Only re-add if the infohash doesn't exist, or
|
||||||
|
if [ ! -z "$found_line" ]; then
|
||||||
|
|
||||||
|
# Seeder counts are different
|
||||||
|
if [ "$found_seeders" != "$seeders" ]; then
|
||||||
|
|
||||||
|
# Delete the original infohash line
|
||||||
|
rg -N -v "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv
|
||||||
|
|
||||||
|
# Append the add lines to the torrents.csv file
|
||||||
|
echo -e "\n$add_line" >> $torrents_csv
|
||||||
|
# truncate -s -1 $torrents_csv # Removing last newline
|
||||||
|
echo -e "Updating Seeders: $torrent_file | $name | $infohash | $seeders"
|
||||||
|
else
|
||||||
|
echo -e "Not adding $name, had identical seeders"
|
||||||
|
fi
|
||||||
|
else
|
||||||
# Append the add lines to the torrents.csv file
|
# Append the add lines to the torrents.csv file
|
||||||
echo -e "\n$add_line" >> $torrents_csv
|
echo -e "\n$add_line" >> $torrents_csv
|
||||||
# truncate -s -1 $torrents_csv # Removing last newline
|
# truncate -s -1 $torrents_csv # Removing last newline
|
||||||
echo -e "Updating Seeders: $torrent_file | $name | $infohash | $seeders"
|
echo -e "New Torrent: $torrent_file | $name | $infohash | $seeders"
|
||||||
else
|
|
||||||
echo -e "Not adding $name, had identical seeders"
|
|
||||||
fi
|
fi
|
||||||
else
|
|
||||||
# Append the add lines to the torrents.csv file
|
|
||||||
echo -e "\n$add_line" >> $torrents_csv
|
|
||||||
# truncate -s -1 $torrents_csv # Removing last newline
|
|
||||||
echo -e "New Torrent: $torrent_file | $name | $infohash | $seeders"
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
|
||||||
else
|
else
|
||||||
# Deleting the line if it existed
|
# Deleting the line if it existed
|
||||||
if [ ! -z "$infohash" ]; then
|
if [ ! -z "$infohash" ]; then
|
||||||
echo -e "$name has no seeders, removing if existed."
|
# This removes lines that have no seeders
|
||||||
grep -v "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv
|
echo -e "$name has no seeders, removing if existed."
|
||||||
|
grep -v "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
echo $infohash >> $scanned_out
|
||||||
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
done
|
done
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue