Changing way of scanning to use torrent-tracker-health
This commit is contained in:
Dessalines
parent
f8cd0db2f4
commit
7bb576805c
12
README.md
12
README.md
|
|
@ -49,15 +49,15 @@ bleh season 1 (1993-)
|
|||
|
||||
If running on a different host, open up `server/ui/src/env.ts` and change your hostname.
|
||||
|
||||
## Uploading
|
||||
## Uploading / Adding Torrents
|
||||
|
||||
An *upload*, consists of making a pull request after running the `add_torrents.sh` script, which adds torrents from a directory you choose to the `.csv` file, after checking that they aren't already there, and that they have seeders.
|
||||
An *upload*, consists of making a pull request after running the `scan_torrents.sh` script, which adds torrents from a directory you choose to the `.csv` file, after checking that they aren't already there, and that they have seeders.
|
||||
|
||||
### Requirements
|
||||
|
||||
- [Torrent tracker scraper](https://github.com/ZigmundVonZaun/torrent-tracker-scraper)
|
||||
- [Transmission-cli](https://transmissionbt.com/)
|
||||
- [Human Friendly](https://humanfriendly.readthedocs.io/en/latest/readme.html#command-line)
|
||||
- [Torrent-Tracker-Health Dessalines branch](https://github.com/dessalines/torrent-tracker-health)
|
||||
- `npm i -g dessalines/torrent-tracker-health`
|
||||
- [jq command line JSON parser](https://stedolan.github.io/jq/)
|
||||
|
||||
### Running
|
||||
|
||||
|
|
@ -65,7 +65,7 @@ An *upload*, consists of making a pull request after running the `add_torrents.s
|
|||
```sh
|
||||
git clone https://gitlab.com/[MY_USER]/torrents.csv
|
||||
cd torrents.csv/scripts
|
||||
./add_torrents.sh MY_TORRENTS_DIR # `MY_TORRENTS_DIR` is `~/.local/share/data/qBittorrent/BT_backup/` for qBittorrent on linux, but you can search for where your torrents are stored for your client.
|
||||
./scan_torrents.sh MY_TORRENTS_DIR # `MY_TORRENTS_DIR` is `~/.local/share/data/qBittorrent/BT_backup/` for qBittorrent on linux, but you can search for where your torrents are stored for your client.
|
||||
git commit -am "Adding my torrents"
|
||||
git push
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,104 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Checking arguments
|
||||
# Help line
|
||||
cd ..
|
||||
torrents_csv="`pwd`/torrents.csv"
|
||||
|
||||
help="Run ./add_torrents.sh [TORRENTS_DIR] \nor goto https://gitlab.com/dessalines/torrents.csv for more help"
|
||||
if [ "$1" == "-h" ] || [ -z "$1" ]; then
|
||||
echo -e $help
|
||||
exit 1
|
||||
fi
|
||||
|
||||
torrents_dir="$1"
|
||||
echo "Torrents dir=$torrents_dir"
|
||||
|
||||
# Check dependencies
|
||||
|
||||
if command -v transmission-cli >/dev/null 2>&1 ; then
|
||||
echo "transmission-cli version: $(transmission-cli --version)"
|
||||
else
|
||||
echo "Install transmission-cli"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if command -v "python -m torrent_tracker_scraper.scraper" >/dev/null 2>&1 ; then
|
||||
echo "torrent-tracker-scraper installed."
|
||||
else
|
||||
echo -e "Installing torrent_tracker_scraper:\npip install torrent-tracker-scraper --user \nhttps://github.com/ZigmundVonZaun/torrent-tracker-scraper"
|
||||
pip install torrent-tracker-scraper --user
|
||||
fi
|
||||
|
||||
if command -v "~/.local/bin/humanfriendly -c" >/dev/null 2>&1 ; then
|
||||
echo "humanfriendly installed."
|
||||
else
|
||||
echo -e "Installing humanfriendly:\npip install humanfriendly --user"
|
||||
pip install humanfriendly --user
|
||||
fi
|
||||
|
||||
# Loop over all torrents
|
||||
pushd $torrents_dir
|
||||
for torrent_file in *.torrent; do
|
||||
|
||||
# Get fields from transmission
|
||||
|
||||
show_text=$(transmission-show "$torrent_file")
|
||||
|
||||
infohash=$(grep -Po 'Hash: \K.*' <<< $show_text)
|
||||
|
||||
# If the infohash already exists, don't do anything
|
||||
found_line=$(rg -n $infohash $torrents_csv | cut -d : -f 1)
|
||||
if [ ! -z $found_line ]; then
|
||||
echo "$infohash already exists"
|
||||
else
|
||||
|
||||
name=$(grep -Po -m 1 'Name: \K.*' <<< $show_text)
|
||||
name=$(sed 's/;/\\;/g' <<< $name) # Escape the commas for .csv
|
||||
|
||||
# Size: Unfortunately this will chop off some sigfigs
|
||||
size=$(grep -Po 'Total Size: \K.*' <<< $show_text)
|
||||
size_bytes=$(~/.local/bin/humanfriendly --parse-size="$size") # Convert to bytes
|
||||
|
||||
|
||||
|
||||
# Convert the created date
|
||||
date_string=$(grep -Po 'Created on: \K.*' <<< $show_text)
|
||||
created_date=""
|
||||
if [[ "$date_string" == "Unknown" ]]; then
|
||||
created_date=$(date +%s)
|
||||
else
|
||||
created_date=$(date -d "${date_string}" +"%s")
|
||||
fi
|
||||
|
||||
# Scrape for seeder counts
|
||||
scrape_text=$(timeout 20 python -m torrent_tracker_scraper.scraper \
|
||||
-i "$infohash" \
|
||||
-t tracker.coppersurfer.tk -p 6969)
|
||||
# -t tracker.internetwarriors.net -p 1337
|
||||
# -t tracker.opentrackr.org -p 1337
|
||||
seeders=$(grep -Po 'Seeds: \K[0-9]+' <<< $scrape_text)
|
||||
leechers=$(grep -Po 'Leechers: \K[0-9]+' <<< $scrape_text)
|
||||
completed=$(grep -Po 'Completed: \K[0-9]+' <<< $scrape_text)
|
||||
scraped_date=$(date +%s)
|
||||
|
||||
# Construct add line
|
||||
add_line="$infohash;$name;$size_bytes;$created_date;$seeders;$leechers;$completed;$scraped_date"
|
||||
|
||||
# Only add the line if there are seeds, and the infohash doesn't already exist
|
||||
if (( $seeders > 0 )); then
|
||||
|
||||
# Append the add lines to the torrents.csv file
|
||||
echo -e "\n$add_line" >> $torrents_csv
|
||||
# truncate -s -1 $torrents_csv # Removing last newline
|
||||
echo -e "Added $name"
|
||||
else
|
||||
echo -e "$name has no seeders."
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
popd
|
||||
cd scripts
|
||||
. prune.sh
|
||||
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Checking arguments
|
||||
# Help line
|
||||
cd ..
|
||||
torrents_csv="`pwd`/torrents.csv"
|
||||
|
||||
help="Run ./scan_torrents.sh [TORRENTS_DIR] \nor goto https://gitlab.com/dessalines/torrents.csv for more help"
|
||||
if [ "$1" == "-h" ] || [ -z "$1" ]; then
|
||||
echo -e $help
|
||||
exit 1
|
||||
fi
|
||||
|
||||
torrents_dir="$1"
|
||||
echo "Torrents dir=$torrents_dir"
|
||||
|
||||
# Check dependencies
|
||||
|
||||
if command -v "torrent-tracker-health" >/dev/null 2>&1 ; then
|
||||
echo "torrent-tracker-health installed."
|
||||
else
|
||||
echo -e "Installing torrent-tracker-health:\nnpm i -g dessalines/torrent-tracker-health \nhttps://github.com/dessalines/torrent-tracker-health\n"
|
||||
npm i -g install dessalines/torrent-tracker-health
|
||||
fi
|
||||
|
||||
# Loop over all torrents
|
||||
pushd $torrents_dir
|
||||
for torrent_file in *.torrent; do
|
||||
|
||||
# Scrape it
|
||||
health_text=$(torrent-tracker-health --torrent $torrent_file --timeout 1000 --addTrackers={udp://tracker.coppersurfer.tk:6969/announce,udp://tracker.internetwarriors.net:1337/announce,udp://tracker.opentrackr.org:1337/announce,udp://exodus.desync.com:6969/announce,udp://explodie.org:6969/announce})
|
||||
|
||||
# echo $health_text
|
||||
|
||||
infohash=$(jq -r '.hash' <<< $health_text)
|
||||
name=$(jq -r '.name' <<< $health_text)
|
||||
size_bytes=$(jq -r '.length' <<< $health_text)
|
||||
seeders=$(jq -r '.seeds' <<< $health_text)
|
||||
leechers=$(jq -r '.peers' <<< $health_text)
|
||||
completed=$(jq -r '.completed' <<< $health_text)
|
||||
date_string=$(jq -r '.created' <<< $health_text)
|
||||
created_date=$(date -d "${date_string}" +"%s")
|
||||
scraped_date=$(date +%s)
|
||||
|
||||
# Construct add line
|
||||
add_line="$infohash;$name;$size_bytes;$created_date;$seeders;$leechers;$completed;$scraped_date"
|
||||
# echo -e $add_line
|
||||
|
||||
if (( $seeders > 0 )); then
|
||||
|
||||
found_line=$(rg -n $infohash $torrents_csv)
|
||||
found_seeders=$(echo -e $found_line | cut -d';' -f 5)
|
||||
|
||||
# Only re-add if the infohash doesn't exist, and the seeder counts are different
|
||||
if [ ! -z "$found_line" ] && [ "$found_seeders" != "$seeders" ]; then
|
||||
|
||||
# Delete the original infohash line
|
||||
grep -v "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv
|
||||
|
||||
# Append the add lines to the torrents.csv file
|
||||
echo -e "\n$add_line" >> $torrents_csv
|
||||
# truncate -s -1 $torrents_csv # Removing last newline
|
||||
echo -e "Added $torrent_file | $name | $infohash | $seeders"
|
||||
else
|
||||
echo -e "Not adding $name, had identical seeders"
|
||||
fi
|
||||
|
||||
else
|
||||
echo -e "$name has no seeders, removing if existed."
|
||||
# Deleting the line if it existed
|
||||
grep -v "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
popd
|
||||
cd scripts
|
||||
. prune.sh
|
||||
|
||||
Loading…
Reference in New Issue