From 7bb576805cdb9ee387b8ce576d0fd843fa30233e Mon Sep 17 00:00:00 2001 From: Dessalines Date: Sun, 25 Nov 2018 16:21:52 -0700 Subject: [PATCH] Changing way of scanning to use torrent-tracker-health --- README.md | 12 ++--- scripts/add_torrents.sh | 104 --------------------------------------- scripts/scan_torrents.sh | 79 +++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 110 deletions(-) delete mode 100755 scripts/add_torrents.sh create mode 100755 scripts/scan_torrents.sh diff --git a/README.md b/README.md index b26cf01..7462a58 100644 --- a/README.md +++ b/README.md @@ -49,15 +49,15 @@ bleh season 1 (1993-) If running on a different host, open up `server/ui/src/env.ts` and change your hostname. -## Uploading +## Uploading / Adding Torrents -An *upload*, consists of making a pull request after running the `add_torrents.sh` script, which adds torrents from a directory you choose to the `.csv` file, after checking that they aren't already there, and that they have seeders. +An *upload*, consists of making a pull request after running the `scan_torrents.sh` script, which adds torrents from a directory you choose to the `.csv` file, after checking that they aren't already there, and that they have seeders. ### Requirements -- [Torrent tracker scraper](https://github.com/ZigmundVonZaun/torrent-tracker-scraper) -- [Transmission-cli](https://transmissionbt.com/) -- [Human Friendly](https://humanfriendly.readthedocs.io/en/latest/readme.html#command-line) +- [Torrent-Tracker-Health Dessalines branch](https://github.com/dessalines/torrent-tracker-health) + - `npm i -g dessalines/torrent-tracker-health` +- [jq command line JSON parser](https://stedolan.github.io/jq/) ### Running @@ -65,7 +65,7 @@ An *upload*, consists of making a pull request after running the `add_torrents.s ```sh git clone https://gitlab.com/[MY_USER]/torrents.csv cd torrents.csv/scripts -./add_torrents.sh MY_TORRENTS_DIR # `MY_TORRENTS_DIR` is `~/.local/share/data/qBittorrent/BT_backup/` for qBittorrent on linux, but you can search for where your torrents are stored for your client. +./scan_torrents.sh MY_TORRENTS_DIR # `MY_TORRENTS_DIR` is `~/.local/share/data/qBittorrent/BT_backup/` for qBittorrent on linux, but you can search for where your torrents are stored for your client. git commit -am "Adding my torrents" git push ``` diff --git a/scripts/add_torrents.sh b/scripts/add_torrents.sh deleted file mode 100755 index ea70511..0000000 --- a/scripts/add_torrents.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash - -# Checking arguments -# Help line -cd .. -torrents_csv="`pwd`/torrents.csv" - -help="Run ./add_torrents.sh [TORRENTS_DIR] \nor goto https://gitlab.com/dessalines/torrents.csv for more help" -if [ "$1" == "-h" ] || [ -z "$1" ]; then - echo -e $help - exit 1 -fi - -torrents_dir="$1" -echo "Torrents dir=$torrents_dir" - -# Check dependencies - -if command -v transmission-cli >/dev/null 2>&1 ; then - echo "transmission-cli version: $(transmission-cli --version)" -else - echo "Install transmission-cli" - exit 0 -fi - -if command -v "python -m torrent_tracker_scraper.scraper" >/dev/null 2>&1 ; then - echo "torrent-tracker-scraper installed." -else - echo -e "Installing torrent_tracker_scraper:\npip install torrent-tracker-scraper --user \nhttps://github.com/ZigmundVonZaun/torrent-tracker-scraper" - pip install torrent-tracker-scraper --user -fi - -if command -v "~/.local/bin/humanfriendly -c" >/dev/null 2>&1 ; then - echo "humanfriendly installed." -else - echo -e "Installing humanfriendly:\npip install humanfriendly --user" - pip install humanfriendly --user -fi - -# Loop over all torrents -pushd $torrents_dir -for torrent_file in *.torrent; do - - # Get fields from transmission - - show_text=$(transmission-show "$torrent_file") - - infohash=$(grep -Po 'Hash: \K.*' <<< $show_text) - - # If the infohash already exists, don't do anything - found_line=$(rg -n $infohash $torrents_csv | cut -d : -f 1) - if [ ! -z $found_line ]; then - echo "$infohash already exists" - else - - name=$(grep -Po -m 1 'Name: \K.*' <<< $show_text) - name=$(sed 's/;/\\;/g' <<< $name) # Escape the commas for .csv - - # Size: Unfortunately this will chop off some sigfigs - size=$(grep -Po 'Total Size: \K.*' <<< $show_text) - size_bytes=$(~/.local/bin/humanfriendly --parse-size="$size") # Convert to bytes - - - - # Convert the created date - date_string=$(grep -Po 'Created on: \K.*' <<< $show_text) - created_date="" - if [[ "$date_string" == "Unknown" ]]; then - created_date=$(date +%s) - else - created_date=$(date -d "${date_string}" +"%s") - fi - - # Scrape for seeder counts - scrape_text=$(timeout 20 python -m torrent_tracker_scraper.scraper \ - -i "$infohash" \ - -t tracker.coppersurfer.tk -p 6969) - # -t tracker.internetwarriors.net -p 1337 - # -t tracker.opentrackr.org -p 1337 - seeders=$(grep -Po 'Seeds: \K[0-9]+' <<< $scrape_text) - leechers=$(grep -Po 'Leechers: \K[0-9]+' <<< $scrape_text) - completed=$(grep -Po 'Completed: \K[0-9]+' <<< $scrape_text) - scraped_date=$(date +%s) - - # Construct add line - add_line="$infohash;$name;$size_bytes;$created_date;$seeders;$leechers;$completed;$scraped_date" - - # Only add the line if there are seeds, and the infohash doesn't already exist - if (( $seeders > 0 )); then - - # Append the add lines to the torrents.csv file - echo -e "\n$add_line" >> $torrents_csv - # truncate -s -1 $torrents_csv # Removing last newline - echo -e "Added $name" - else - echo -e "$name has no seeders." - fi - fi -done - -popd -cd scripts -. prune.sh - diff --git a/scripts/scan_torrents.sh b/scripts/scan_torrents.sh new file mode 100755 index 0000000..1dbff00 --- /dev/null +++ b/scripts/scan_torrents.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Checking arguments +# Help line +cd .. +torrents_csv="`pwd`/torrents.csv" + +help="Run ./scan_torrents.sh [TORRENTS_DIR] \nor goto https://gitlab.com/dessalines/torrents.csv for more help" +if [ "$1" == "-h" ] || [ -z "$1" ]; then + echo -e $help + exit 1 +fi + +torrents_dir="$1" +echo "Torrents dir=$torrents_dir" + +# Check dependencies + +if command -v "torrent-tracker-health" >/dev/null 2>&1 ; then + echo "torrent-tracker-health installed." +else + echo -e "Installing torrent-tracker-health:\nnpm i -g dessalines/torrent-tracker-health \nhttps://github.com/dessalines/torrent-tracker-health\n" + npm i -g install dessalines/torrent-tracker-health +fi + +# Loop over all torrents +pushd $torrents_dir +for torrent_file in *.torrent; do + + # Scrape it + health_text=$(torrent-tracker-health --torrent $torrent_file --timeout 1000 --addTrackers={udp://tracker.coppersurfer.tk:6969/announce,udp://tracker.internetwarriors.net:1337/announce,udp://tracker.opentrackr.org:1337/announce,udp://exodus.desync.com:6969/announce,udp://explodie.org:6969/announce}) + + # echo $health_text + + infohash=$(jq -r '.hash' <<< $health_text) + name=$(jq -r '.name' <<< $health_text) + size_bytes=$(jq -r '.length' <<< $health_text) + seeders=$(jq -r '.seeds' <<< $health_text) + leechers=$(jq -r '.peers' <<< $health_text) + completed=$(jq -r '.completed' <<< $health_text) + date_string=$(jq -r '.created' <<< $health_text) + created_date=$(date -d "${date_string}" +"%s") + scraped_date=$(date +%s) + + # Construct add line + add_line="$infohash;$name;$size_bytes;$created_date;$seeders;$leechers;$completed;$scraped_date" + # echo -e $add_line + + if (( $seeders > 0 )); then + + found_line=$(rg -n $infohash $torrents_csv) + found_seeders=$(echo -e $found_line | cut -d';' -f 5) + + # Only re-add if the infohash doesn't exist, and the seeder counts are different + if [ ! -z "$found_line" ] && [ "$found_seeders" != "$seeders" ]; then + + # Delete the original infohash line + grep -v "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv + + # Append the add lines to the torrents.csv file + echo -e "\n$add_line" >> $torrents_csv + # truncate -s -1 $torrents_csv # Removing last newline + echo -e "Added $torrent_file | $name | $infohash | $seeders" + else + echo -e "Not adding $name, had identical seeders" + fi + + else + echo -e "$name has no seeders, removing if existed." + # Deleting the line if it existed + grep -v "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv + fi + +done + +popd +cd scripts +. prune.sh +