Changing way of scanning to use torrent-tracker-health

README.md

@@ -49,15 +49,15 @@ bleh season 1 (1993-)
 
 If running on a different host, open up `server/ui/src/env.ts` and change your hostname.
 
-## Uploading
+## Uploading / Adding Torrents
 
-An *upload*, consists of making a pull request after running the `add_torrents.sh` script, which adds torrents from a directory you choose to the `.csv` file, after checking that they aren't already there, and that they have seeders.
+An *upload*, consists of making a pull request after running the `scan_torrents.sh` script, which adds torrents from a directory you choose to the `.csv` file, after checking that they aren't already there, and that they have seeders.
 
 ### Requirements
 
-- [Torrent tracker scraper](https://github.com/ZigmundVonZaun/torrent-tracker-scraper)
+- [Torrent-Tracker-Health Dessalines branch](https://github.com/dessalines/torrent-tracker-health)
-- [Transmission-cli](https://transmissionbt.com/)
+  - `npm i -g dessalines/torrent-tracker-health`
-- [Human Friendly](https://humanfriendly.readthedocs.io/en/latest/readme.html#command-line)
+- [jq command line JSON parser](https://stedolan.github.io/jq/)
 
 ### Running
 
@@ -65,7 +65,7 @@ An *upload*, consists of making a pull request after running the `add_torrents.s
 ```sh
 git clone https://gitlab.com/[MY_USER]/torrents.csv
 cd torrents.csv/scripts
-./add_torrents.sh MY_TORRENTS_DIR # `MY_TORRENTS_DIR` is `~/.local/share/data/qBittorrent/BT_backup/` for qBittorrent on linux, but you can search for where your torrents are stored for your client.
+./scan_torrents.sh MY_TORRENTS_DIR # `MY_TORRENTS_DIR` is `~/.local/share/data/qBittorrent/BT_backup/` for qBittorrent on linux, but you can search for where your torrents are stored for your client.
 git commit -am "Adding my torrents"
 git push
 ```

scripts/add_torrents.sh

@@ -1,104 +0,0 @@
-#!/bin/bash
-
-# Checking arguments
-# Help line
-cd ..
-torrents_csv="`pwd`/torrents.csv"
-
-help="Run ./add_torrents.sh [TORRENTS_DIR] \nor goto https://gitlab.com/dessalines/torrents.csv for more help"
-if [ "$1" == "-h" ] || [ -z "$1" ]; then
-  echo -e $help
-  exit 1
-fi
-
-torrents_dir="$1"
-echo "Torrents dir=$torrents_dir"
-
-# Check dependencies
-
-if command -v transmission-cli >/dev/null 2>&1 ; then
-  echo "transmission-cli version: $(transmission-cli --version)"
-else
-  echo "Install transmission-cli"
-  exit 0
-fi
-
-if command -v "python -m torrent_tracker_scraper.scraper" >/dev/null 2>&1 ; then
-  echo "torrent-tracker-scraper installed."
-else
-  echo -e "Installing torrent_tracker_scraper:\npip install torrent-tracker-scraper --user \nhttps://github.com/ZigmundVonZaun/torrent-tracker-scraper"
-  pip install torrent-tracker-scraper --user
-fi
-
-if command -v "~/.local/bin/humanfriendly -c" >/dev/null 2>&1 ; then
-  echo "humanfriendly installed."
-else
-  echo -e "Installing humanfriendly:\npip install humanfriendly --user"
-  pip install humanfriendly --user
-fi
-
-# Loop over all torrents
-pushd $torrents_dir
-for torrent_file in *.torrent; do
-
-  # Get fields from transmission
-  
-  show_text=$(transmission-show "$torrent_file")
-
-  infohash=$(grep -Po 'Hash: \K.*' <<< $show_text)
-
-    # If the infohash already exists, don't do anything
-    found_line=$(rg -n $infohash $torrents_csv | cut -d : -f 1) 
-    if [ ! -z $found_line ]; then
-      echo "$infohash already exists"
-    else
-
-      name=$(grep -Po -m 1 'Name: \K.*' <<< $show_text)
-      name=$(sed 's/;/\\;/g' <<< $name) # Escape the commas for .csv
-
-      # Size: Unfortunately this will chop off some sigfigs
-      size=$(grep -Po 'Total Size: \K.*' <<< $show_text) 
-      size_bytes=$(~/.local/bin/humanfriendly --parse-size="$size") # Convert to bytes
-
-
-
-      # Convert the created date
-      date_string=$(grep -Po 'Created on: \K.*' <<< $show_text)
-      created_date=""
-      if [[ "$date_string" == "Unknown" ]]; then
-        created_date=$(date +%s)
-      else
-        created_date=$(date -d "${date_string}" +"%s")
-      fi
-      
-      # Scrape for seeder counts
-      scrape_text=$(timeout 20 python -m torrent_tracker_scraper.scraper \
-        -i "$infohash" \
-        -t tracker.coppersurfer.tk -p 6969)
-        # -t tracker.internetwarriors.net -p 1337
-        # -t tracker.opentrackr.org -p 1337
-      seeders=$(grep -Po 'Seeds: \K[0-9]+' <<< $scrape_text)
-      leechers=$(grep -Po 'Leechers: \K[0-9]+' <<< $scrape_text)
-      completed=$(grep -Po 'Completed: \K[0-9]+' <<< $scrape_text)
-      scraped_date=$(date +%s)
-
-      # Construct add line
-      add_line="$infohash;$name;$size_bytes;$created_date;$seeders;$leechers;$completed;$scraped_date"
-
-      # Only add the line if there are seeds, and the infohash doesn't already exist
-      if (( $seeders > 0 )); then
-
-        # Append the add lines to the torrents.csv file
-        echo -e "\n$add_line" >> $torrents_csv
-        # truncate -s -1 $torrents_csv # Removing last newline
-        echo -e "Added $name"
-      else 
-        echo -e "$name has no seeders."
-      fi
-    fi
-done
-
-popd
-cd scripts
-. prune.sh
-

scripts/scan_torrents.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# Checking arguments
+# Help line
+cd ..
+torrents_csv="`pwd`/torrents.csv"
+
+help="Run ./scan_torrents.sh [TORRENTS_DIR] \nor goto https://gitlab.com/dessalines/torrents.csv for more help"
+if [ "$1" == "-h" ] || [ -z "$1" ]; then
+  echo -e $help
+  exit 1
+fi
+
+torrents_dir="$1"
+echo "Torrents dir=$torrents_dir"
+
+# Check dependencies
+
+if command -v "torrent-tracker-health" >/dev/null 2>&1 ; then
+  echo "torrent-tracker-health installed."
+else
+  echo -e "Installing torrent-tracker-health:\nnpm i -g dessalines/torrent-tracker-health \nhttps://github.com/dessalines/torrent-tracker-health\n"
+  npm i -g install dessalines/torrent-tracker-health
+fi
+
+# Loop over all torrents
+pushd $torrents_dir
+for torrent_file in *.torrent; do
+
+  # Scrape it
+  health_text=$(torrent-tracker-health --torrent $torrent_file --timeout 1000 --addTrackers={udp://tracker.coppersurfer.tk:6969/announce,udp://tracker.internetwarriors.net:1337/announce,udp://tracker.opentrackr.org:1337/announce,udp://exodus.desync.com:6969/announce,udp://explodie.org:6969/announce})
+
+  # echo $health_text
+
+  infohash=$(jq  -r '.hash' <<< $health_text)
+  name=$(jq -r '.name' <<< $health_text)
+  size_bytes=$(jq -r '.length' <<< $health_text)
+  seeders=$(jq -r '.seeds' <<< $health_text)
+  leechers=$(jq -r '.peers' <<< $health_text)
+  completed=$(jq -r '.completed' <<< $health_text)
+  date_string=$(jq -r '.created' <<< $health_text)
+  created_date=$(date -d "${date_string}" +"%s")
+  scraped_date=$(date +%s)
+
+  # Construct add line
+  add_line="$infohash;$name;$size_bytes;$created_date;$seeders;$leechers;$completed;$scraped_date"
+  # echo -e $add_line
+  
+  if (( $seeders > 0 )); then
+
+    found_line=$(rg -n $infohash $torrents_csv)
+    found_seeders=$(echo -e $found_line | cut -d';' -f 5)
+
+    # Only re-add if the infohash doesn't exist, and the seeder counts are different
+    if [ ! -z "$found_line" ] && [ "$found_seeders" != "$seeders" ]; then
+
+      # Delete the original infohash line
+      grep -v "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv
+     
+      # Append the add lines to the torrents.csv file
+      echo -e "\n$add_line" >> $torrents_csv
+      # truncate -s -1 $torrents_csv # Removing last newline
+      echo -e "Added $torrent_file | $name | $infohash | $seeders"
+    else
+      echo -e "Not adding $name, had identical seeders"
+    fi
+
+  else 
+    echo -e "$name has no seeders, removing if existed."
+    # Deleting the line if it existed
+    grep -v "$infohash" $torrents_csv > torfile.tmp.2; mv torfile.tmp.2 $torrents_csv
+  fi
+
+done
+
+popd
+cd scripts
+. prune.sh
+