Updates to readme, scanning.

.gitignore

@@ -1,3 +1,4 @@
 run.out
 old_greps.sh
 torrents.db
+.vscode

README.md

@@ -68,6 +68,8 @@ An *upload*, consists of making a pull request after running the `scan_torrents.
 - [Torrent-Tracker-Health Dessalines branch](https://github.com/dessalines/torrent-tracker-health)
   - `npm i -g dessalines/torrent-tracker-health`
 - [jq command line JSON parser](https://stedolan.github.io/jq/)
+- [Cloudflare Scrape](https://github.com/Anorov/cloudflare-scrape)
+  - `sudo pip install cfscrape`
 
 ### Running
 

scripts/scan_torrents.sh

@@ -40,6 +40,9 @@ find `pwd` -name "*.torrent" | sort -n | grep -vFf $scanned_out | while read tor
   echo $(basename "$torrent_file" .torrent) >> $names_out
 done
 
+# Delete null torrents from the temp dir
+find $tmp_torrent_dir -name "*.torrent" -size -2k -delete
+
 if [ -z "$(ls -A $tmp_torrent_dir)" ]; then
   echo "No new torrents."
 else
@@ -48,8 +51,14 @@ else
 
   echo -e "$health_out"
 
-  results=$(jq -r '.results | map([.hash, .name, .length, (.created | strptime("%Y-%m-%dT%H:%M:%S.000Z") | mktime), .seeders, .leechers, .completed, (now | floor)] | join(";")) | join("\n")' $health_out)
-
+  # Convert the json results to csv format
+  results=$(jq -r '.results | map([.hash, .name, .length, (.created | .[0:16] | strptime("%Y-%m-%dT%H:%M") | mktime), .seeders, .leechers, .completed, (now | floor)] | join(";")) | join("\n")' $health_out)
+# // "1970-01-01T00:00:00.000Z"
+  # If there are no results
+  if [ -z "$results" ]; then
+    echo "There were no results for some reason."
+  else
+    # Update the torrents.csv and infohashes scanned file
     echo -e "$results" >> $torrents_csv
     cat "$names_out" >> $scanned_out
 
@@ -58,6 +67,9 @@ else
     . prune.sh
   fi
 
+
+fi
+
 # Remove the temp dir
 rm -rf "$tmp_torrent_dir"
 rm "$names_out"

scripts/update.sh

@@ -1,7 +1,6 @@
 # This fetches from several torrent websites for new updates
+mkdir ../torrents
 cd ../new_torrents_fetcher
-cargo run
-cat tmp_adds.csv >> ../torrents.csv
-#rm tmp_adds.csv
+cargo run --release -- -s ../torrents -t ../torrents.csv
 cd ../scripts
-. prune.sh
+. scan_torrents.sh torrents