Updates to readme, scanning.

.gitignore

@@ -1,3 +1,4 @@
 run.out
 old_greps.sh
 torrents.db
+.vscode

README.md

@@ -68,6 +68,8 @@ An *upload*, consists of making a pull request after running the `scan_torrents.
 - [Torrent-Tracker-Health Dessalines branch](https://github.com/dessalines/torrent-tracker-health)
   - `npm i -g dessalines/torrent-tracker-health`
 - [jq command line JSON parser](https://stedolan.github.io/jq/)
+- [Cloudflare Scrape](https://github.com/Anorov/cloudflare-scrape)
+  - `sudo pip install cfscrape`
 
 ### Running
 

scripts/scan_torrents.sh

@@ -40,6 +40,9 @@ find `pwd` -name "*.torrent" | sort -n | grep -vFf $scanned_out | while read tor
   echo $(basename "$torrent_file" .torrent) >> $names_out
 done
 
+# Delete null torrents from the temp dir
+find $tmp_torrent_dir -name "*.torrent" -size -2k -delete
+
 if [ -z "$(ls -A $tmp_torrent_dir)" ]; then
   echo "No new torrents."
 else
@@ -48,14 +51,23 @@ else
 
   echo -e "$health_out"
 
-  results=$(jq -r '.results | map([.hash, .name, .length, (.created | strptime("%Y-%m-%dT%H:%M:%S.000Z") | mktime), .seeders, .leechers, .completed, (now | floor)] | join(";")) | join("\n")' $health_out)
+  # Convert the json results to csv format
+  results=$(jq -r '.results | map([.hash, .name, .length, (.created | .[0:16] | strptime("%Y-%m-%dT%H:%M") | mktime), .seeders, .leechers, .completed, (now | floor)] | join(";")) | join("\n")' $health_out)
+# // "1970-01-01T00:00:00.000Z"
+  # If there are no results
+  if [ -z "$results" ]; then
+    echo "There were no results for some reason."
+  else
+    # Update the torrents.csv and infohashes scanned file
+    echo -e "$results" >> $torrents_csv
+    cat "$names_out" >> $scanned_out
+
+    popd
+    cd scripts
+    . prune.sh
+  fi
 
-  echo -e "$results" >> $torrents_csv
-  cat "$names_out" >> $scanned_out
 
-  popd
-  cd scripts
-  . prune.sh
 fi
 
 # Remove the temp dir

scripts/update.sh

@@ -1,7 +1,6 @@
 # This fetches from several torrent websites for new updates
+mkdir ../torrents
 cd ../new_torrents_fetcher
-cargo run
+cargo run --release -- -s ../torrents -t ../torrents.csv
-cat tmp_adds.csv >> ../torrents.csv
-#rm tmp_adds.csv
 cd ../scripts
-. prune.sh
+. scan_torrents.sh torrents