From 2b7275836565286338ec45bc91b8963ef1a363b7 Mon Sep 17 00:00:00 2001
From: Dessalines <tyhou13@gmx.com>
Date: Sat, 26 Jan 2019 00:25:07 -0800
Subject: [PATCH] Updates to readme, scanning.

---
 .gitignore               |  1 +
 README.md                |  2 ++
 scripts/scan_torrents.sh | 24 ++++++++++++++++++------
 scripts/update.sh        |  7 +++----
 4 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/.gitignore b/.gitignore
index 2277566..311c8f6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 run.out
 old_greps.sh
 torrents.db
+.vscode
diff --git a/README.md b/README.md
index 1e53332..3e92142 100644
--- a/README.md
+++ b/README.md
@@ -68,6 +68,8 @@ An *upload*, consists of making a pull request after running the `scan_torrents.
 - [Torrent-Tracker-Health Dessalines branch](https://github.com/dessalines/torrent-tracker-health)
   - `npm i -g dessalines/torrent-tracker-health`
 - [jq command line JSON parser](https://stedolan.github.io/jq/)
+- [Cloudflare Scrape](https://github.com/Anorov/cloudflare-scrape)
+  - `sudo pip install cfscrape`
 
 ### Running
 
diff --git a/scripts/scan_torrents.sh b/scripts/scan_torrents.sh
index e491a16..2eaef8a 100755
--- a/scripts/scan_torrents.sh
+++ b/scripts/scan_torrents.sh
@@ -40,6 +40,9 @@ find `pwd` -name "*.torrent" | sort -n | grep -vFf $scanned_out | while read tor
   echo $(basename "$torrent_file" .torrent) >> $names_out
 done
 
+# Delete null torrents from the temp dir
+find $tmp_torrent_dir -name "*.torrent" -size -2k -delete
+
 if [ -z "$(ls -A $tmp_torrent_dir)" ]; then
   echo "No new torrents."
 else
@@ -48,14 +51,23 @@ else
 
   echo -e "$health_out"
 
-  results=$(jq -r '.results | map([.hash, .name, .length, (.created | strptime("%Y-%m-%dT%H:%M:%S.000Z") | mktime), .seeders, .leechers, .completed, (now | floor)] | join(";")) | join("\n")' $health_out)
+  # Convert the json results to csv format
+  results=$(jq -r '.results | map([.hash, .name, .length, (.created | .[0:16] | strptime("%Y-%m-%dT%H:%M") | mktime), .seeders, .leechers, .completed, (now | floor)] | join(";")) | join("\n")' $health_out)
+# // "1970-01-01T00:00:00.000Z"
+  # If there are no results
+  if [ -z "$results" ]; then
+    echo "There were no results for some reason."
+  else
+    # Update the torrents.csv and infohashes scanned file
+    echo -e "$results" >> $torrents_csv
+    cat "$names_out" >> $scanned_out
+
+    popd
+    cd scripts
+    . prune.sh
+  fi
 
-  echo -e "$results" >> $torrents_csv
-  cat "$names_out" >> $scanned_out
 
-  popd
-  cd scripts
-  . prune.sh
 fi
 
 # Remove the temp dir
diff --git a/scripts/update.sh b/scripts/update.sh
index 2318c89..289f8e3 100755
--- a/scripts/update.sh
+++ b/scripts/update.sh
@@ -1,7 +1,6 @@
 # This fetches from several torrent websites for new updates
+mkdir ../torrents
 cd ../new_torrents_fetcher
-cargo run
-cat tmp_adds.csv >> ../torrents.csv
-#rm tmp_adds.csv
+cargo run --release -- -s ../torrents -t ../torrents.csv
 cd ../scripts
-. prune.sh
+. scan_torrents.sh torrents
\ No newline at end of file