Adding a hash file fetcher.

new_torrents_fetcher/Cargo.lock

@@ -1,3 +1,5 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
 [[package]]
 name = "adler32"
 version = "1.0.3"

new_torrents_fetcher/src/cf.py

@@ -1,7 +1,7 @@
-import cloudscraper
+import cfscrape
 request = "GET / HTTP/1.1\r\n"
 
-cookie_value, user_agent = cloudscraper.get_cookie_string("https://itorrents.org/torrent/B415C913643E5FF49FE37D304BBB5E6E11AD5101.torrent")
+cookie_value, user_agent = cfscrape.get_cookie_string("https://itorrents.org/torrent/B415C913643E5FF49FE37D304BBB5E6E11AD5101.torrent")
 request += "Cookie: %s\r\nUser-Agent: %s\r\n" % (cookie_value, user_agent)
 # cookie = "Cookie: %s" % (cookie_value)
 

new_torrents_fetcher/src/main.rs

@@ -10,6 +10,8 @@ use std::fs;
 use std::path::Path;
 use std::process::Command;
 use std::{thread, time};
+use std::io::{BufRead, BufReader};
+use std::fs::File;
 
 static mut COOKIE: &str = "";
 static mut USER_AGENT: &str = "";
@@ -28,6 +30,20 @@ fn main() {
       .help("Where to save the torrent files.")
       .required(true),
       )
+    .arg(
+      Arg::with_name("FETCH_SITES")
+      .short("fs")
+      .long("fetch_sites")
+      .help("Fetches from various torrent sites.")
+      )
+    .arg(
+      Arg::with_name("HASH_FILE")
+      .short("hf")
+      .long("hash_file")
+      .value_name("FILE")
+      .takes_value(true)
+      .help("The location of a file containing rows of infohashes. If given, it will download those infohashes."),
+      )
     .arg(
       Arg::with_name("TORRENTS_CSV_FILE")
       .short("t")
@@ -42,17 +58,32 @@ fn main() {
 
   fetch_cloudflare_cookie();
 
+  if matches.is_present("FETCH_SITES") {
     // torrentz2(save_dir);
     thepiratebay(save_dir);
     magnetdl(save_dir);
     leetx(save_dir);
     skytorrents(save_dir);
+  }
+
+  if let Some(t) = matches.value_of("HASH_FILE") {
+    hash_file_scan(Path::new(t), save_dir);
+  }
 
   if let Some(t) = matches.value_of("TORRENTS_CSV_FILE") {
     torrents_csv_scan(Path::new(t), save_dir);
   }
 }
 
+fn hash_file_scan(hash_file: &Path, save_dir: &Path) {
+  let f = File::open(&hash_file).expect("Unable to open file");
+  let f = BufReader::new(f);
+  for line in f.lines() {
+    let hash = line.expect("Unable to read line");
+    fetch_torrent(hash, save_dir);
+  }
+}
+
 fn torrents_csv_scan(torrents_csv_file: &Path, save_dir: &Path) {
   for hash in collect_info_hashes(torrents_csv_file) {
     fetch_torrent(hash, save_dir);

scripts/update.sh

@@ -1,7 +1,7 @@
 # This fetches from several torrent websites for new updates
 cd ../new_torrents_fetcher
 sort -r --field-separator=';' -n --key=5 ../torrents.csv > ../torrents.csv.seeders.desc
-cargo run --release -- -s "$1" -t ../torrents.csv.seeders.desc
+cargo run --release -- -s "$1" -f -t ../torrents.csv.seeders.desc
 rm ../torrents.csv.seeders.desc
 cd ../scripts
 . scan_torrents.sh "$1"