diff --git a/new_torrents_fetcher/Cargo.lock b/new_torrents_fetcher/Cargo.lock index ad04884..41c33b1 100644 --- a/new_torrents_fetcher/Cargo.lock +++ b/new_torrents_fetcher/Cargo.lock @@ -1,3 +1,5 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. [[package]] name = "adler32" version = "1.0.3" diff --git a/new_torrents_fetcher/src/cf.py b/new_torrents_fetcher/src/cf.py index ae55443..3db2e37 100644 --- a/new_torrents_fetcher/src/cf.py +++ b/new_torrents_fetcher/src/cf.py @@ -1,7 +1,7 @@ -import cloudscraper +import cfscrape request = "GET / HTTP/1.1\r\n" -cookie_value, user_agent = cloudscraper.get_cookie_string("https://itorrents.org/torrent/B415C913643E5FF49FE37D304BBB5E6E11AD5101.torrent") +cookie_value, user_agent = cfscrape.get_cookie_string("https://itorrents.org/torrent/B415C913643E5FF49FE37D304BBB5E6E11AD5101.torrent") request += "Cookie: %s\r\nUser-Agent: %s\r\n" % (cookie_value, user_agent) # cookie = "Cookie: %s" % (cookie_value) diff --git a/new_torrents_fetcher/src/main.rs b/new_torrents_fetcher/src/main.rs index ffca4f0..277c1bc 100644 --- a/new_torrents_fetcher/src/main.rs +++ b/new_torrents_fetcher/src/main.rs @@ -10,6 +10,8 @@ use std::fs; use std::path::Path; use std::process::Command; use std::{thread, time}; +use std::io::{BufRead, BufReader}; +use std::fs::File; static mut COOKIE: &str = ""; static mut USER_AGENT: &str = ""; @@ -21,38 +23,67 @@ fn main() { .about("Fetches new torrent files from various sites.") .arg( Arg::with_name("TORRENT_SAVE_DIR") - .short("s") - .long("save_dir") - .value_name("DIR") - .takes_value(true) - .help("Where to save the torrent files.") - .required(true), - ) + .short("s") + .long("save_dir") + .value_name("DIR") + .takes_value(true) + .help("Where to save the torrent files.") + .required(true), + ) + .arg( + Arg::with_name("FETCH_SITES") + .short("fs") + .long("fetch_sites") + .help("Fetches from various torrent sites.") + ) + .arg( + Arg::with_name("HASH_FILE") + .short("hf") + .long("hash_file") + .value_name("FILE") + .takes_value(true) + .help("The location of a file containing rows of infohashes. If given, it will download those infohashes."), + ) .arg( Arg::with_name("TORRENTS_CSV_FILE") - .short("t") - .long("torrents_csv") - .value_name("FILE") - .takes_value(true) - .help("The location of a torrents.csv file. If given, it will download those infohashes."), - ) + .short("t") + .long("torrents_csv") + .value_name("FILE") + .takes_value(true) + .help("The location of a torrents.csv file. If given, it will download those infohashes."), + ) .get_matches(); let save_dir = Path::new(matches.value_of("TORRENT_SAVE_DIR").unwrap()); fetch_cloudflare_cookie(); - // torrentz2(save_dir); - thepiratebay(save_dir); - magnetdl(save_dir); - leetx(save_dir); - skytorrents(save_dir); + if matches.is_present("FETCH_SITES") { + // torrentz2(save_dir); + thepiratebay(save_dir); + magnetdl(save_dir); + leetx(save_dir); + skytorrents(save_dir); + } + + if let Some(t) = matches.value_of("HASH_FILE") { + hash_file_scan(Path::new(t), save_dir); + } if let Some(t) = matches.value_of("TORRENTS_CSV_FILE") { torrents_csv_scan(Path::new(t), save_dir); } } +fn hash_file_scan(hash_file: &Path, save_dir: &Path) { + let f = File::open(&hash_file).expect("Unable to open file"); + let f = BufReader::new(f); + for line in f.lines() { + let hash = line.expect("Unable to read line"); + fetch_torrent(hash, save_dir); + } +} + fn torrents_csv_scan(torrents_csv_file: &Path, save_dir: &Path) { for hash in collect_info_hashes(torrents_csv_file) { fetch_torrent(hash, save_dir); @@ -285,9 +316,9 @@ fn leetx(save_dir: &Path) { for row in document.find( Class("table-list") - .descendant(Name("tbody")) - .descendant(Name("tr")), - ) { + .descendant(Name("tbody")) + .descendant(Name("tr")), + ) { let detail_page_url_col = match row.find(Class("coll-1")).nth(0) { Some(t) => t, None => continue, @@ -312,10 +343,10 @@ fn leetx(save_dir: &Path) { let hash = match detail_document .find(Class("infohash-box").descendant(Name("span"))) .nth(0) - { - Some(t) => t.text().to_lowercase(), - None => continue, - }; + { + Some(t) => t.text().to_lowercase(), + None => continue, + }; fetch_torrent(hash, save_dir); } @@ -328,7 +359,7 @@ fn fetch_torrent(hash: String, save_dir: &Path) { let url = format!( "https://itorrents.org/torrent/{}.torrent", &hash.to_ascii_uppercase() - ); + ); let full_path = save_dir .join(&file_name) @@ -340,15 +371,15 @@ fn fetch_torrent(hash: String, save_dir: &Path) { unsafe { Command::new("curl") .args(&[ - &url, - "-H", - USER_AGENT, - "-H", - COOKIE, - "--compressed", - "-o", - &full_path, - "-s", + &url, + "-H", + USER_AGENT, + "-H", + COOKIE, + "--compressed", + "-o", + &full_path, + "-s", ]) .output() .expect("curl command failed"); diff --git a/scripts/update.sh b/scripts/update.sh index b2a687b..300d606 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -1,7 +1,7 @@ # This fetches from several torrent websites for new updates cd ../new_torrents_fetcher sort -r --field-separator=';' -n --key=5 ../torrents.csv > ../torrents.csv.seeders.desc -cargo run --release -- -s "$1" -t ../torrents.csv.seeders.desc +cargo run --release -- -s "$1" -f -t ../torrents.csv.seeders.desc rm ../torrents.csv.seeders.desc cd ../scripts . scan_torrents.sh "$1"