Adding a hash file fetcher.

This commit is contained in:
Dessalines 2019-09-06 23:01:52 -07:00
parent 993e6d4f08
commit f77da92e5c
4 changed files with 71 additions and 38 deletions

View File

@ -1,3 +1,5 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]] [[package]]
name = "adler32" name = "adler32"
version = "1.0.3" version = "1.0.3"

View File

@ -1,7 +1,7 @@
import cloudscraper import cfscrape
request = "GET / HTTP/1.1\r\n" request = "GET / HTTP/1.1\r\n"
cookie_value, user_agent = cloudscraper.get_cookie_string("https://itorrents.org/torrent/B415C913643E5FF49FE37D304BBB5E6E11AD5101.torrent") cookie_value, user_agent = cfscrape.get_cookie_string("https://itorrents.org/torrent/B415C913643E5FF49FE37D304BBB5E6E11AD5101.torrent")
request += "Cookie: %s\r\nUser-Agent: %s\r\n" % (cookie_value, user_agent) request += "Cookie: %s\r\nUser-Agent: %s\r\n" % (cookie_value, user_agent)
# cookie = "Cookie: %s" % (cookie_value) # cookie = "Cookie: %s" % (cookie_value)

View File

@ -10,6 +10,8 @@ use std::fs;
use std::path::Path; use std::path::Path;
use std::process::Command; use std::process::Command;
use std::{thread, time}; use std::{thread, time};
use std::io::{BufRead, BufReader};
use std::fs::File;
static mut COOKIE: &str = ""; static mut COOKIE: &str = "";
static mut USER_AGENT: &str = ""; static mut USER_AGENT: &str = "";
@ -28,6 +30,20 @@ fn main() {
.help("Where to save the torrent files.") .help("Where to save the torrent files.")
.required(true), .required(true),
) )
.arg(
Arg::with_name("FETCH_SITES")
.short("fs")
.long("fetch_sites")
.help("Fetches from various torrent sites.")
)
.arg(
Arg::with_name("HASH_FILE")
.short("hf")
.long("hash_file")
.value_name("FILE")
.takes_value(true)
.help("The location of a file containing rows of infohashes. If given, it will download those infohashes."),
)
.arg( .arg(
Arg::with_name("TORRENTS_CSV_FILE") Arg::with_name("TORRENTS_CSV_FILE")
.short("t") .short("t")
@ -42,17 +58,32 @@ fn main() {
fetch_cloudflare_cookie(); fetch_cloudflare_cookie();
if matches.is_present("FETCH_SITES") {
// torrentz2(save_dir); // torrentz2(save_dir);
thepiratebay(save_dir); thepiratebay(save_dir);
magnetdl(save_dir); magnetdl(save_dir);
leetx(save_dir); leetx(save_dir);
skytorrents(save_dir); skytorrents(save_dir);
}
if let Some(t) = matches.value_of("HASH_FILE") {
hash_file_scan(Path::new(t), save_dir);
}
if let Some(t) = matches.value_of("TORRENTS_CSV_FILE") { if let Some(t) = matches.value_of("TORRENTS_CSV_FILE") {
torrents_csv_scan(Path::new(t), save_dir); torrents_csv_scan(Path::new(t), save_dir);
} }
} }
fn hash_file_scan(hash_file: &Path, save_dir: &Path) {
let f = File::open(&hash_file).expect("Unable to open file");
let f = BufReader::new(f);
for line in f.lines() {
let hash = line.expect("Unable to read line");
fetch_torrent(hash, save_dir);
}
}
fn torrents_csv_scan(torrents_csv_file: &Path, save_dir: &Path) { fn torrents_csv_scan(torrents_csv_file: &Path, save_dir: &Path) {
for hash in collect_info_hashes(torrents_csv_file) { for hash in collect_info_hashes(torrents_csv_file) {
fetch_torrent(hash, save_dir); fetch_torrent(hash, save_dir);

View File

@ -1,7 +1,7 @@
# This fetches from several torrent websites for new updates # This fetches from several torrent websites for new updates
cd ../new_torrents_fetcher cd ../new_torrents_fetcher
sort -r --field-separator=';' -n --key=5 ../torrents.csv > ../torrents.csv.seeders.desc sort -r --field-separator=';' -n --key=5 ../torrents.csv > ../torrents.csv.seeders.desc
cargo run --release -- -s "$1" -t ../torrents.csv.seeders.desc cargo run --release -- -s "$1" -f -t ../torrents.csv.seeders.desc
rm ../torrents.csv.seeders.desc rm ../torrents.csv.seeders.desc
cd ../scripts cd ../scripts
. scan_torrents.sh "$1" . scan_torrents.sh "$1"