rob
/
torrents.csv
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' of https://gitlab.com/dessalines/torrents.csv

This commit is contained in:
Dessalines 2019-01-28 15:27:47 -08:00
parent d1e7c211b7 2544dfb8a5
commit f5b1282a47
1 changed files with 15 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
new_torrents_fetcher/src/main.rs
View File

@ -2,6 +2,7 @@ extern crate clap;
extern crate csv; extern crate csv;
extern crate reqwest; extern crate reqwest;
extern crate select; extern crate select;
use clap::{App, Arg}; use clap::{App, Arg};
use select::document::Document; use select::document::Document;
use select::predicate::{Attr, Class, Name, Predicate}; use select::predicate::{Attr, Class, Name, Predicate};
@ -179,6 +180,7 @@ fn skytorrents(save_dir: &Path) {
for page in pages.iter() { for page in pages.iter() {
println!("Fetching page {}", page); println!("Fetching page {}", page);
let html = match fetch_html(page) { let html = match fetch_html(page) {
Ok(t) => t, Ok(t) => t,
_err => continue, _err => continue,
@ -289,19 +291,19 @@ fn fetch_torrent(hash: String, save_dir: &Path) {
if !Path::new(&full_path).exists() { if !Path::new(&full_path).exists() {
unsafe { unsafe {
Command::new("curl") Command::new("curl")
.args(&[ .args(&[
&url, &url,
"-H", "-H",
USER_AGENT, USER_AGENT,
"-H", "-H",
COOKIE, COOKIE,
"--compressed", "--compressed",
"-o", "-o",
&full_path, &full_path,
"-s", "-s",
]) ])
.output() .output()
.expect("curl command failed"); .expect("curl command failed");
check_cloud_flare(Path::new(&full_path)); check_cloud_flare(Path::new(&full_path));
thread::sleep(time::Duration::from_millis(2742)); thread::sleep(time::Duration::from_millis(2742));
println!("{} saved.", &full_path); println!("{} saved.", &full_path);