From 85063c82328ea72fbdc8a7f1ba08e0551fe6cba6 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Tue, 27 Nov 2018 16:36:18 -0700 Subject: [PATCH] Updating readme --- README.md | 84 ++++++++++++++++++------------- scripts/scrape.sh | 13 ----- server/ui/src/components/home.tsx | 6 +-- 3 files changed, 53 insertions(+), 50 deletions(-) delete mode 100755 scripts/scrape.sh diff --git a/README.md b/README.md index 7462a58..983726d 100644 --- a/README.md +++ b/README.md @@ -2,41 +2,21 @@ -`Torrents.csv` is a collaborative, *vetted* repository of torrents, consisting of a single, searchable `torrents.csv` file. Its initially populated with a January 2017 backup of the pirate bay, and new torrents are periodically added from various torrents sites via a rust script. +[Demo Server](https://torrents-csv.ml) -`Torrents.csv` will only store torrents with at least one seeder to keep the file small, and will be periodically purged of non-seeded torrents, and sorted by seeders descending. +`Torrents.csv` is a *collaborative* repository of torrents, consisting of a single, searchable `torrents.csv` file. Its initially populated with a January 2017 backup of the pirate bay, and new torrents are periodically added from various torrents sites. It comes with a self-hostable [Torrents.csv webserver](https://torrents-csv.ml), a command line search, and a folder scanner to add torrents. -It also comes with a self-hostable [Torrents.csv webserver](https://torrents-csv.ml) +`Torrents.csv` will only store torrents with at least one seeder to keep the file small, will be periodically purged of non-seeded torrents, and sorted by seeders descending. -![img](https://i.imgur.com/qVmSVMC.png) +![img](https://i.imgur.com/GJQE3Xi.png) -To request more torrents, or add your own to the file, submit a pull request here. +To request more torrents, or add your own, go [here](https://gitlab.com/dessalines/torrents.csv/issues). Made with [Rust](https://www.rust-lang.org), [ripgrep](https://github.com/BurntSushi/ripgrep), [Actix](https://actix.rs/), [Inferno](https://www.infernojs.org), [Typescript](https://www.typescriptlang.org/). -## Searching +## Webserver -### Requirements - -- [ripgrep](https://github.com/BurntSushi/ripgrep) - -### Running - -``` -git clone https://gitlab.com/dessalines/torrents.csv -cd torrents.csv -./search.sh "bleh season 1" -``` - -``` -... -bleh season 1 (1993-) - seeders: 33 - size: 13GiB - link: magnet:?xt=urn:btih:INFO_HASH_HERE -``` - -## Running the webserver +`Torrents.csv` comes with a simple webserver. [Demo Server](https://torrents-csv.ml) ### Requirements @@ -45,11 +25,33 @@ bleh season 1 (1993-) ### Running -`Torrents.csv` comes with a simple webserver. Run `cd scripts && ./webserver.sh`, and goto http://localhost:8080 +``` +git clone https://gitlab.com/dessalines/torrents.csv +cd torrents.csv/scripts +./webserver.sh +``` +and goto http://localhost:8080 If running on a different host, open up `server/ui/src/env.ts` and change your hostname. -## Uploading / Adding Torrents +## Command Line Searching + +### Requirements + +- [ripgrep](https://github.com/BurntSushi/ripgrep) + +### Running +``` +git clone https://gitlab.com/dessalines/torrents.csv +cd torrents.csv +./search.sh "bleh season 1" +bleh season 1 (1993-) + seeders: 33 + size: 13GiB + link: magnet:?xt=urn:btih:INFO_HASH_HERE +``` + +## Uploading / Adding Torrents from a Directory An *upload*, consists of making a pull request after running the `scan_torrents.sh` script, which adds torrents from a directory you choose to the `.csv` file, after checking that they aren't already there, and that they have seeders. @@ -66,13 +68,29 @@ An *upload*, consists of making a pull request after running the `scan_torrents. git clone https://gitlab.com/[MY_USER]/torrents.csv cd torrents.csv/scripts ./scan_torrents.sh MY_TORRENTS_DIR # `MY_TORRENTS_DIR` is `~/.local/share/data/qBittorrent/BT_backup/` for qBittorrent on linux, but you can search for where your torrents are stored for your client. -git commit -am "Adding my torrents" +git commit -am "Adding [MY_USER] torrents" git push ``` Then [click here](https://gitlab.com/dessalines/torrents.csv/merge_requests/new) to do a pull/merge request to my branch. -## How the file looks +## Web scraping torrents + +`Torrents.csv` has a `Rust` repository for scraping new and top torrents from some torrent sites in the `new_torrents_fetcher` folder. It currently only has skytorrents, but more will be added later. + +### Requirements + +- Rust + +### Running + +``` +git clone https://gitlab.com/dessalines/torrents.csv +cd torrents.csv/scripts +./update.sh +``` + +## How the .csv file looks ```sh infohash;name;size_bytes;created_unix;seeders;leechers;completed;scraped_date @@ -81,9 +99,7 @@ infohash;name;size_bytes;created_unix;seeders;leechers;completed;scraped_date Made with [Rust](https://www.rust-lang.org), [ripgrep](https://github.com/BurntSushi/ripgrep), [Actix](https://actix.rs/), [Inferno](https://www.infernojs.org), and [Typescript](https://www.typescriptlang.org/). - -## Potential sources for new torrents - +## Sources for new torrents - https://www.skytorrents.lol/top100 - https://1337x.to/top-100 - https://1337x.to/trending diff --git a/scripts/scrape.sh b/scripts/scrape.sh deleted file mode 100755 index 1334fa8..0000000 --- a/scripts/scrape.sh +++ /dev/null @@ -1,13 +0,0 @@ - - - found_line=$(rg -n $infohash $torrents_csv | cut -d : -f 1) - - if [ ! -z $found_line ]; then - sed -i "$found_line c$add_line" $torrents_csv - echo -e "Found $name, updating peers" - else - # Append the add lines to the torrents.csv file - echo -e "\n$add_line" >> $torrents_csv - truncate -s -1 $torrents_csv # Removing last newline - echo -e "Added $name" - fi \ No newline at end of file diff --git a/server/ui/src/components/home.tsx b/server/ui/src/components/home.tsx index c4f2694..d04ac74 100644 --- a/server/ui/src/components/home.tsx +++ b/server/ui/src/components/home.tsx @@ -15,9 +15,9 @@ export class Home extends Component { return (

- Torrents.csv is a collaborative, vetted git repository of torrents, consisting of a single, searchable torrents.csv file. Its initially populated with a January 2017 backup of the pirate bay, and new torrents are periodically added from various torrents sites via a rust script.



- Torrents.csv will only store torrents with at least one seeder to keep the file small, and will be periodically purged of non-seeded torrents, and sorted by seeders descending.



- To request more torrents, or add your own to the file, go here.



+ Torrents.csv is a collaborative git repository of torrents, consisting of a single, searchable torrents.csv file. Its initially populated with a January 2017 backup of the pirate bay, and new torrents are periodically added from various torrents sites. It comes with a self-hostable webserver, a command line search, and a folder scanner to add torrents.

+ Torrents.csv will only store torrents with at least one seeder to keep the file small, will be periodically purged of non-seeded torrents, and sorted by seeders descending.

+ To request more torrents, or add your own, go here.

Made with Rust, ripgrep, Actix, Inferno, and Typescript.