Added a Rust + Inferno Webserver.

.gitignore

@@ -1,3 +1,2 @@
 run.out
 old_greps.sh
-new_torrents_fetcher/target

README.md

@@ -1,18 +1,29 @@
 # Torrents.csv
 
-`Torrents.csv` is a collaborative, vetted database of torrents, consisting of a single, searchable `torrents.csv` file. Its initially populated with a January 2017 backup of the pirate bay, and new torrents are periodically added from various torrents sites via a rust script. 
+`Torrents.csv` is a collaborative, vetted git repository of torrents, consisting of a single, searchable `torrents.csv` file. Its initially populated with a January 2017 backup of the pirate bay, and new torrents are periodically added from various torrents sites via a rust script. 
 
 `Torrents.csv` will only store torrents with at least one seeder to keep the file small, and will be periodically purged of non-seeded torrents, and sorted by seeders descending.
 
+It also comes with a simple [Torrents.csv webserver](TODO)
+
+To request more torrents, or add your own to the file, submit a pull request here.
+
+Made with [Rust](https://www.rust-lang.org), [ripgrep](https://github.com/BurntSushi/ripgrep), [Actix](https://actix.rs/), [Inferno](https://www.infernojs.org), [Typescript](https://www.typescriptlang.org/).
+
 ## Searching
-To find torrents, run `./search.sh "frasier s01"`
+
+To find torrents, run `./search.sh "bleh season 1"`
 ```
-Frasier S01-S11 (1993-)
+bleh season 1 (1993-)
 	seeders: 33
 	size: 13GiB
-	link: magnet:?xt=urn:btih:3cc5142d0d139bcc9ea9925239a142770b98cf74
+	link: magnet:?xt=urn:btih:INFO_HASH_HERE
 ```
 
+## Running the webserver
+
+`Torrents.csv` comes with a simple webserver. Run `./webserver.sh`, and goto http://localhost:8080
+
 ## Uploading
 
 An *upload*, consists of making a pull request after running the `add_torrents.sh` script, which adds torrents from a directory you choose to the `.csv` file, after checking that they aren't already there, and that they have seeders.
@@ -29,6 +40,7 @@ git push
 Then [click here](https://gitlab.com/dessalines/torrents.csv/merge_requests/new) to do a pull/merge request to my branch.
 
 ## How the file looks
+
 ```sh
 infohash;name;size_bytes;created_unix;seeders;leechers;completed;scraped_date
 # torrents here...
@@ -39,13 +51,20 @@ infohash;name;size_bytes;created_unix;seeders;leechers;completed;scraped_date
 ### Searching
 - [ripgrep](https://github.com/BurntSushi/ripgrep)
 
+### Web server
+
+- Rust
+- ripgrep
+- Yarn
+
 ### Uploading
+
 - [Torrent tracker scraper](https://github.com/ZigmundVonZaun/torrent-tracker-scraper)
 - [Transmission-cli](https://transmissionbt.com/)
 - [Human Friendly](https://humanfriendly.readthedocs.io/en/latest/readme.html#command-line)
 
-
 ## Potential sources for new torrents
+
 - https://www.skytorrents.lol/top100
 - https://1337x.to/top-100
 - https://1337x.to/trending

new_torrents_fetcher/.gitignore

@@ -0,0 +1,2 @@
+/target
+**/*.rs.bk

server/service/.gitignore

@@ -0,0 +1,2 @@
+/target
+**/*.rs.bk

server/service/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "torrents-csv-service"
+version = "0.1.0"
+authors = ["Dessalines <happydooby@gmail.com>"]
+
+[dependencies]
+actix-web = "*"
+serde = "*"
+serde_derive = "*"
+pipers = "*"

server/service/src/main.rs

@@ -0,0 +1,63 @@
+extern crate actix_web;
+// extern crate Deserialize;
+extern crate serde;
+#[macro_use]
+extern crate serde_derive;
+extern crate pipers;
+
+use actix_web::{fs, http, server, App, HttpResponse, Query};
+use pipers::Pipe;
+
+fn main() {
+    server::new(|| {
+        App::new()
+            .route("/service/search", http::Method::GET, search)
+            .handler(
+                "/",
+                fs::StaticFiles::new("../ui/dist/")
+                    .unwrap()
+                    .index_file("index.html"),
+            )
+            .finish()
+    }).bind("127.0.0.1:8080")
+        .unwrap()
+        .run();
+}
+
+#[derive(Deserialize)]
+struct SearchQuery {
+    q: String,
+    page: Option<u32>,
+    size: Option<u32>,
+}
+
+fn search(query: Query<SearchQuery>) -> HttpResponse {
+    HttpResponse::Ok()
+        .header("Access-Control-Allow-Origin", "*")
+        .content_type("text/csv")
+        .body(ripgrep(query))
+}
+
+fn ripgrep(query: Query<SearchQuery>) -> String {
+    let page = query.page.unwrap_or(1);
+    let size = query.size.unwrap_or(10);
+    let offset = size * (page - 1) + 1;
+    let rg_search = format!("rg -i {} ../../torrents.csv", query.q.replace(" ", ".*"));
+
+    println!(
+        "search = {} , page = {}, size = {}, offset = {}",
+        rg_search, page, size, offset
+    );
+
+    let out = Pipe::new(&rg_search)
+        .then(format!("tail -n +{}", offset).as_mut_str())
+        .then(format!("head -n {}", size).as_mut_str())
+        .finally()
+        .expect("Commands did not pipe")
+        .wait_with_output()
+        .expect("failed to wait on child");
+
+    let mut results = format!("{}", String::from_utf8_lossy(&out.stdout));
+    results.pop(); // Remove last newline for some reason
+    results
+}

server/ui/.babelrc

@@ -0,0 +1,12 @@
+{
+  "compact": false,
+  "presets": [
+    ["es2015", {"loose": true, "modules": false}]
+  ],
+  "plugins": [
+    "transform-class-properties",
+    "transform-object-rest-spread",
+    "babel-plugin-syntax-jsx",
+    ["babel-plugin-inferno", {"imports": true}]
+  ]
+}

server/ui/.editorconfig

@@ -0,0 +1,9 @@
+[*.{js,jsx,ts,tsx,json}]
+indent_style = tab
+indent_size = 2
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+trim_trailing_whitespace = true
+insert_final_newline = true

server/ui/.gitignore

@@ -0,0 +1,30 @@
+dist
+
+_site
+.alm
+.history
+.git
+build
+.build
+.git
+.history
+.idea
+.jshintrc
+.nyc_output
+.sass-cache
+.vscode
+build
+coverage
+jsconfig.json
+Gemfile.lock
+node_modules
+.DS_Store
+*.map
+*.log
+*.swp
+*~
+test/data/result.json
+
+
+package-lock.json
+*.orig

server/ui/README.md

@@ -0,0 +1 @@
+# A simple UI for Torrents.csv written in Inferno

server/ui/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "torrents_csv_ui",
+  "version": "1.0.0",
+  "description": "A simple UI for Torrents.csv",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build": "webpack -p",
+    "lint": "tslint src/*.ts{,x} src/**/*.ts{,x}",
+    "start": "webpack-dev-server --inline"
+  },
+  "keywords": [
+  ],
+  "author": "Dominic Gannaway",
+  "license": "MIT",
+  "dependencies": {
+    "css-loader": "^1.0.0",
+    "inferno": "^4.0.1",
+    "moment": "^2.22.2",
+    "style-loader": "^0.23.1"
+  },
+  "devDependencies": {
+    "babel-core": "^6.26.0",
+    "babel-loader": "^7.1.2",
+    "babel-plugin-inferno": "^4.0.0",
+    "babel-plugin-syntax-object-rest-spread": "^6.13.0",
+    "babel-plugin-transform-class-properties": "^6.24.1",
+    "babel-plugin-transform-object-rest-spread": "^6.26.0",
+    "babel-preset-es2015": "^6.24.1",
+    "clean-webpack-plugin": "^0.1.18",
+    "html-webpack-plugin": "^2.30.1",
+    "source-map-loader": "^0.2.3",
+    "ts-loader": "^3.5.0",
+    "tslint": "^5.9.1",
+    "typescript": "^2.7.1",
+    "webpack": "3.11.0",
+    "webpack-dev-server": "2.11.1"
+  }
+}

server/ui/src/Main.css

@@ -0,0 +1,4 @@
+.red {
+    text-align: center;
+    color: red;
+}

server/ui/src/env.ts

@@ -0,0 +1 @@
+export const endpoint = "http://localhost:8080";

server/ui/src/index.html

@@ -0,0 +1,11 @@
+<html>
+	<head>
+		<title>Torrents.csv</title>
+		<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
+		<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.3.1/css/solid.css" integrity="sha384-VGP9aw4WtGH/uPAOseYxZ+Vz/vaTb1ehm1bwx92Fm8dTrE+3boLfF1SpAtB1z7HW" crossorigin="anonymous">
+		<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.3.1/css/fontawesome.css" integrity="sha384-1rquJLNOM3ijoueaaeS5m+McXPJCGdr5HcA03/VHXxcp2kX2sUrQDmFc3jR5i/C7" crossorigin="anonymous">
+	</head>
+	<body>
+		<div id="app"></div>
+	</body>
+</html>

server/ui/src/index.tsx

@@ -0,0 +1,207 @@
+import { render, Component, linkEvent } from 'inferno';
+import moment from 'moment';
+
+import { endpoint } from './env';
+import { SearchParams, Results, State } from './interfaces';
+import { convertCsvToJson, humanFileSize, magnetLink } from './utils';
+
+import './Main.css';
+
+const container = document.getElementById('app');
+
+class TorrentSearchComponent extends Component<any, State> {
+
+	state: State = {
+		results: {
+			torrents: []
+		},
+		searchParams: {
+			q: "",
+			page: 1
+		},
+		searching: false
+	};
+
+	constructor(props, context) {
+		super(props, context);
+
+	}
+
+	search(i: TorrentSearchComponent, event) {
+		event.preventDefault();
+
+		if (!!i.state.searchParams.q) {
+			i.setState({ searching: true, results: { torrents: [] } });
+			i.fetchData(i.state.searchParams)
+				.then(results => {
+					if (!!results) {
+						i.setState({
+							results: results
+						});
+					}
+				}).catch(error => {
+					console.error('request failed', error);
+				}).then(() => i.setState({ searching: false }));
+		} else {
+			i.setState({ results: { torrents: [] } });
+		}
+	}
+
+	fetchData(searchParams: SearchParams): Promise<Results> {
+		let q = encodeURI(searchParams.q);
+		return fetch(`${endpoint}/service/search?q=${q}&page=${searchParams.page}`)
+			.then(data => data.text())
+			.then(csv => convertCsvToJson(csv));
+	}
+
+	render() {
+		return (
+			<div>
+				{this.navbar()}
+				<div className={this.state.results.torrents[0]? "container-fluid" : "container"}>
+					<div class="row mt-2">
+						<div class="col-12">
+							{
+								this.state.searching ?
+									this.spinner()
+									:
+									this.state.results.torrents[0] ? this.table() : this.onboard()
+							}
+						</div>
+					</div>
+				</div>
+			</div>
+		);
+	}
+
+	table() {
+		return (
+			<div class="table-responsive-sm">
+				<table class="table table-hover">
+					<thead>
+						<tr>
+							<th>Name</th>
+							<th align="right">Size</th>
+							<th align="right">Seeds</th>
+							<th align="right">Leeches</th>
+							<th>Created</th>
+							<th>Scraped</th>
+							<th></th>
+						</tr>
+					</thead>
+					<tbody>
+						{this.state.results.torrents.map(torrent => (
+							<tr>
+								<td>{torrent.name}</td>
+								<td align="right">{humanFileSize(torrent.size_bytes, true)}</td>
+								<td align="right">{torrent.seeders}</td>
+								<td align="right">{torrent.leechers}</td>
+								<td>{moment(torrent.created_unix * 1000).fromNow()}</td>
+								<td>{moment(torrent.scraped_date * 1000).fromNow()}</td>
+								<td align="right">
+									<a href={magnetLink(torrent.infohash)}>
+										<i class="fas fa-magnet"></i>
+									</a>
+								</td>
+							</tr>
+						))}
+					</tbody>
+				</table>
+				{this.paginator()}
+			</div>
+		);
+	}
+
+	navbar() {
+		return (
+			<nav class="navbar navbar-light bg-light">
+				<a class="navbar-brand" href="#">
+					<i class="fas fa-fw fa-database mr-1"></i>Torrents.csv
+				</a>
+				<div class="col-12 col-sm-6">
+					{this.searchForm()}
+				</div>
+			</nav>
+		);
+	}
+
+	// TODO
+	// https://www.codeply.com/go/xBVaM3q5X4/bootstrap-4-navbar-search-full-width
+	searchForm() {
+		return (
+			<form class="my-2 my-lg-0 d-inline w-100" onSubmit={linkEvent(this, this.search)}>
+				<div class="input-group">
+					<input value={this.state.searchParams.q} onInput={linkEvent(this, this.searchChange)} type="text" class="form-control border" placeholder="Search..." />
+					<span class="input-group-append"></span>
+					<button type="submit" class="btn btn-outline-secondary border border-left-0">
+						<i className={(this.state.searching) ? "fas fa-spinner fa-spin" : "fas fa-fw fa-search"}></i>
+					</button>
+				</div>
+			</form>
+		);
+	}
+
+	spinner() {
+		return (
+			<div class="text-center">
+				<i class="fas fa-spinner fa-spin fa-5x"></i>
+			</div>
+		);
+	}
+
+	onboard() {
+		let site: string = "https://gitlab.com/dessalines/torrents.csv";
+		return (
+			<div>
+				<a href={site}>Torrents.csv</a> is a collaborative, <b>vetted</b> git repository of torrents, consisting of a single, searchable <code>torrents.csv</code> file. Its initially populated with a January 2017 backup of the pirate bay, and new torrents are periodically added from various torrents sites via a rust script.<br></br><br></br>
+				<a href={site}>Torrents.csv</a> will only store torrents with at least one seeder to keep the file small, and will be periodically purged of non-seeded torrents, and sorted by seeders descending.<br></br><br></br>
+				To request more torrents, or add your own to the file, go <a href={site}>here</a>.<br></br><br></br>
+				Made with <a href="https://www.rust-lang.org">Rust</a>, <a href="https://github.com/BurntSushi/ripgrep">ripgrep</a>, <a href="https://actix.rs/">Actix</a>, <a href="https://www.infernojs.org">Inferno</a>, and <a href="https://www.typescriptlang.org/">Typescript</a>.
+			</div>
+		);
+	}
+
+	paginator() {
+		return (
+			<nav>
+				<ul class="pagination">
+					<li className={(this.state.searchParams.page == 1) ? "page-item disabled" : "page-item"}>
+						<button class="page-link"
+							onClick={linkEvent({ i: this, nextPage: false }, this.switchPage)}
+						>
+							Previous</button>
+					</li>
+					<li class="page-item">
+						<button class="page-link"
+							onClick={linkEvent({ i: this, nextPage: true }, this.switchPage)}>
+							Next
+							</button>
+					</li>
+				</ul>
+			</nav>
+		);
+	}
+
+	searchChange(i, event) {
+		let searchParams: SearchParams = {
+			q: event.target.value,
+			page: 1
+		}
+		i.setState({ searchParams: searchParams });
+	}
+
+
+	switchPage(a: { i: TorrentSearchComponent, nextPage: boolean }, event) {
+
+		let newSearch = a.i.state.searchParams;
+		newSearch.page += (a.nextPage) ? 1 : -1;
+		a.i.setState({
+			searchParams: newSearch
+		});
+
+		a.i.search(a.i, event);
+	}
+
+}
+
+render(<TorrentSearchComponent />, container);

server/ui/src/interfaces.ts

@@ -0,0 +1,26 @@
+export interface SearchParams {
+	q: string;
+	page: number;
+	size?: number;
+}
+
+export interface Results {
+	torrents: Array<Torrent>;
+}
+
+export interface Torrent {
+	infohash: string;
+	name: string;
+	size_bytes: number;
+	created_unix: number;
+	seeders: number;
+	leechers: number;
+	completed: number;
+	scraped_date: number;
+}
+
+export interface State {
+	results: Results;
+	searchParams: SearchParams;
+	searching: Boolean;
+}

server/ui/src/utils.ts

@@ -0,0 +1,54 @@
+import { Results, Torrent } from './interfaces';
+
+export function magnetLink(infohash: string): string {
+  return "magnet:?xt=urn:btih:" + infohash +
+    "&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969%2Fannounce" +
+    "&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce" +
+    "&tr=udp%3A%2F%2Ftracker.internetwarriors.net%3A1337%2Fannounce" +
+    "&tr=udp%3A%2F%2Ftracker.openbittorrent.com%3A80"
+}
+
+export function humanFileSize(bytes, si): string {
+  var thresh = si ? 1000 : 1024;
+  if (Math.abs(bytes) < thresh) {
+    return bytes + ' B';
+  }
+  var units = si
+    ? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
+    : ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
+  var u = -1;
+  do {
+    bytes /= thresh;
+    ++u;
+  } while (Math.abs(bytes) >= thresh && u < units.length - 1);
+  return bytes.toFixed(1) + ' ' + units[u];
+}
+
+export function convertCsvToJson(csv: string): Results {
+
+  let lines = csv.split("\n");
+  let torrents: Array<Torrent> = [];
+
+  for (let line of lines) {
+    let cols = line.split(";");
+
+    // Sometimes it gets back weird newlines
+    if (cols[0]) {
+      torrents.push({
+        infohash: cols[0],
+        name: cols[1],
+        size_bytes: Number(cols[2]),
+        created_unix: Number(cols[3]),
+        seeders: Number(cols[4]),
+        leechers: Number(cols[5]),
+        completed: Number(cols[6]),
+        scraped_date: Number(cols[7])
+      });
+    }
+  }
+
+  let result = {
+    torrents: torrents
+  }
+  return result;
+}

server/ui/tsconfig.json

@@ -0,0 +1,31 @@
+{
+	"version": "2.2.2",
+	"compilerOptions": {
+		"target": "es6",
+		"module": "es6",
+		"allowJs": false,
+		"allowSyntheticDefaultImports": true,
+		"preserveConstEnums": true,
+		"sourceMap": true,
+		"moduleResolution": "node",
+		"lib": [
+			"es6",
+			"es7",
+			"dom"
+		],
+		"types": [
+			"inferno"
+		],
+		"jsx": "preserve",
+		"noUnusedLocals": true,
+		"strictNullChecks": true,
+		"removeComments": false
+	},
+	"include": [
+		"src/**/*"
+	],
+	"exclude": [
+		"node_modules"
+	],
+	"compileOnSave": false
+}

server/ui/tslint.json

@@ -0,0 +1,28 @@
+{
+    "extends": "tslint:recommended",
+    "rules": {
+			"forin": false,
+			"indent": [ true, "tabs" ],
+			"interface-name": false,
+			"ban-types": true,
+			"max-classes-per-file": true,
+			"max-line-length": false,
+			"member-access": true,
+			"member-ordering": false,
+			"no-bitwise": false,
+			"no-conditional-assignment": false,
+			"no-debugger": false,
+			"no-empty": true,
+			"no-namespace": false,
+			"no-unused-expression": true,
+			"object-literal-sort-keys": true,
+			"one-variable-per-declaration": [true, "ignore-for-loop"],
+			"only-arrow-functions": [false],
+			"ordered-imports": true,
+			"prefer-const": true,
+			"prefer-for-of": false,
+			"quotemark": [ true, "single", "jsx-double" ],
+			"trailing-comma": [true, {"multiline": "never", "singleline": "never"}],
+			"variable-name": false
+    }
+}

server/ui/webpack.config.js

@@ -0,0 +1,49 @@
+const webpack = require('webpack');
+const CleanWebpackPlugin = require('clean-webpack-plugin');
+const HtmlWebpackPlugin = require('html-webpack-plugin');
+
+module.exports = {
+	entry: "./src/index.tsx", // Point to main file
+	output: {
+		path: __dirname + "/dist",
+		filename: "bundle.js"
+	},
+	resolve: {
+		extensions: ['.js', '.jsx', '.ts', '.tsx']
+	},
+	performance: {
+		hints: false
+	},
+	module: {
+		loaders: [
+			{
+				test: /\.tsx?$/, 						  // All ts and tsx files will be process by
+				loaders: ['babel-loader', 'ts-loader'], // first babel-loader, then ts-loader
+				exclude: /node_modules/                   // ignore node_modules
+			}, {
+				test: /\.jsx?$/,                          // all js and jsx files will be processed by
+				loader: 'babel-loader',                   // babel-loader
+				exclude: /node_modules/                  // ignore node_modules
+			},
+			{ test: /\.css$/, loader: "style-loader!css-loader" },
+		]
+	},
+	devServer: {
+		contentBase: "src/",
+		historyApiFallback: true
+	},
+	plugins: [
+		new HtmlWebpackPlugin(
+			{
+				template: "./src/index.html",
+				inject: "body"
+			}
+		),
+		new CleanWebpackPlugin(
+			["dist"], {
+				verbose: true
+			}
+		),
+		new webpack.HotModuleReplacementPlugin()
+	]
+};

webserver.sh

@@ -0,0 +1,4 @@
+cd server/ui
+yarn build
+cd ../service
+cargo run