From 54ace61d7dbc00c84ac42b20275aa80435e6cd00 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Sun, 24 Nov 2019 12:07:54 -0800 Subject: [PATCH] Adding connection pooling. --- docker/dev/Dockerfile | 2 +- server/service/Cargo.lock | 59 ++++++++++++++-- server/service/Cargo.toml | 4 +- server/service/src/main.rs | 136 ++++++++++++++++++++++--------------- 4 files changed, 137 insertions(+), 64 deletions(-) diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index c98ac6d..9eb2604 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -9,7 +9,7 @@ RUN yarn install --pure-lockfile COPY server/ui /app/ui RUN yarn build -FROM rust:1.37 as rust +FROM rust:1.39 as rust # Install musl RUN apt-get update diff --git a/server/service/Cargo.lock b/server/service/Cargo.lock index b8eec90..a6d48c9 100644 --- a/server/service/Cargo.lock +++ b/server/service/Cargo.lock @@ -508,6 +508,16 @@ dependencies = [ "synstructure 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "flate2" version = "1.0.12" @@ -700,7 +710,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "libsqlite3-sys" -version = "0.10.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)", @@ -957,6 +967,25 @@ dependencies = [ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "r2d2" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "scheduled-thread-pool 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "r2d2_sqlite" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "r2d2 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rand" version = "0.6.5" @@ -1130,12 +1159,15 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.15.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libsqlite3-sys 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", "lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1157,6 +1189,14 @@ name = "ryu" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "scheduled-thread-pool" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "scopeguard" version = "1.0.0" @@ -1452,7 +1492,9 @@ dependencies = [ "actix-files 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "actix-web 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rusqlite 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "r2d2 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)", + "r2d2_sqlite 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1703,6 +1745,8 @@ dependencies = [ "checksum enum-as-inner 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3d58266c97445680766be408285e798d3401c6d4c378ec5552e78737e681e37d" "checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9" "checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08" +"checksum fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +"checksum fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" "checksum flate2 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)" = "ad3c5233c9a940c8719031b423d7e6c16af66e031cb0420b0896f5245bf181d3" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" @@ -1727,7 +1771,7 @@ dependencies = [ "checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" "checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8" -"checksum libsqlite3-sys 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "742b695cbfb89e549dca6960a55e6802f67d352e33e97859ee46dee835211b0f" +"checksum libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5e5b95e89c330291768dc840238db7f9e204fd208511ab6319b56193a7f2ae25" "checksum linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" "checksum lock_api 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f8912e782533a93a167888781b836336a6ca5da6175c05944c86cf28c31104dc" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" @@ -1759,6 +1803,8 @@ dependencies = [ "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" "checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" "checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +"checksum r2d2 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)" = "e48fa64898ef0286b6ee4b4d8f61483f9182acf5e44e62a398b1c7f56f2f861d" +"checksum r2d2_sqlite 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "806e268035ce9e5a604bf617ac8a073ef28b59ef2e48e8338db0baf530caef33" "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" "checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412" "checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" @@ -1778,10 +1824,11 @@ dependencies = [ "checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd" "checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716" "checksum resolv-conf 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b263b4aa1b5de9ffc0054a2386f96992058bb6870aab516f8cdeb8a667d56dcb" -"checksum rusqlite 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "39bae767eb27866f5c0be918635ae54af705bc09db11be2c43a3c6b361cf3462" +"checksum rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2a194373ef527035645a1bc21b10dc2125f73497e6e155771233eb187aedd051" "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" +"checksum scheduled-thread-pool 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bd07742e081ff6c077f5f6b283f12f32b9e7cc765b316160d66289b74546fbb3" "checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" diff --git a/server/service/Cargo.toml b/server/service/Cargo.toml index e8c5014..89710b8 100644 --- a/server/service/Cargo.toml +++ b/server/service/Cargo.toml @@ -11,7 +11,9 @@ serde_json = "*" serde_derive = "*" time = "*" failure = "*" +r2d2 = "0.8.6" +r2d2_sqlite = "0.12.0" [dependencies.rusqlite] -version = "0.15.0" +version = "0.20.0" features = ["bundled"] diff --git a/server/service/src/main.rs b/server/service/src/main.rs index b436e0c..fc2486c 100644 --- a/server/service/src/main.rs +++ b/server/service/src/main.rs @@ -8,24 +8,27 @@ extern crate rusqlite; extern crate time; #[macro_use] extern crate failure; +extern crate r2d2; +extern crate r2d2_sqlite; use actix_files as fs; use actix_files::NamedFile; use actix_web::{web, App, HttpRequest, HttpResponse, HttpServer}; use failure::Error; +use r2d2_sqlite::SqliteConnectionManager; +use rusqlite::params; use std::collections::HashMap; use std::env; use std::ops::Deref; use std::sync::Mutex; use std::time::SystemTime; -use rusqlite::Connection; - const RATE_LIMIT: i32 = 10; const RATE_LIMIT_PER_SECOND: i32 = 60; pub struct State { rate_limits: Mutex>, + pool: r2d2::Pool, } #[derive(Debug)] @@ -37,8 +40,12 @@ pub struct RateLimitBucket { fn main() { println!("Access me at {}", endpoint()); + let manager = SqliteConnectionManager::file(torrents_db_file()); + let pool = r2d2::Pool::builder().max_size(15).build(manager).unwrap(); + let shared_data = web::Data::new(State { rate_limits: Mutex::new(HashMap::new()), + pool: pool, }); HttpServer::new(move || { @@ -99,11 +106,13 @@ fn search( .body(format!("{{\"error\": \"{}\"}}", "Empty query".to_string())); } + let conn = data.pool.get().unwrap(); + match check_rate_limit_full(data, &ip, RATE_LIMIT, RATE_LIMIT_PER_SECOND) { Ok(_) => HttpResponse::Ok() .header("Access-Control-Allow-Origin", "*") .content_type("application/json") - .body(search_query(query)), + .body(search_query(query, conn).unwrap()), Err(e) => HttpResponse::BadRequest() .header("Access-Control-Allow-Origin", "*") .content_type("application/json") @@ -111,7 +120,10 @@ fn search( } } -fn search_query(query: web::Query) -> String { +fn search_query( + query: web::Query, + conn: r2d2::PooledConnection, +) -> Result { let page = query.page.unwrap_or(1); let size = query.size.unwrap_or(10); let type_ = query.type_.as_ref().map_or("torrent", String::deref); @@ -123,11 +135,11 @@ fn search_query(query: web::Query) -> String { ); if type_ == "file" { - let results = torrent_file_search(&query.q, size, offset); - serde_json::to_string(&results).unwrap() + let results = torrent_file_search(conn, &query.q, size, offset).unwrap(); + serde_json::to_string(&results) } else { - let results = torrent_search(&query.q, size, offset); - serde_json::to_string(&results).unwrap() + let results = torrent_search(conn, &query.q, size, offset).unwrap(); + serde_json::to_string(&results) } } @@ -143,35 +155,39 @@ struct Torrent { scraped_date: u32, } -fn torrent_search(query: &str, size: usize, offset: usize) -> Vec { +fn torrent_search( + conn: r2d2::PooledConnection, + query: &str, + size: usize, + offset: usize, +) -> Result, Error> { let stmt_str = "select * from torrents where name like '%' || ?1 || '%' limit ?2 offset ?3"; - let conn = Connection::open(torrents_db_file()).unwrap(); - let mut stmt = conn.prepare(&stmt_str).unwrap(); - let torrent_iter = stmt - .query_map( - &[ - query.replace(" ", "%"), - size.to_string(), - offset.to_string(), - ], - |row| Torrent { - infohash: row.get(0), - name: row.get(1), - size_bytes: row.get(2), - created_unix: row.get(3), - seeders: row.get(4), - leechers: row.get(5), - completed: row.get(6), - scraped_date: row.get(7), - }, - ) - .unwrap(); + let mut stmt = conn.prepare(&stmt_str)?; + let torrent_iter = stmt.query_map( + params![ + query.replace(" ", "%"), + size.to_string(), + offset.to_string(), + ], + |row| { + Ok(Torrent { + infohash: row.get(0)?, + name: row.get(1)?, + size_bytes: row.get(2)?, + created_unix: row.get(3)?, + seeders: row.get(4)?, + leechers: row.get(5)?, + completed: row.get(6)?, + scraped_date: row.get(7)?, + }) + }, + )?; let mut torrents = Vec::new(); for torrent in torrent_iter { torrents.push(torrent.unwrap()); } - torrents + Ok(torrents) } #[derive(Debug, Serialize, Deserialize)] @@ -187,36 +203,40 @@ struct File { scraped_date: u32, } -fn torrent_file_search(query: &str, size: usize, offset: usize) -> Vec { +fn torrent_file_search( + conn: r2d2::PooledConnection, + query: &str, + size: usize, + offset: usize, +) -> Result, Error> { let stmt_str = "select * from files where path like '%' || ?1 || '%' limit ?2 offset ?3"; - let conn = Connection::open(torrents_db_file()).unwrap(); let mut stmt = conn.prepare(&stmt_str).unwrap(); - let file_iter = stmt - .query_map( - &[ - query.replace(" ", "%"), - size.to_string(), - offset.to_string(), - ], - |row| File { - infohash: row.get(0), - index_: row.get(1), - path: row.get(2), - size_bytes: row.get(3), - created_unix: row.get(4), - seeders: row.get(5), - leechers: row.get(6), - completed: row.get(7), - scraped_date: row.get(8), - }, - ) - .unwrap(); + let file_iter = stmt.query_map( + params![ + query.replace(" ", "%"), + size.to_string(), + offset.to_string(), + ], + |row| { + Ok(File { + infohash: row.get(0)?, + index_: row.get(1)?, + path: row.get(2)?, + size_bytes: row.get(3)?, + created_unix: row.get(4)?, + seeders: row.get(5)?, + leechers: row.get(6)?, + completed: row.get(7)?, + scraped_date: row.get(8)?, + }) + }, + )?; let mut files = Vec::new(); for file in file_iter { files.push(file.unwrap()); } - files + Ok(files) } fn check_rate_limit_full( @@ -243,7 +263,10 @@ fn check_rate_limit_full( }; let current = SystemTime::now(); - let time_passed = current.duration_since(rate_limit.last_checked)?.as_secs() as f64; + let time_passed = current + .duration_since(rate_limit.last_checked) + .unwrap() + .as_secs() as f64; rate_limit.last_checked = current; rate_limit.allowance += time_passed * (rate as f64 / per as f64); if rate_limit.allowance > rate as f64 { @@ -260,7 +283,8 @@ fn check_rate_limit_full( &ip, rate, per - ))? + )) + .unwrap() } else { rate_limit.allowance -= 1.0; Ok(())