Adding connection pooling.

docker/dev/Dockerfile

@@ -9,7 +9,7 @@ RUN yarn install --pure-lockfile
 COPY server/ui /app/ui
 RUN yarn build
 
-FROM rust:1.37 as rust
+FROM rust:1.39 as rust
 
 # Install musl
 RUN apt-get update

server/service/Cargo.lock

@@ -508,6 +508,16 @@ dependencies = [
  "synstructure 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "fallible-iterator"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "fallible-streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "flate2"
 version = "1.0.12"
@@ -700,7 +710,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "libsqlite3-sys"
-version = "0.10.0"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -957,6 +967,25 @@ dependencies = [
  "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "r2d2"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "scheduled-thread-pool 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "r2d2_sqlite"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "r2d2 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rand"
 version = "0.6.5"
@@ -1130,12 +1159,15 @@ dependencies = [
 
 [[package]]
 name = "rusqlite"
-version = "0.15.0"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "libsqlite3-sys 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1157,6 +1189,14 @@ name = "ryu"
 version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "scheduled-thread-pool"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.0.0"
@@ -1452,7 +1492,9 @@ dependencies = [
  "actix-files 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "actix-web 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "rusqlite 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "r2d2 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "r2d2_sqlite 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1703,6 +1745,8 @@ dependencies = [
 "checksum enum-as-inner 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3d58266c97445680766be408285e798d3401c6d4c378ec5552e78737e681e37d"
 "checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9"
 "checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08"
+"checksum fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+"checksum fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
 "checksum flate2 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)" = "ad3c5233c9a940c8719031b423d7e6c16af66e031cb0420b0896f5245bf181d3"
 "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
 "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
@@ -1727,7 +1771,7 @@ dependencies = [
 "checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a"
 "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 "checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8"
-"checksum libsqlite3-sys 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "742b695cbfb89e549dca6960a55e6802f67d352e33e97859ee46dee835211b0f"
+"checksum libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5e5b95e89c330291768dc840238db7f9e204fd208511ab6319b56193a7f2ae25"
 "checksum linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
 "checksum lock_api 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f8912e782533a93a167888781b836336a6ca5da6175c05944c86cf28c31104dc"
 "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
@@ -1759,6 +1803,8 @@ dependencies = [
 "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
 "checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
 "checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe"
+"checksum r2d2 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)" = "e48fa64898ef0286b6ee4b4d8f61483f9182acf5e44e62a398b1c7f56f2f861d"
+"checksum r2d2_sqlite 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "806e268035ce9e5a604bf617ac8a073ef28b59ef2e48e8338db0baf530caef33"
 "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
 "checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412"
 "checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
@@ -1778,10 +1824,11 @@ dependencies = [
 "checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd"
 "checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
 "checksum resolv-conf 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b263b4aa1b5de9ffc0054a2386f96992058bb6870aab516f8cdeb8a667d56dcb"
-"checksum rusqlite 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "39bae767eb27866f5c0be918635ae54af705bc09db11be2c43a3c6b361cf3462"
+"checksum rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2a194373ef527035645a1bc21b10dc2125f73497e6e155771233eb187aedd051"
 "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
 "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
 "checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8"
+"checksum scheduled-thread-pool 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bd07742e081ff6c077f5f6b283f12f32b9e7cc765b316160d66289b74546fbb3"
 "checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d"
 "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
 "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"

server/service/Cargo.toml

@@ -11,7 +11,9 @@ serde_json = "*"
 serde_derive = "*"
 time = "*"
 failure = "*"
+r2d2 = "0.8.6"
+r2d2_sqlite = "0.12.0"
 
 [dependencies.rusqlite]
-version = "0.15.0"
+version = "0.20.0"
 features = ["bundled"]

server/service/src/main.rs

@@ -8,24 +8,27 @@ extern crate rusqlite;
 extern crate time;
 #[macro_use]
 extern crate failure;
+extern crate r2d2;
+extern crate r2d2_sqlite;
 
 use actix_files as fs;
 use actix_files::NamedFile;
 use actix_web::{web, App, HttpRequest, HttpResponse, HttpServer};
 use failure::Error;
+use r2d2_sqlite::SqliteConnectionManager;
+use rusqlite::params;
 use std::collections::HashMap;
 use std::env;
 use std::ops::Deref;
 use std::sync::Mutex;
 use std::time::SystemTime;
 
-use rusqlite::Connection;
-
 const RATE_LIMIT: i32 = 10;
 const RATE_LIMIT_PER_SECOND: i32 = 60;
 
 pub struct State {
   rate_limits: Mutex<HashMap<String, RateLimitBucket>>,
+  pool: r2d2::Pool<SqliteConnectionManager>,
 }
 
 #[derive(Debug)]
@@ -37,8 +40,12 @@ pub struct RateLimitBucket {
 fn main() {
   println!("Access me at {}", endpoint());
 
+  let manager = SqliteConnectionManager::file(torrents_db_file());
+  let pool = r2d2::Pool::builder().max_size(15).build(manager).unwrap();
+
   let shared_data = web::Data::new(State {
     rate_limits: Mutex::new(HashMap::new()),
+    pool: pool,
   });
 
   HttpServer::new(move || {
@@ -99,11 +106,13 @@ fn search(
       .body(format!("{{\"error\": \"{}\"}}", "Empty query".to_string()));
   }
 
+  let conn = data.pool.get().unwrap();
+
   match check_rate_limit_full(data, &ip, RATE_LIMIT, RATE_LIMIT_PER_SECOND) {
     Ok(_) => HttpResponse::Ok()
       .header("Access-Control-Allow-Origin", "*")
       .content_type("application/json")
-      .body(search_query(query)),
+      .body(search_query(query, conn).unwrap()),
     Err(e) => HttpResponse::BadRequest()
       .header("Access-Control-Allow-Origin", "*")
       .content_type("application/json")
@@ -111,7 +120,10 @@ fn search(
   }
 }
 
-fn search_query(query: web::Query<SearchQuery>) -> String {
+fn search_query(
+  query: web::Query<SearchQuery>,
+  conn: r2d2::PooledConnection<SqliteConnectionManager>,
+) -> Result<String, serde_json::Error> {
   let page = query.page.unwrap_or(1);
   let size = query.size.unwrap_or(10);
   let type_ = query.type_.as_ref().map_or("torrent", String::deref);
@@ -123,11 +135,11 @@ fn search_query(query: web::Query<SearchQuery>) -> String {
   );
 
   if type_ == "file" {
-    let results = torrent_file_search(&query.q, size, offset);
+    let results = torrent_file_search(conn, &query.q, size, offset).unwrap();
-    serde_json::to_string(&results).unwrap()
+    serde_json::to_string(&results)
   } else {
-    let results = torrent_search(&query.q, size, offset);
+    let results = torrent_search(conn, &query.q, size, offset).unwrap();
-    serde_json::to_string(&results).unwrap()
+    serde_json::to_string(&results)
   }
 }
 
@@ -143,35 +155,39 @@ struct Torrent {
   scraped_date: u32,
 }
 
-fn torrent_search(query: &str, size: usize, offset: usize) -> Vec<Torrent> {
+fn torrent_search(
+  conn: r2d2::PooledConnection<SqliteConnectionManager>,
+  query: &str,
+  size: usize,
+  offset: usize,
+) -> Result<Vec<Torrent>, Error> {
   let stmt_str = "select * from torrents where name like '%' || ?1 || '%' limit ?2 offset ?3";
-  let conn = Connection::open(torrents_db_file()).unwrap();
+  let mut stmt = conn.prepare(&stmt_str)?;
-  let mut stmt = conn.prepare(&stmt_str).unwrap();
+  let torrent_iter = stmt.query_map(
-  let torrent_iter = stmt
+    params![
-    .query_map(
+      query.replace(" ", "%"),
-      &[
+      size.to_string(),
-        query.replace(" ", "%"),
+      offset.to_string(),
-        size.to_string(),
+    ],
-        offset.to_string(),
+    |row| {
-      ],
+      Ok(Torrent {
-      |row| Torrent {
+        infohash: row.get(0)?,
-        infohash: row.get(0),
+        name: row.get(1)?,
-        name: row.get(1),
+        size_bytes: row.get(2)?,
-        size_bytes: row.get(2),
+        created_unix: row.get(3)?,
-        created_unix: row.get(3),
+        seeders: row.get(4)?,
-        seeders: row.get(4),
+        leechers: row.get(5)?,
-        leechers: row.get(5),
+        completed: row.get(6)?,
-        completed: row.get(6),
+        scraped_date: row.get(7)?,
-        scraped_date: row.get(7),
+      })
-      },
+    },
-    )
+  )?;
-    .unwrap();
 
   let mut torrents = Vec::new();
   for torrent in torrent_iter {
     torrents.push(torrent.unwrap());
   }
-  torrents
+  Ok(torrents)
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -187,36 +203,40 @@ struct File {
   scraped_date: u32,
 }
 
-fn torrent_file_search(query: &str, size: usize, offset: usize) -> Vec<File> {
+fn torrent_file_search(
+  conn: r2d2::PooledConnection<SqliteConnectionManager>,
+  query: &str,
+  size: usize,
+  offset: usize,
+) -> Result<Vec<File>, Error> {
   let stmt_str = "select * from files where path like '%' || ?1 || '%' limit ?2 offset ?3";
-  let conn = Connection::open(torrents_db_file()).unwrap();
   let mut stmt = conn.prepare(&stmt_str).unwrap();
-  let file_iter = stmt
+  let file_iter = stmt.query_map(
-    .query_map(
+    params![
-      &[
+      query.replace(" ", "%"),
-        query.replace(" ", "%"),
+      size.to_string(),
-        size.to_string(),
+      offset.to_string(),
-        offset.to_string(),
+    ],
-      ],
+    |row| {
-      |row| File {
+      Ok(File {
-        infohash: row.get(0),
+        infohash: row.get(0)?,
-        index_: row.get(1),
+        index_: row.get(1)?,
-        path: row.get(2),
+        path: row.get(2)?,
-        size_bytes: row.get(3),
+        size_bytes: row.get(3)?,
-        created_unix: row.get(4),
+        created_unix: row.get(4)?,
-        seeders: row.get(5),
+        seeders: row.get(5)?,
-        leechers: row.get(6),
+        leechers: row.get(6)?,
-        completed: row.get(7),
+        completed: row.get(7)?,
-        scraped_date: row.get(8),
+        scraped_date: row.get(8)?,
-      },
+      })
-    )
+    },
-    .unwrap();
+  )?;
 
   let mut files = Vec::new();
   for file in file_iter {
     files.push(file.unwrap());
   }
-  files
+  Ok(files)
 }
 
 fn check_rate_limit_full(
@@ -243,7 +263,10 @@ fn check_rate_limit_full(
     };
 
     let current = SystemTime::now();
-    let time_passed = current.duration_since(rate_limit.last_checked)?.as_secs() as f64;
+    let time_passed = current
+      .duration_since(rate_limit.last_checked)
+      .unwrap()
+      .as_secs() as f64;
     rate_limit.last_checked = current;
     rate_limit.allowance += time_passed * (rate as f64 / per as f64);
     if rate_limit.allowance > rate as f64 {
@@ -260,7 +283,8 @@ fn check_rate_limit_full(
         &ip,
         rate,
         per
-      ))?
+      ))
+      .unwrap()
     } else {
       rate_limit.allowance -= 1.0;
       Ok(())