Adding connection pooling.
This commit is contained in:
Dessalines
parent
cd3b313d88
commit
54ace61d7d
|
|
@ -9,7 +9,7 @@ RUN yarn install --pure-lockfile
|
|||
COPY server/ui /app/ui
|
||||
RUN yarn build
|
||||
|
||||
FROM rust:1.37 as rust
|
||||
FROM rust:1.39 as rust
|
||||
|
||||
# Install musl
|
||||
RUN apt-get update
|
||||
|
|
|
|||
|
|
@ -508,6 +508,16 @@ dependencies = [
|
|||
"synstructure 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-streaming-iterator"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.0.12"
|
||||
|
|
@ -700,7 +710,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
|
||||
[[package]]
|
||||
name = "libsqlite3-sys"
|
||||
version = "0.10.0"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -957,6 +967,25 @@ dependencies = [
|
|||
"proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r2d2"
|
||||
version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"scheduled-thread-pool 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r2d2_sqlite"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"r2d2 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.6.5"
|
||||
|
|
@ -1130,12 +1159,15 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rusqlite"
|
||||
version = "0.15.0"
|
||||
version = "0.20.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libsqlite3-sys 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
|
|
@ -1157,6 +1189,14 @@ name = "ryu"
|
|||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "scheduled-thread-pool"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.0.0"
|
||||
|
|
@ -1452,7 +1492,9 @@ dependencies = [
|
|||
"actix-files 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"actix-web 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rusqlite 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"r2d2 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"r2d2_sqlite 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -1703,6 +1745,8 @@ dependencies = [
|
|||
"checksum enum-as-inner 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3d58266c97445680766be408285e798d3401c6d4c378ec5552e78737e681e37d"
|
||||
"checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9"
|
||||
"checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08"
|
||||
"checksum fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
|
||||
"checksum fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
||||
"checksum flate2 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)" = "ad3c5233c9a940c8719031b423d7e6c16af66e031cb0420b0896f5245bf181d3"
|
||||
"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
|
||||
"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
|
||||
|
|
@ -1727,7 +1771,7 @@ dependencies = [
|
|||
"checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a"
|
||||
"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
"checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8"
|
||||
"checksum libsqlite3-sys 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "742b695cbfb89e549dca6960a55e6802f67d352e33e97859ee46dee835211b0f"
|
||||
"checksum libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5e5b95e89c330291768dc840238db7f9e204fd208511ab6319b56193a7f2ae25"
|
||||
"checksum linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
|
||||
"checksum lock_api 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f8912e782533a93a167888781b836336a6ca5da6175c05944c86cf28c31104dc"
|
||||
"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
|
||||
|
|
@ -1759,6 +1803,8 @@ dependencies = [
|
|||
"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
|
||||
"checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
|
||||
"checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe"
|
||||
"checksum r2d2 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)" = "e48fa64898ef0286b6ee4b4d8f61483f9182acf5e44e62a398b1c7f56f2f861d"
|
||||
"checksum r2d2_sqlite 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "806e268035ce9e5a604bf617ac8a073ef28b59ef2e48e8338db0baf530caef33"
|
||||
"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
|
||||
"checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412"
|
||||
"checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
|
||||
|
|
@ -1778,10 +1824,11 @@ dependencies = [
|
|||
"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd"
|
||||
"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
|
||||
"checksum resolv-conf 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b263b4aa1b5de9ffc0054a2386f96992058bb6870aab516f8cdeb8a667d56dcb"
|
||||
"checksum rusqlite 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "39bae767eb27866f5c0be918635ae54af705bc09db11be2c43a3c6b361cf3462"
|
||||
"checksum rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2a194373ef527035645a1bc21b10dc2125f73497e6e155771233eb187aedd051"
|
||||
"checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
|
||||
"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
|
||||
"checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8"
|
||||
"checksum scheduled-thread-pool 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bd07742e081ff6c077f5f6b283f12f32b9e7cc765b316160d66289b74546fbb3"
|
||||
"checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d"
|
||||
"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
|
||||
"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
|
||||
|
|
|
|||
|
|
@ -11,7 +11,9 @@ serde_json = "*"
|
|||
serde_derive = "*"
|
||||
time = "*"
|
||||
failure = "*"
|
||||
r2d2 = "0.8.6"
|
||||
r2d2_sqlite = "0.12.0"
|
||||
|
||||
[dependencies.rusqlite]
|
||||
version = "0.15.0"
|
||||
version = "0.20.0"
|
||||
features = ["bundled"]
|
||||
|
|
|
|||
|
|
@ -8,24 +8,27 @@ extern crate rusqlite;
|
|||
extern crate time;
|
||||
#[macro_use]
|
||||
extern crate failure;
|
||||
extern crate r2d2;
|
||||
extern crate r2d2_sqlite;
|
||||
|
||||
use actix_files as fs;
|
||||
use actix_files::NamedFile;
|
||||
use actix_web::{web, App, HttpRequest, HttpResponse, HttpServer};
|
||||
use failure::Error;
|
||||
use r2d2_sqlite::SqliteConnectionManager;
|
||||
use rusqlite::params;
|
||||
use std::collections::HashMap;
|
||||
use std::env;
|
||||
use std::ops::Deref;
|
||||
use std::sync::Mutex;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use rusqlite::Connection;
|
||||
|
||||
const RATE_LIMIT: i32 = 10;
|
||||
const RATE_LIMIT_PER_SECOND: i32 = 60;
|
||||
|
||||
pub struct State {
|
||||
rate_limits: Mutex<HashMap<String, RateLimitBucket>>,
|
||||
pool: r2d2::Pool<SqliteConnectionManager>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
|
|
@ -37,8 +40,12 @@ pub struct RateLimitBucket {
|
|||
fn main() {
|
||||
println!("Access me at {}", endpoint());
|
||||
|
||||
let manager = SqliteConnectionManager::file(torrents_db_file());
|
||||
let pool = r2d2::Pool::builder().max_size(15).build(manager).unwrap();
|
||||
|
||||
let shared_data = web::Data::new(State {
|
||||
rate_limits: Mutex::new(HashMap::new()),
|
||||
pool: pool,
|
||||
});
|
||||
|
||||
HttpServer::new(move || {
|
||||
|
|
@ -99,11 +106,13 @@ fn search(
|
|||
.body(format!("{{\"error\": \"{}\"}}", "Empty query".to_string()));
|
||||
}
|
||||
|
||||
let conn = data.pool.get().unwrap();
|
||||
|
||||
match check_rate_limit_full(data, &ip, RATE_LIMIT, RATE_LIMIT_PER_SECOND) {
|
||||
Ok(_) => HttpResponse::Ok()
|
||||
.header("Access-Control-Allow-Origin", "*")
|
||||
.content_type("application/json")
|
||||
.body(search_query(query)),
|
||||
.body(search_query(query, conn).unwrap()),
|
||||
Err(e) => HttpResponse::BadRequest()
|
||||
.header("Access-Control-Allow-Origin", "*")
|
||||
.content_type("application/json")
|
||||
|
|
@ -111,7 +120,10 @@ fn search(
|
|||
}
|
||||
}
|
||||
|
||||
fn search_query(query: web::Query<SearchQuery>) -> String {
|
||||
fn search_query(
|
||||
query: web::Query<SearchQuery>,
|
||||
conn: r2d2::PooledConnection<SqliteConnectionManager>,
|
||||
) -> Result<String, serde_json::Error> {
|
||||
let page = query.page.unwrap_or(1);
|
||||
let size = query.size.unwrap_or(10);
|
||||
let type_ = query.type_.as_ref().map_or("torrent", String::deref);
|
||||
|
|
@ -123,11 +135,11 @@ fn search_query(query: web::Query<SearchQuery>) -> String {
|
|||
);
|
||||
|
||||
if type_ == "file" {
|
||||
let results = torrent_file_search(&query.q, size, offset);
|
||||
serde_json::to_string(&results).unwrap()
|
||||
let results = torrent_file_search(conn, &query.q, size, offset).unwrap();
|
||||
serde_json::to_string(&results)
|
||||
} else {
|
||||
let results = torrent_search(&query.q, size, offset);
|
||||
serde_json::to_string(&results).unwrap()
|
||||
let results = torrent_search(conn, &query.q, size, offset).unwrap();
|
||||
serde_json::to_string(&results)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -143,35 +155,39 @@ struct Torrent {
|
|||
scraped_date: u32,
|
||||
}
|
||||
|
||||
fn torrent_search(query: &str, size: usize, offset: usize) -> Vec<Torrent> {
|
||||
fn torrent_search(
|
||||
conn: r2d2::PooledConnection<SqliteConnectionManager>,
|
||||
query: &str,
|
||||
size: usize,
|
||||
offset: usize,
|
||||
) -> Result<Vec<Torrent>, Error> {
|
||||
let stmt_str = "select * from torrents where name like '%' || ?1 || '%' limit ?2 offset ?3";
|
||||
let conn = Connection::open(torrents_db_file()).unwrap();
|
||||
let mut stmt = conn.prepare(&stmt_str).unwrap();
|
||||
let torrent_iter = stmt
|
||||
.query_map(
|
||||
&[
|
||||
let mut stmt = conn.prepare(&stmt_str)?;
|
||||
let torrent_iter = stmt.query_map(
|
||||
params![
|
||||
query.replace(" ", "%"),
|
||||
size.to_string(),
|
||||
offset.to_string(),
|
||||
],
|
||||
|row| Torrent {
|
||||
infohash: row.get(0),
|
||||
name: row.get(1),
|
||||
size_bytes: row.get(2),
|
||||
created_unix: row.get(3),
|
||||
seeders: row.get(4),
|
||||
leechers: row.get(5),
|
||||
completed: row.get(6),
|
||||
scraped_date: row.get(7),
|
||||
|row| {
|
||||
Ok(Torrent {
|
||||
infohash: row.get(0)?,
|
||||
name: row.get(1)?,
|
||||
size_bytes: row.get(2)?,
|
||||
created_unix: row.get(3)?,
|
||||
seeders: row.get(4)?,
|
||||
leechers: row.get(5)?,
|
||||
completed: row.get(6)?,
|
||||
scraped_date: row.get(7)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
)?;
|
||||
|
||||
let mut torrents = Vec::new();
|
||||
for torrent in torrent_iter {
|
||||
torrents.push(torrent.unwrap());
|
||||
}
|
||||
torrents
|
||||
Ok(torrents)
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
|
|
@ -187,36 +203,40 @@ struct File {
|
|||
scraped_date: u32,
|
||||
}
|
||||
|
||||
fn torrent_file_search(query: &str, size: usize, offset: usize) -> Vec<File> {
|
||||
fn torrent_file_search(
|
||||
conn: r2d2::PooledConnection<SqliteConnectionManager>,
|
||||
query: &str,
|
||||
size: usize,
|
||||
offset: usize,
|
||||
) -> Result<Vec<File>, Error> {
|
||||
let stmt_str = "select * from files where path like '%' || ?1 || '%' limit ?2 offset ?3";
|
||||
let conn = Connection::open(torrents_db_file()).unwrap();
|
||||
let mut stmt = conn.prepare(&stmt_str).unwrap();
|
||||
let file_iter = stmt
|
||||
.query_map(
|
||||
&[
|
||||
let file_iter = stmt.query_map(
|
||||
params![
|
||||
query.replace(" ", "%"),
|
||||
size.to_string(),
|
||||
offset.to_string(),
|
||||
],
|
||||
|row| File {
|
||||
infohash: row.get(0),
|
||||
index_: row.get(1),
|
||||
path: row.get(2),
|
||||
size_bytes: row.get(3),
|
||||
created_unix: row.get(4),
|
||||
seeders: row.get(5),
|
||||
leechers: row.get(6),
|
||||
completed: row.get(7),
|
||||
scraped_date: row.get(8),
|
||||
|row| {
|
||||
Ok(File {
|
||||
infohash: row.get(0)?,
|
||||
index_: row.get(1)?,
|
||||
path: row.get(2)?,
|
||||
size_bytes: row.get(3)?,
|
||||
created_unix: row.get(4)?,
|
||||
seeders: row.get(5)?,
|
||||
leechers: row.get(6)?,
|
||||
completed: row.get(7)?,
|
||||
scraped_date: row.get(8)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
)?;
|
||||
|
||||
let mut files = Vec::new();
|
||||
for file in file_iter {
|
||||
files.push(file.unwrap());
|
||||
}
|
||||
files
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn check_rate_limit_full(
|
||||
|
|
@ -243,7 +263,10 @@ fn check_rate_limit_full(
|
|||
};
|
||||
|
||||
let current = SystemTime::now();
|
||||
let time_passed = current.duration_since(rate_limit.last_checked)?.as_secs() as f64;
|
||||
let time_passed = current
|
||||
.duration_since(rate_limit.last_checked)
|
||||
.unwrap()
|
||||
.as_secs() as f64;
|
||||
rate_limit.last_checked = current;
|
||||
rate_limit.allowance += time_passed * (rate as f64 / per as f64);
|
||||
if rate_limit.allowance > rate as f64 {
|
||||
|
|
@ -260,7 +283,8 @@ fn check_rate_limit_full(
|
|||
&ip,
|
||||
rate,
|
||||
per
|
||||
))?
|
||||
))
|
||||
.unwrap()
|
||||
} else {
|
||||
rate_limit.allowance -= 1.0;
|
||||
Ok(())
|
||||
|
|
|
|||
Loading…
Reference in New Issue