Converting webserver to sqlite version for speed. Fixes #23

Adding a performance comparison. Fixes #21
Fixing some favicon asset copying issues.

.gitignore

@@ -1,2 +1,3 @@
 run.out
 old_greps.sh
+torrents.db

scripts/build_sqlite.sh

@@ -0,0 +1,24 @@
+# Remove quotes
+sed 's/\"//g' ../torrents.csv > torrents_removed_quotes.csv
+
+rm ../torrents.db
+
+sqlite3 -batch ../torrents.db <<"EOF"
+create table torrents(
+  "infohash" TEXT,
+  "name" TEXT,
+  "size_bytes" INTEGER,
+  "created_unix" INTEGER,
+  "seeders" INTEGER,
+  "leechers" INTEGER,
+  "completed" INTEGER,
+  "scraped_date" INTEGER
+);
+.separator ";"
+.import torrents_removed_quotes.csv torrents
+UPDATE torrents SET completed=NULL WHERE completed = '';
+create index name_index on torrents (name);
+EOF
+rm torrents_removed_quotes.csv
+
+

scripts/sqlite_performance_test.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+touch test.db
+# Remove quotes
+sed 's/\"//g' ../torrents.csv > torrents_removed_quotes.csv
+
+
+sqlite3 -batch test.db <<"EOF"
+create table torrents ();
+.separator ";"
+.import torrents_removed_quotes.csv torrents
+create index name_index on torrents (name);
+EOF
+
+echo "Ripgrep time for pearl jam"
+time rg -i "pearl.*jam" ../torrents.csv > /dev/null
+
+echo "Sqlite3 time"
+time sqlite3 test.db "select * from torrents where name like '%pearl%jam%fix%' limit 10" > /dev/null
+
+echo "Ripgrep time for avengers infinity"
+time rg -i "avengers.*infinity" ../torrents.csv > /dev/null
+
+echo "Sqlite3 time"
+time sqlite3 test.db "select * from torrents where name like '%avengers%infinity%' limit 10" > /dev/null
+
+echo "Ripgrep time for w"
+time rg -i "w" ../torrents.csv > /dev/null
+
+echo "Sqlite3 time"
+time sqlite3 test.db "select * from torrents where name like '%w%' limit 10" > /dev/null
+
+echo "Ripgrep time for prestige 2006"
+time rg -i "prestige.*2006" ../torrents.csv > /dev/null
+
+echo "Sqlite3 time"
+time sqlite3 test.db "select * from torrents where name like '%prestige%2006%'  limit 10" > /dev/null
+
+rm test.db
+rm torrents_removed_quotes.csv

scripts/webserver.sh

@@ -4,6 +4,9 @@
 # export TORRENTS_CSV_FILE=`pwd`/../../torrents.csv
 # export TORRENTS_CSV_FRONT_END_DIR=`pwd`/../ui/dist
 
+# Build the sqlite db from the CSV
+. build_sqlite.sh
+
 # Build front end
 cd ../server/ui
 yarn

server/service/Cargo.lock

@@ -130,16 +130,6 @@ dependencies = [
  "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "atty"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
- "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "backtrace"
 version = "0.2.3"
@@ -207,11 +197,6 @@ dependencies = [
  "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "bytecount"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "byteorder"
 version = "1.2.6"
@@ -378,22 +363,6 @@ name = "encoding_index_tests"
 version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "encoding_rs"
-version = "0.8.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "encoding_rs_io"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "encoding_rs 0.8.10 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "error-chain"
 version = "0.1.12"
@@ -472,94 +441,6 @@ dependencies = [
  "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "globset"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "grep"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "grep-cli 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "grep-matcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "grep-printer 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "grep-regex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "grep-searcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "grep-cli"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "same-file 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "grep-matcher"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "grep-printer"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "grep-matcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "grep-searcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)",
- "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "grep-regex"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "grep-matcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "grep-searcher"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bytecount 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "encoding_rs 0.8.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "encoding_rs_io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "grep-matcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "h2"
 version = "0.1.12"
@@ -679,6 +560,15 @@ name = "libc"
 version = "0.2.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "libsqlite3-sys"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
+ "vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "linked-hash-map"
 version = "0.4.2"
@@ -724,15 +614,6 @@ dependencies = [
  "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "memmap"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "memoffset"
 version = "0.2.1"
@@ -897,6 +778,11 @@ dependencies = [
  "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "pkg-config"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "proc-macro2"
 version = "0.4.20"
@@ -958,14 +844,6 @@ name = "redox_syscall"
 version = "0.1.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "redox_termios"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "regex"
 version = "1.0.5"
@@ -1006,6 +884,17 @@ dependencies = [
  "untrusted 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rusqlite"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libsqlite3-sys 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lru-cache 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.9"
@@ -1029,14 +918,6 @@ name = "safemem"
 version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "same-file"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "scopeguard"
 version = "0.3.3"
@@ -1166,24 +1047,6 @@ dependencies = [
  "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "termcolor"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "termion"
-version = "1.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
- "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
- "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "thread_local"
 version = "0.3.6"
@@ -1375,9 +1238,10 @@ name = "torrents-csv-service"
 version = "0.1.0"
 dependencies = [
  "actix-web 0.7.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "grep 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rusqlite 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1547,6 +1411,11 @@ dependencies = [
  "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "vcpkg"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "version_check"
 version = "0.1.5"
@@ -1586,28 +1455,11 @@ name = "winapi-i686-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "winapi-util"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "wincolor"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "winreg"
 version = "0.5.1"
@@ -1640,7 +1492,6 @@ dependencies = [
 "checksum actix_derive 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b9d1525ef45e5e021f0b93dace157dcab5d792acb4cc78f3213787d65e2bb92"
 "checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a"
 "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"
-"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652"
 "checksum backtrace 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "346d7644f0b5f9bc73082d3b2236b69a05fd35cce0cfa3724e184e6a5c9e2a2f"
 "checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a"
 "checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0"
@@ -1648,7 +1499,6 @@ dependencies = [
 "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12"
 "checksum brotli-sys 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd"
 "checksum brotli2 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e"
-"checksum bytecount 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f861d9ce359f56dbcb6e0c2a1cb84e52ad732cadb57b806adeb3c7668caccbd8"
 "checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781"
 "checksum bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0ce55bd354b095246fc34caf4e9e242f5297a7fd938b090cadfea6eee614aa62"
 "checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16"
@@ -1669,8 +1519,6 @@ dependencies = [
 "checksum encoding-index-singlebyte 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a"
 "checksum encoding-index-tradchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18"
 "checksum encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
-"checksum encoding_rs 0.8.10 (registry+https://github.com/rust-lang/crates.io-index)" = "065f4d0c826fdaef059ac45487169d918558e3cf86c9d89f6e81cf52369126e5"
-"checksum encoding_rs_io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "098f6a0ab73a9ba256b71344dc82c6d7e252736ad9db7f4e35345f3a1f8713f5"
 "checksum error-chain 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faa976b4fd2e4c2b2f3f486874b19e61944d3de3de8b61c9fcf835d583871bcc"
 "checksum error-chain 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6930e04918388a9a2e41d518c25cf679ccafe26733fb4127dbf21993f2575d46"
 "checksum failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7efb22686e4a466b1ec1a15c2898f91fa9cb340452496dca654032de20ff95b9"
@@ -1681,13 +1529,6 @@ dependencies = [
 "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
 "checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b"
 "checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4"
-"checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865"
-"checksum grep 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d64cad6bc6831698fb55b64e7a973eec7ac9910d8b8b3755e523bf8fec7d0737"
-"checksum grep-cli 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5933126a04a66e612378d38bee4ea76fc1a3e0086ae1122d5a2faed802d6c8c3"
-"checksum grep-matcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "820946e0c314510779a8d86c5cd03240e0ae0993dabcdb98733a8f6a9001b607"
-"checksum grep-printer 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bb877af207b46e3baacd41c5be5bbda79cf1d0da95583051a3e7e110cb27a49b"
-"checksum grep-regex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8d7ebe41242224840865003a4bc4bf9dea411367cee03832e958ec4ef78c0c9b"
-"checksum grep-searcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6eb23805170ff0e96894a24847019500de11e9baaabe3dafed75b35a897636e1"
 "checksum h2 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "a27e7ed946e8335bdf9a191bc1b9b14a03ba822d013d2f58437f4fabcbd7fc2c"
 "checksum hostname 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "21ceb46a83a85e824ef93669c8b390009623863b5c195d1ba747292c0c72f94e"
 "checksum htmlescape 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
@@ -1703,13 +1544,13 @@ dependencies = [
 "checksum lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca488b89a5657b0a2ecd45b95609b3e848cf1755da332a0da46e2b2b1cb371a7"
 "checksum lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ddba4c30a78328befecec92fc94970e53b3ae385827d28620f0f5bb2493081e0"
 "checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d"
+"checksum libsqlite3-sys 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "742b695cbfb89e549dca6960a55e6802f67d352e33e97859ee46dee835211b0f"
 "checksum linked-hash-map 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7860ec297f7008ff7a1e3382d7f7e1dcd69efc94751a2284bafc3d013c2aa939"
 "checksum lock_api 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "775751a3e69bde4df9b38dd00a1b5d6ac13791e4223d4a0506577f0dd27cfb7a"
 "checksum log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fcce5fa49cc693c312001daf1d13411c4a5283796bac1084299ea3e567113f"
 "checksum lru-cache 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4d06ff7ff06f729ce5f4e227876cb88d10bc59cd4ae1e09fbb2bde15c850dc21"
 "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
 "checksum memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4b3629fe9fdbff6daa6c33b90f7c08355c1aca05a3d01fa8063b822fcf185f3b"
-"checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff"
 "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3"
 "checksum mime 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "4b082692d3f6cf41b453af73839ce3dfc212c4411cbb2441dff80a716e38bd79"
 "checksum mime_guess 2.0.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "30de2e4613efcba1ec63d8133f344076952090c122992a903359be5a4f99c3ed"
@@ -1728,6 +1569,7 @@ dependencies = [
 "checksum phf_codegen 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "7d187f00cd98d5afbcd8898f6cf181743a449162aeb329dcd2f3849009e605ad"
 "checksum phf_generator 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "03dc191feb9b08b0dc1330d6549b795b9d81aec19efe6b4a45aec8d4caee0c4b"
 "checksum phf_shared 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "b539898d22d4273ded07f64a05737649dc69095d92cb87c7097ec68e3f150b93"
+"checksum pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "676e8eb2b1b4c9043511a9b7bea0915320d7e502b0a079fb03f9635a5252b18c"
 "checksum proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)" = "3d7b7eaaa90b4a90a932a9ea6666c95a389e424eff347f0f793979289429feee"
 "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
 "checksum quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "dd636425967c33af890042c483632d33fa7a18f19ad1d7ea72e8998c6ef8dea5"
@@ -1736,16 +1578,15 @@ dependencies = [
 "checksum rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1961a422c4d189dfb50ffa9320bf1f2a9bd54ecb92792fb9477f99a1045f3372"
 "checksum rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0905b6b7079ec73b314d4c748701f6931eb79fd97c668caa3f1899b22b32c6db"
 "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1"
-"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
 "checksum regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2069749032ea3ec200ca51e4a31df41759190a88edca0d2d86ee8bedf7073341"
 "checksum regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "747ba3b235651f6e2f67dfa8bcdcd073ddb7c243cb21c442fc12395dfcac212d"
 "checksum resolv-conf 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c62bd95a41841efdf7fca2ae9951e64a8d8eae7e5da196d8ce489a2241491a92"
 "checksum ring 0.13.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe642b9dd1ba0038d78c4a3999d1ee56178b4d415c1e1fbaba83b06dce012f0"
+"checksum rusqlite 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "39bae767eb27866f5c0be918635ae54af705bc09db11be2c43a3c6b361cf3462"
 "checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395"
 "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
 "checksum ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7153dd96dade874ab973e098cb62fcdbb89a03682e46b144fd09550998d4a4a7"
 "checksum safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8dca453248a96cb0749e36ccdfe2b0b4e54a61bfef89fb97ec621eb8e0a93dd9"
-"checksum same-file 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "10f7794e2fda7f594866840e95f5c5962e886e228e68b6505885811a94dd728c"
 "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
 "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
 "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
@@ -1763,8 +1604,6 @@ dependencies = [
 "checksum syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)" = "261ae9ecaa397c42b960649561949d69311f08eeaea86a65696e6e46517cf741"
 "checksum syn 0.15.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b10ee269228fb723234fce98e9aac0eaed2bd5f1ad2f6930e8d5b93f04445a1a"
 "checksum synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb9b7550d063ea184027c9b8c20ac167cd36d3e06b3a40bceb9d746dc1a7b7"
-"checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f"
-"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
 "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
 "checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b"
 "checksum tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "6e93c78d23cc61aa245a8acd2c4a79c4d7fa7fb5c3ca90d5737029f043a84895"
@@ -1796,6 +1635,7 @@ dependencies = [
 "checksum url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2a321979c09843d272956e73700d12c4e7d3d92b2ee112b31548aef0d4efc5a6"
 "checksum utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd70f467df6810094968e2fce0ee1bd0e87157aceb026a8c083bcf5e25b9efe4"
 "checksum uuid 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dab5c5526c5caa3d106653401a267fed923e7046f35895ffcb5ca42db64942e6"
+"checksum vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "def296d3eb3b12371b2c7d0e83bfe1403e4db2d7a0bba324a12b21c4ee13143d"
 "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
 "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
 "checksum widestring 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7157704c2e12e3d2189c507b7482c52820a16dfa4465ba91add92f266667cadb"
@@ -1803,9 +1643,7 @@ dependencies = [
 "checksum winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0"
 "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
 "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-"checksum winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "afc5508759c5bf4285e61feb862b6083c8480aec864fa17a81fdec6f69b461ab"
 "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-"checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba"
 "checksum winreg 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a27a759395c1195c4cc5cda607ef6f8f6498f64e78f7900f5de0a127a424704a"
 "checksum winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7daf138b6b14196e3830a588acf1e86966c694d3e8fb026fb105b8b5dca07e6e"
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"

server/service/Cargo.toml

@@ -6,6 +6,7 @@ authors = ["Dessalines <happydooby@gmail.com>"]
 [dependencies]
 actix-web = "*"
 serde = "*"
+serde_json = "*"
 serde_derive = "*"
-grep = "*"
+rusqlite = "*"
 time = "*"

server/service/src/main.rs

@@ -1,27 +1,20 @@
 extern crate actix_web;
 extern crate serde;
+extern crate serde_json;
 #[macro_use]
 extern crate serde_derive;
-extern crate grep;
+extern crate rusqlite;
 extern crate time;
 
 use actix_web::{fs, fs::NamedFile, http, server, App, HttpRequest, HttpResponse, Query};
-
-use std::error::Error;
-
-use grep::regex::RegexMatcherBuilder;
-use grep::searcher::sinks::Lossy;
-use grep::searcher::{BinaryDetection, SearcherBuilder};
 use std::env;
-use std::fs::File;
+
+use rusqlite::{Connection, NO_PARAMS};
 
 fn main() {
-
-  
   server::new(|| {
     App::new()
       .route("/service/search", http::Method::GET, search)
-      .resource("/favicon.ico", |r| r.f(favicon))
       .resource("/", |r| r.f(index))
       .handler(
         "/static",
@@ -39,16 +32,12 @@ fn index(_req: &HttpRequest) -> Result<NamedFile, actix_web::error::Error> {
   Ok(NamedFile::open(front_end_dir() + "/index.html")?)
 }
 
-fn favicon(_req: &HttpRequest) -> Result<NamedFile, actix_web::error::Error> {
-  Ok(NamedFile::open(front_end_dir() + "/favicon.ico")?)
-}
-
 fn front_end_dir() -> String {
   env::var("TORRENTS_CSV_FRONT_END_DIR").unwrap_or("../ui/dist".to_string())
 }
 
-fn torrents_csv_file() -> String {
+fn torrents_db_file() -> String {
-  env::var("TORRENTS_CSV_FILE").unwrap_or("../../torrents.csv".to_string())
+  env::var("TORRENTS_CSV_DB_FILE").unwrap_or("../../torrents.db".to_string())
 }
 
 #[derive(Deserialize)]
@@ -62,65 +51,73 @@ fn search(query: Query<SearchQuery>) -> HttpResponse {
   HttpResponse::Ok()
     .header("Access-Control-Allow-Origin", "*")
     .content_type("text/csv")
-    .body(ripgrep(query))
+    .body(search_query(query))
 }
 
-fn ripgrep(query: Query<SearchQuery>) -> String {
+fn search_query(query: Query<SearchQuery>) -> String {
   let page = query.page.unwrap_or(1);
   let size = query.size.unwrap_or(10);
   let offset = size * (page - 1);
-  let csv_file = File::open(torrents_csv_file());
 
   println!("query = {} , page = {}, size = {}", query.q, page, size);
 
-  let results = search_file(csv_file.unwrap(), &query.q).unwrap();
+  let results = sql_search(&query.q, size, offset);
 
-  let last: usize = if offset + size < results.len() {
+  serde_json::to_string(&results).unwrap()
-    offset + size
-  } else {
-    results.len()
-  };
-
-  results[offset..last].join("")
 }
 
-fn search_file(file: File, query: &str) -> Result<Vec<String>, Box<Error>> {
+#[derive(Debug, Serialize, Deserialize)]
-  let pattern = query.replace(" ", ".*");
+struct Torrent {
+  infohash: String,
+  name: String,
+  size_bytes: isize,
+  created_unix: u32,
+  seeders: u32,
+  leechers: u32,
+  completed: Option<u32>,
+  scraped_date: u32,
+}
 
-  let matcher = RegexMatcherBuilder::new()
+fn sql_search(query: &str, size: usize, offset: usize) -> Vec<Torrent> {
-    .case_insensitive(true)
+  let stmt_str = format!(
-    .build(&pattern)
+    "select * from torrents where name like '%{}%' limit {} offset {}",
+    query.replace(" ", "%"),
+    size,
+    offset
+  );
+
+  let conn = Connection::open(torrents_db_file()).unwrap();
+
+  let mut stmt = conn.prepare(&stmt_str).unwrap();
+  let torrent_iter = stmt
+    .query_map(NO_PARAMS, |row| Torrent {
+      infohash: row.get(0),
+      name: row.get(1),
+      size_bytes: row.get(2),
+      created_unix: row.get(3),
+      seeders: row.get(4),
+      leechers: row.get(5),
+      completed: row.get(6),
+      scraped_date: row.get(7),
+    })
     .unwrap();
 
-  let mut matches: Vec<String> = vec![];
+  let mut torrents = Vec::new();
-
+  for torrent in torrent_iter {
-  let mut searcher = SearcherBuilder::new()
+    torrents.push(torrent.unwrap());
-    .binary_detection(BinaryDetection::quit(b'\x00'))
+  }
-    // .line_number(false)
+  torrents
-    .build();
-
-  searcher.search_file(
-    &matcher,
-    &file,
-    Lossy(|_lnum, line| {
-      matches.push(line.to_string());
-      Ok(true)
-    }),
-  )?;
-
-  Ok(matches)
 }
 
 #[cfg(test)]
 mod tests {
-  use std::fs::File;
   use time::PreciseTime;
 
   #[test]
   fn test() {
     let start = PreciseTime::now();
     let results =
-      super::search_file(File::open(super::torrents_csv_file()).unwrap(), "sherlock").unwrap();
+      super::sql_search("sherlock", 10, 0);
     assert!(results.len() > 2);
     let end = PreciseTime::now();
     println!("Query took {} seconds.", start.to(end));

server/ui/fuse.js

@@ -4,8 +4,7 @@ const {
   EnvPlugin,
   CSSPlugin,
   WebIndexPlugin,
-  QuantumPlugin,
+  QuantumPlugin
-  CopyPlugin
 } = require('fuse-box');
 // const transformInferno = require('../../dist').default
 const transformInferno = require('ts-transform-inferno').default;
@@ -44,12 +43,13 @@ Sparky.task('config', _ => {
 });
 Sparky.task('clean', _ => Sparky.src('dist/').clean('dist/'));
 Sparky.task('env', _ => (isProduction = true));
-Sparky.task('dev', ['clean', 'config'], _ => {
+Sparky.task('copy-assets', () => Sparky.src('assets/*.ico').dest('dist/'));
+Sparky.task('dev', ['clean', 'config', 'copy-assets'], _ => {
   fuse.dev();
   app.hmr().watch();
   return fuse.run();
 });
-Sparky.task('prod', ['clean', 'env', 'config'], _ => {
+Sparky.task('prod', ['clean', 'env', 'config', 'copy-assets'], _ => {
   // fuse.dev({ reload: true }); // remove after demo
   return fuse.run();
 });

server/ui/src/components/search.tsx

@@ -2,8 +2,8 @@ import { Component, linkEvent } from 'inferno';
 import * as moment from 'moment';
 
 import { endpoint } from '../env';
-import { SearchParams, Results } from '../interfaces';
+import { SearchParams, Results, Torrent } from '../interfaces';
-import { convertCsvToJson, humanFileSize, magnetLink } from '../utils';
+import { humanFileSize, magnetLink } from '../utils';
 
 interface State {
   results: Results;
@@ -52,10 +52,12 @@ export class Search extends Component<any, State> {
     if (!!this.state.searchParams.q) {
       this.setState({ searching: true, results: { torrents: [] } });
       this.fetchData(this.state.searchParams)
-        .then(results => {
+        .then(torrents => {
-          if (!!results) {
+          if (!!torrents) {
             this.setState({
-              results: results
+              results: {
+                torrents: torrents
+              }
             });
           }
         }).catch(error => {
@@ -66,11 +68,10 @@ export class Search extends Component<any, State> {
     }
   }
 
-  fetchData(searchParams: SearchParams): Promise<Results> {
+  fetchData(searchParams: SearchParams): Promise<Array<Torrent>> {
     let q = encodeURI(searchParams.q);
     return fetch(`${endpoint}/service/search?q=${q}&page=${searchParams.page}`)
-      .then(data => data.text())
+      .then(data => data.json());
-      .then(csv => convertCsvToJson(csv));
   }
 
   render() {

server/ui/src/env.ts

@@ -1 +1 @@
-export const endpoint = "http://0.0.0.0:8080";
+export const endpoint = 'http://0.0.0.0:8080';

server/ui/src/index.html

@@ -4,8 +4,7 @@
 	<!-- Required meta tags -->
 	<meta charset="utf-8">
 	<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-
+	<link rel="shortcut icon" type="image/ico" href="/static/assets/favicon.ico" />
-	<link rel="shortcut icon" type="image/ico" href="/favicon.ico" />
 
 	<title>Torrents.csv</title>
 	<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">

server/ui/src/utils.ts

@@ -1,5 +1,3 @@
-import { Results, Torrent } from './interfaces';
-
 export function magnetLink(infohash: string, name: string): string {
   return `magnet:?xt=urn:btih:${infohash}&dn=${name}${trackerListToUrl(trackerList)})}`;
 }
@@ -35,33 +33,4 @@ export function humanFileSize(bytes, si): string {
   return bytes.toFixed(1) + ' ' + units[u];
 }
 
-export function convertCsvToJson(csv: string): Results {
-
-  let lines = csv.split("\n");
-  let torrents: Array<Torrent> = [];
-
-  for (let line of lines) {
-    let cols = line.split(";");
-
-    // Sometimes it gets back weird newlines
-    if (cols[0]) {
-      torrents.push({
-        infohash: cols[0],
-        name: cols[1],
-        size_bytes: Number(cols[2]),
-        created_unix: Number(cols[3]),
-        seeders: Number(cols[4]),
-        leechers: Number(cols[5]),
-        completed: Number(cols[6]),
-        scraped_date: Number(cols[7])
-      });
-    }
-  }
-
-  let result = {
-    torrents: torrents
-  }
-  return result;
-}
-
 export let repoUrl = 'https://gitlab.com/dessalines/torrents.csv';