diff --git a/.gitignore b/.gitignore index 5045b3c..2277566 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ run.out old_greps.sh +torrents.db diff --git a/scripts/build_sqlite.sh b/scripts/build_sqlite.sh new file mode 100755 index 0000000..a67272f --- /dev/null +++ b/scripts/build_sqlite.sh @@ -0,0 +1,24 @@ +# Remove quotes +sed 's/\"//g' ../torrents.csv > torrents_removed_quotes.csv + +rm ../torrents.db + +sqlite3 -batch ../torrents.db <<"EOF" +create table torrents( + "infohash" TEXT, + "name" TEXT, + "size_bytes" INTEGER, + "created_unix" INTEGER, + "seeders" INTEGER, + "leechers" INTEGER, + "completed" INTEGER, + "scraped_date" INTEGER +); +.separator ";" +.import torrents_removed_quotes.csv torrents +UPDATE torrents SET completed=NULL WHERE completed = ''; +create index name_index on torrents (name); +EOF +rm torrents_removed_quotes.csv + + diff --git a/scripts/sqlite_performance_test.sh b/scripts/sqlite_performance_test.sh new file mode 100755 index 0000000..9520c6f --- /dev/null +++ b/scripts/sqlite_performance_test.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +touch test.db +# Remove quotes +sed 's/\"//g' ../torrents.csv > torrents_removed_quotes.csv + + +sqlite3 -batch test.db <<"EOF" +create table torrents (); +.separator ";" +.import torrents_removed_quotes.csv torrents +create index name_index on torrents (name); +EOF + +echo "Ripgrep time for pearl jam" +time rg -i "pearl.*jam" ../torrents.csv > /dev/null + +echo "Sqlite3 time" +time sqlite3 test.db "select * from torrents where name like '%pearl%jam%fix%' limit 10" > /dev/null + +echo "Ripgrep time for avengers infinity" +time rg -i "avengers.*infinity" ../torrents.csv > /dev/null + +echo "Sqlite3 time" +time sqlite3 test.db "select * from torrents where name like '%avengers%infinity%' limit 10" > /dev/null + +echo "Ripgrep time for w" +time rg -i "w" ../torrents.csv > /dev/null + +echo "Sqlite3 time" +time sqlite3 test.db "select * from torrents where name like '%w%' limit 10" > /dev/null + +echo "Ripgrep time for prestige 2006" +time rg -i "prestige.*2006" ../torrents.csv > /dev/null + +echo "Sqlite3 time" +time sqlite3 test.db "select * from torrents where name like '%prestige%2006%' limit 10" > /dev/null + +rm test.db +rm torrents_removed_quotes.csv \ No newline at end of file diff --git a/scripts/webserver.sh b/scripts/webserver.sh index 9468a9f..88f4d35 100755 --- a/scripts/webserver.sh +++ b/scripts/webserver.sh @@ -4,6 +4,9 @@ # export TORRENTS_CSV_FILE=`pwd`/../../torrents.csv # export TORRENTS_CSV_FRONT_END_DIR=`pwd`/../ui/dist +# Build the sqlite db from the CSV +. build_sqlite.sh + # Build front end cd ../server/ui yarn diff --git a/server/service/Cargo.lock b/server/service/Cargo.lock index 5307a6f..b75bc61 100644 --- a/server/service/Cargo.lock +++ b/server/service/Cargo.lock @@ -130,16 +130,6 @@ dependencies = [ "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "atty" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", - "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "backtrace" version = "0.2.3" @@ -207,11 +197,6 @@ dependencies = [ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "bytecount" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "byteorder" version = "1.2.6" @@ -378,22 +363,6 @@ name = "encoding_index_tests" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "encoding_rs" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "encoding_rs_io" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "encoding_rs 0.8.10 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "error-chain" version = "0.1.12" @@ -472,94 +441,6 @@ dependencies = [ "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "globset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "grep" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "grep-cli 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "grep-matcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "grep-printer 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "grep-regex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "grep-searcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "grep-cli" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", - "same-file 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "grep-matcher" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "grep-printer" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", - "grep-matcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "grep-searcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", - "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "grep-regex" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "grep-matcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "grep-searcher" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bytecount 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "encoding_rs 0.8.10 (registry+https://github.com/rust-lang/crates.io-index)", - "encoding_rs_io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "grep-matcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "h2" version = "0.1.12" @@ -679,6 +560,15 @@ name = "libc" version = "0.2.43" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "libsqlite3-sys" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "linked-hash-map" version = "0.4.2" @@ -724,15 +614,6 @@ dependencies = [ "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "memmap" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "memoffset" version = "0.2.1" @@ -897,6 +778,11 @@ dependencies = [ "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "pkg-config" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "proc-macro2" version = "0.4.20" @@ -958,14 +844,6 @@ name = "redox_syscall" version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "redox_termios" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "regex" version = "1.0.5" @@ -1006,6 +884,17 @@ dependencies = [ "untrusted 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rusqlite" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libsqlite3-sys 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lru-cache 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rustc-demangle" version = "0.1.9" @@ -1029,14 +918,6 @@ name = "safemem" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "same-file" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "scopeguard" version = "0.3.3" @@ -1166,24 +1047,6 @@ dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "termcolor" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "termion" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "thread_local" version = "0.3.6" @@ -1375,9 +1238,10 @@ name = "torrents-csv-service" version = "0.1.0" dependencies = [ "actix-web 0.7.12 (registry+https://github.com/rust-lang/crates.io-index)", - "grep 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rusqlite 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1547,6 +1411,11 @@ dependencies = [ "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "vcpkg" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "version_check" version = "0.1.5" @@ -1586,28 +1455,11 @@ name = "winapi-i686-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "winapi-util" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "wincolor" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "winreg" version = "0.5.1" @@ -1640,7 +1492,6 @@ dependencies = [ "checksum actix_derive 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b9d1525ef45e5e021f0b93dace157dcab5d792acb4cc78f3213787d65e2bb92" "checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a" "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" -"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" "checksum backtrace 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "346d7644f0b5f9bc73082d3b2236b69a05fd35cce0cfa3724e184e6a5c9e2a2f" "checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" "checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0" @@ -1648,7 +1499,6 @@ dependencies = [ "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" "checksum brotli-sys 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" "checksum brotli2 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" -"checksum bytecount 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f861d9ce359f56dbcb6e0c2a1cb84e52ad732cadb57b806adeb3c7668caccbd8" "checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781" "checksum bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0ce55bd354b095246fc34caf4e9e242f5297a7fd938b090cadfea6eee614aa62" "checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16" @@ -1669,8 +1519,6 @@ dependencies = [ "checksum encoding-index-singlebyte 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a" "checksum encoding-index-tradchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" "checksum encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" -"checksum encoding_rs 0.8.10 (registry+https://github.com/rust-lang/crates.io-index)" = "065f4d0c826fdaef059ac45487169d918558e3cf86c9d89f6e81cf52369126e5" -"checksum encoding_rs_io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "098f6a0ab73a9ba256b71344dc82c6d7e252736ad9db7f4e35345f3a1f8713f5" "checksum error-chain 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faa976b4fd2e4c2b2f3f486874b19e61944d3de3de8b61c9fcf835d583871bcc" "checksum error-chain 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6930e04918388a9a2e41d518c25cf679ccafe26733fb4127dbf21993f2575d46" "checksum failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7efb22686e4a466b1ec1a15c2898f91fa9cb340452496dca654032de20ff95b9" @@ -1681,13 +1529,6 @@ dependencies = [ "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b" "checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" -"checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865" -"checksum grep 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d64cad6bc6831698fb55b64e7a973eec7ac9910d8b8b3755e523bf8fec7d0737" -"checksum grep-cli 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5933126a04a66e612378d38bee4ea76fc1a3e0086ae1122d5a2faed802d6c8c3" -"checksum grep-matcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "820946e0c314510779a8d86c5cd03240e0ae0993dabcdb98733a8f6a9001b607" -"checksum grep-printer 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bb877af207b46e3baacd41c5be5bbda79cf1d0da95583051a3e7e110cb27a49b" -"checksum grep-regex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8d7ebe41242224840865003a4bc4bf9dea411367cee03832e958ec4ef78c0c9b" -"checksum grep-searcher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6eb23805170ff0e96894a24847019500de11e9baaabe3dafed75b35a897636e1" "checksum h2 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "a27e7ed946e8335bdf9a191bc1b9b14a03ba822d013d2f58437f4fabcbd7fc2c" "checksum hostname 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "21ceb46a83a85e824ef93669c8b390009623863b5c195d1ba747292c0c72f94e" "checksum htmlescape 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" @@ -1703,13 +1544,13 @@ dependencies = [ "checksum lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca488b89a5657b0a2ecd45b95609b3e848cf1755da332a0da46e2b2b1cb371a7" "checksum lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ddba4c30a78328befecec92fc94970e53b3ae385827d28620f0f5bb2493081e0" "checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d" +"checksum libsqlite3-sys 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "742b695cbfb89e549dca6960a55e6802f67d352e33e97859ee46dee835211b0f" "checksum linked-hash-map 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7860ec297f7008ff7a1e3382d7f7e1dcd69efc94751a2284bafc3d013c2aa939" "checksum lock_api 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "775751a3e69bde4df9b38dd00a1b5d6ac13791e4223d4a0506577f0dd27cfb7a" "checksum log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fcce5fa49cc693c312001daf1d13411c4a5283796bac1084299ea3e567113f" "checksum lru-cache 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4d06ff7ff06f729ce5f4e227876cb88d10bc59cd4ae1e09fbb2bde15c850dc21" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4b3629fe9fdbff6daa6c33b90f7c08355c1aca05a3d01fa8063b822fcf185f3b" -"checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" "checksum mime 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "4b082692d3f6cf41b453af73839ce3dfc212c4411cbb2441dff80a716e38bd79" "checksum mime_guess 2.0.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "30de2e4613efcba1ec63d8133f344076952090c122992a903359be5a4f99c3ed" @@ -1728,6 +1569,7 @@ dependencies = [ "checksum phf_codegen 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "7d187f00cd98d5afbcd8898f6cf181743a449162aeb329dcd2f3849009e605ad" "checksum phf_generator 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "03dc191feb9b08b0dc1330d6549b795b9d81aec19efe6b4a45aec8d4caee0c4b" "checksum phf_shared 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "b539898d22d4273ded07f64a05737649dc69095d92cb87c7097ec68e3f150b93" +"checksum pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "676e8eb2b1b4c9043511a9b7bea0915320d7e502b0a079fb03f9635a5252b18c" "checksum proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)" = "3d7b7eaaa90b4a90a932a9ea6666c95a389e424eff347f0f793979289429feee" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" "checksum quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "dd636425967c33af890042c483632d33fa7a18f19ad1d7ea72e8998c6ef8dea5" @@ -1736,16 +1578,15 @@ dependencies = [ "checksum rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1961a422c4d189dfb50ffa9320bf1f2a9bd54ecb92792fb9477f99a1045f3372" "checksum rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0905b6b7079ec73b314d4c748701f6931eb79fd97c668caa3f1899b22b32c6db" "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" -"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2069749032ea3ec200ca51e4a31df41759190a88edca0d2d86ee8bedf7073341" "checksum regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "747ba3b235651f6e2f67dfa8bcdcd073ddb7c243cb21c442fc12395dfcac212d" "checksum resolv-conf 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c62bd95a41841efdf7fca2ae9951e64a8d8eae7e5da196d8ce489a2241491a92" "checksum ring 0.13.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe642b9dd1ba0038d78c4a3999d1ee56178b4d415c1e1fbaba83b06dce012f0" +"checksum rusqlite 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "39bae767eb27866f5c0be918635ae54af705bc09db11be2c43a3c6b361cf3462" "checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7153dd96dade874ab973e098cb62fcdbb89a03682e46b144fd09550998d4a4a7" "checksum safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8dca453248a96cb0749e36ccdfe2b0b4e54a61bfef89fb97ec621eb8e0a93dd9" -"checksum same-file 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "10f7794e2fda7f594866840e95f5c5962e886e228e68b6505885811a94dd728c" "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" @@ -1763,8 +1604,6 @@ dependencies = [ "checksum syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)" = "261ae9ecaa397c42b960649561949d69311f08eeaea86a65696e6e46517cf741" "checksum syn 0.15.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b10ee269228fb723234fce98e9aac0eaed2bd5f1ad2f6930e8d5b93f04445a1a" "checksum synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb9b7550d063ea184027c9b8c20ac167cd36d3e06b3a40bceb9d746dc1a7b7" -"checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f" -"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" "checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b" "checksum tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "6e93c78d23cc61aa245a8acd2c4a79c4d7fa7fb5c3ca90d5737029f043a84895" @@ -1796,6 +1635,7 @@ dependencies = [ "checksum url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2a321979c09843d272956e73700d12c4e7d3d92b2ee112b31548aef0d4efc5a6" "checksum utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd70f467df6810094968e2fce0ee1bd0e87157aceb026a8c083bcf5e25b9efe4" "checksum uuid 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dab5c5526c5caa3d106653401a267fed923e7046f35895ffcb5ca42db64942e6" +"checksum vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "def296d3eb3b12371b2c7d0e83bfe1403e4db2d7a0bba324a12b21c4ee13143d" "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum widestring 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7157704c2e12e3d2189c507b7482c52820a16dfa4465ba91add92f266667cadb" @@ -1803,9 +1643,7 @@ dependencies = [ "checksum winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -"checksum winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "afc5508759c5bf4285e61feb862b6083c8480aec864fa17a81fdec6f69b461ab" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" "checksum winreg 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a27a759395c1195c4cc5cda607ef6f8f6498f64e78f7900f5de0a127a424704a" "checksum winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7daf138b6b14196e3830a588acf1e86966c694d3e8fb026fb105b8b5dca07e6e" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" diff --git a/server/service/Cargo.toml b/server/service/Cargo.toml index e650af7..2ccea24 100644 --- a/server/service/Cargo.toml +++ b/server/service/Cargo.toml @@ -6,6 +6,7 @@ authors = ["Dessalines "] [dependencies] actix-web = "*" serde = "*" +serde_json = "*" serde_derive = "*" -grep = "*" +rusqlite = "*" time = "*" \ No newline at end of file diff --git a/server/service/src/main.rs b/server/service/src/main.rs index 11b9f33..b37f18c 100644 --- a/server/service/src/main.rs +++ b/server/service/src/main.rs @@ -1,27 +1,20 @@ extern crate actix_web; extern crate serde; +extern crate serde_json; #[macro_use] extern crate serde_derive; -extern crate grep; +extern crate rusqlite; extern crate time; use actix_web::{fs, fs::NamedFile, http, server, App, HttpRequest, HttpResponse, Query}; - -use std::error::Error; - -use grep::regex::RegexMatcherBuilder; -use grep::searcher::sinks::Lossy; -use grep::searcher::{BinaryDetection, SearcherBuilder}; use std::env; -use std::fs::File; + +use rusqlite::{Connection, NO_PARAMS}; fn main() { - - server::new(|| { App::new() .route("/service/search", http::Method::GET, search) - .resource("/favicon.ico", |r| r.f(favicon)) .resource("/", |r| r.f(index)) .handler( "/static", @@ -39,16 +32,12 @@ fn index(_req: &HttpRequest) -> Result { Ok(NamedFile::open(front_end_dir() + "/index.html")?) } -fn favicon(_req: &HttpRequest) -> Result { - Ok(NamedFile::open(front_end_dir() + "/favicon.ico")?) -} - fn front_end_dir() -> String { env::var("TORRENTS_CSV_FRONT_END_DIR").unwrap_or("../ui/dist".to_string()) } -fn torrents_csv_file() -> String { - env::var("TORRENTS_CSV_FILE").unwrap_or("../../torrents.csv".to_string()) +fn torrents_db_file() -> String { + env::var("TORRENTS_CSV_DB_FILE").unwrap_or("../../torrents.db".to_string()) } #[derive(Deserialize)] @@ -62,65 +51,73 @@ fn search(query: Query) -> HttpResponse { HttpResponse::Ok() .header("Access-Control-Allow-Origin", "*") .content_type("text/csv") - .body(ripgrep(query)) + .body(search_query(query)) } -fn ripgrep(query: Query) -> String { +fn search_query(query: Query) -> String { let page = query.page.unwrap_or(1); let size = query.size.unwrap_or(10); let offset = size * (page - 1); - let csv_file = File::open(torrents_csv_file()); println!("query = {} , page = {}, size = {}", query.q, page, size); - let results = search_file(csv_file.unwrap(), &query.q).unwrap(); + let results = sql_search(&query.q, size, offset); - let last: usize = if offset + size < results.len() { - offset + size - } else { - results.len() - }; - - results[offset..last].join("") + serde_json::to_string(&results).unwrap() } -fn search_file(file: File, query: &str) -> Result, Box> { - let pattern = query.replace(" ", ".*"); +#[derive(Debug, Serialize, Deserialize)] +struct Torrent { + infohash: String, + name: String, + size_bytes: isize, + created_unix: u32, + seeders: u32, + leechers: u32, + completed: Option, + scraped_date: u32, +} - let matcher = RegexMatcherBuilder::new() - .case_insensitive(true) - .build(&pattern) +fn sql_search(query: &str, size: usize, offset: usize) -> Vec { + let stmt_str = format!( + "select * from torrents where name like '%{}%' limit {} offset {}", + query.replace(" ", "%"), + size, + offset + ); + + let conn = Connection::open(torrents_db_file()).unwrap(); + + let mut stmt = conn.prepare(&stmt_str).unwrap(); + let torrent_iter = stmt + .query_map(NO_PARAMS, |row| Torrent { + infohash: row.get(0), + name: row.get(1), + size_bytes: row.get(2), + created_unix: row.get(3), + seeders: row.get(4), + leechers: row.get(5), + completed: row.get(6), + scraped_date: row.get(7), + }) .unwrap(); - let mut matches: Vec = vec![]; - - let mut searcher = SearcherBuilder::new() - .binary_detection(BinaryDetection::quit(b'\x00')) - // .line_number(false) - .build(); - - searcher.search_file( - &matcher, - &file, - Lossy(|_lnum, line| { - matches.push(line.to_string()); - Ok(true) - }), - )?; - - Ok(matches) + let mut torrents = Vec::new(); + for torrent in torrent_iter { + torrents.push(torrent.unwrap()); + } + torrents } #[cfg(test)] mod tests { - use std::fs::File; use time::PreciseTime; #[test] fn test() { let start = PreciseTime::now(); let results = - super::search_file(File::open(super::torrents_csv_file()).unwrap(), "sherlock").unwrap(); + super::sql_search("sherlock", 10, 0); assert!(results.len() > 2); let end = PreciseTime::now(); println!("Query took {} seconds.", start.to(end)); diff --git a/server/ui/src/favicon.ico b/server/ui/assets/favicon.ico similarity index 100% rename from server/ui/src/favicon.ico rename to server/ui/assets/favicon.ico diff --git a/server/ui/fuse.js b/server/ui/fuse.js index cc0f075..ff1e6d1 100644 --- a/server/ui/fuse.js +++ b/server/ui/fuse.js @@ -4,8 +4,7 @@ const { EnvPlugin, CSSPlugin, WebIndexPlugin, - QuantumPlugin, - CopyPlugin + QuantumPlugin } = require('fuse-box'); // const transformInferno = require('../../dist').default const transformInferno = require('ts-transform-inferno').default; @@ -44,12 +43,13 @@ Sparky.task('config', _ => { }); Sparky.task('clean', _ => Sparky.src('dist/').clean('dist/')); Sparky.task('env', _ => (isProduction = true)); -Sparky.task('dev', ['clean', 'config'], _ => { +Sparky.task('copy-assets', () => Sparky.src('assets/*.ico').dest('dist/')); +Sparky.task('dev', ['clean', 'config', 'copy-assets'], _ => { fuse.dev(); app.hmr().watch(); return fuse.run(); }); -Sparky.task('prod', ['clean', 'env', 'config'], _ => { +Sparky.task('prod', ['clean', 'env', 'config', 'copy-assets'], _ => { // fuse.dev({ reload: true }); // remove after demo return fuse.run(); }); diff --git a/server/ui/src/components/search.tsx b/server/ui/src/components/search.tsx index 3a0a4b7..6f68de0 100644 --- a/server/ui/src/components/search.tsx +++ b/server/ui/src/components/search.tsx @@ -2,8 +2,8 @@ import { Component, linkEvent } from 'inferno'; import * as moment from 'moment'; import { endpoint } from '../env'; -import { SearchParams, Results } from '../interfaces'; -import { convertCsvToJson, humanFileSize, magnetLink } from '../utils'; +import { SearchParams, Results, Torrent } from '../interfaces'; +import { humanFileSize, magnetLink } from '../utils'; interface State { results: Results; @@ -52,10 +52,12 @@ export class Search extends Component { if (!!this.state.searchParams.q) { this.setState({ searching: true, results: { torrents: [] } }); this.fetchData(this.state.searchParams) - .then(results => { - if (!!results) { + .then(torrents => { + if (!!torrents) { this.setState({ - results: results + results: { + torrents: torrents + } }); } }).catch(error => { @@ -66,11 +68,10 @@ export class Search extends Component { } } - fetchData(searchParams: SearchParams): Promise { + fetchData(searchParams: SearchParams): Promise> { let q = encodeURI(searchParams.q); return fetch(`${endpoint}/service/search?q=${q}&page=${searchParams.page}`) - .then(data => data.text()) - .then(csv => convertCsvToJson(csv)); + .then(data => data.json()); } render() { diff --git a/server/ui/src/env.ts b/server/ui/src/env.ts index aa7fcad..00834bb 100644 --- a/server/ui/src/env.ts +++ b/server/ui/src/env.ts @@ -1 +1 @@ -export const endpoint = "http://0.0.0.0:8080"; +export const endpoint = 'http://0.0.0.0:8080'; \ No newline at end of file diff --git a/server/ui/src/index.html b/server/ui/src/index.html index 5b33daf..165a0ab 100644 --- a/server/ui/src/index.html +++ b/server/ui/src/index.html @@ -4,8 +4,7 @@ - - + Torrents.csv diff --git a/server/ui/src/utils.ts b/server/ui/src/utils.ts index 4ca7fed..bcb538c 100644 --- a/server/ui/src/utils.ts +++ b/server/ui/src/utils.ts @@ -1,5 +1,3 @@ -import { Results, Torrent } from './interfaces'; - export function magnetLink(infohash: string, name: string): string { return `magnet:?xt=urn:btih:${infohash}&dn=${name}${trackerListToUrl(trackerList)})}`; } @@ -35,33 +33,4 @@ export function humanFileSize(bytes, si): string { return bytes.toFixed(1) + ' ' + units[u]; } -export function convertCsvToJson(csv: string): Results { - - let lines = csv.split("\n"); - let torrents: Array = []; - - for (let line of lines) { - let cols = line.split(";"); - - // Sometimes it gets back weird newlines - if (cols[0]) { - torrents.push({ - infohash: cols[0], - name: cols[1], - size_bytes: Number(cols[2]), - created_unix: Number(cols[3]), - seeders: Number(cols[4]), - leechers: Number(cols[5]), - completed: Number(cols[6]), - scraped_date: Number(cols[7]) - }); - } - } - - let result = { - torrents: torrents - } - return result; -} - export let repoUrl = 'https://gitlab.com/dessalines/torrents.csv';