diff --git a/.dockerignore b/.dockerignore index 8fef629..afa7185 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,4 +2,6 @@ server/ui/node_modules server/ui/dist server/service/target new_torrents_fetcher -.git \ No newline at end of file +.git +torrents.db +backups diff --git a/Dockerfile b/Dockerfile index cdec3dc..46f7d34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,16 @@ RUN yarn RUN yarn build FROM rust:1.33 as rust -COPY server/service /app/server/service +WORKDIR /app/server +# cache the build +RUN USER=root cargo new service WORKDIR /app/server/service +# copy over your manifests +COPY server/service/Cargo.toml server/service/Cargo.lock ./ +RUN cargo build --release +RUN rm src/*.rs +COPY server/service/src ./src +RUN rm -r ./target/release/.fingerprint/torrents-csv-* RUN cargo build --release FROM debian:jessie-slim as volume diff --git a/scripts/build_sqlite.sh b/scripts/build_sqlite.sh index 99af315..139c1dd 100755 --- a/scripts/build_sqlite.sh +++ b/scripts/build_sqlite.sh @@ -34,11 +34,16 @@ rm torrents_removed_quotes.csv # Cache torrent files if they exist if [ -f $torrent_files_json ]; then echo "Building files DB from $torrent_files_json ..." - jq -r 'to_entries[] | {hash: .key, val: .value[]} | [.hash, .val.i, .val.p, .val.l] | join(";")' $torrent_files_json > torrent_files_temp - # Removing those with too many ; - rg "^([^;]*;){3}[^;]+$" torrent_files_temp > torrent_files_temp_2 - mv torrent_files_temp_2 torrent_files_temp + # Old way, doesn't work with too much ram + # jq -r 'to_entries[] | {hash: .key, val: .value[]} | [.hash, .val.i, .val.p, .val.l] | join(";")' $torrent_files_json > torrent_files_temp + + # New way, credit to ogusismail : https://stackoverflow.com/a/55600294/1655478 + jq --stream -n -r 'foreach inputs as $pv ([[],[]]; if ($pv|length) == 2 then (.[0] |= if . == [] then . + [$pv[0][0],$pv[1]] else . + [$pv[1]] end) else [[],.[0]] end; if .[0] == [] and .[1] != [] then .[1] else empty end) | join(";")' $torrent_files_json > torrent_files_temp + +# Removing those with too many ; +rg "^([^;]*;){3}[^;]+$" torrent_files_temp > torrent_files_temp_2 +mv torrent_files_temp_2 torrent_files_temp sqlite3 -batch db_tmp< { {moment(torrent.created_unix * 1000).fromNow()} - - + - - +