Fixing docker build. #19

Dockerfile

@@ -1,6 +1,5 @@
 FROM node:10-jessie as node
 #If encounter Invalid cross-device error -run on host 'echo N | sudo tee /sys/module/overlay/parameters/metacopy'
-ARG TORRENTS_CSV_ENDPOINT
 COPY server/ui /app/server/ui
 RUN cd /app/server/ui && yarn && yarn build
 
@@ -9,22 +8,27 @@ COPY server/service /app/server/service
 RUN cd /app/server/service && cargo build --release
 
 FROM debian:jessie-slim as volume
-
 COPY torrents.csv  /db/
+COPY torrent_files.json /db/
 
 FROM debian:jessie-slim
-
+RUN apt update && apt install -y sqlite3 curl
-RUN apt update && apt install -y sqlite3
+RUN curl -LO https://github.com/BurntSushi/ripgrep/releases/download/0.10.0/ripgrep_0.10.0_amd64.deb
+RUN dpkg -i ripgrep_0.10.0_amd64.deb
+RUN curl -LO https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+RUN chmod +x jq-linux64
+RUN mv jq-linux64 /usr/bin/jq
 
 COPY --from=node /app/server/ui/dist /app/dist
 COPY --from=rust /app/server/service/target/release/torrents-csv-service /app/
 COPY --from=volume /db/torrents.csv /app/
+COPY --from=volume /db/torrent_files.json /app/
 COPY scripts /app/scripts
-RUN cd /app/scripts && . ./build_sqlite.sh
+WORKDIR /app/scripts/
+RUN ["/bin/bash", "-c", "/app/scripts/build_sqlite.sh"]
 
 EXPOSE 8080
 WORKDIR /app/
 ARG TORRENTS_CSV_DB_FILE
 RUN sqlite3 ${TORRENTS_CSV_DB_FILE} 'select * from torrents limit 10'
 
-CMD /app/torrents-csv-service

README.md

@@ -15,20 +15,21 @@ Its initially populated with a January 2017 backup of the pirate bay, and new to
 To request more torrents, or add your own, go [here](https://gitlab.com/dessalines/torrents.csv/issues).
 
 Made with [Rust](https://www.rust-lang.org), [ripgrep](https://github.com/BurntSushi/ripgrep), [Actix](https://actix.rs/), [Inferno](https://www.infernojs.org), [Typescript](https://www.typescriptlang.org/).
-
 ## Webserver
-
 `Torrents.csv` comes with a simple webserver. [Demo Server](https://torrents-csv.ml)
-
+### Docker
-### Requirements
+```
-
+git clone https://gitlab.com/dessalines/torrents.csv
+cd torrents.csv
+docker-compose up
+```
+### Local
+#### Requirements
 - [Rust](https://www.rust-lang.org/)
 - [Yarn](https://yarnpkg.com/en/)
 - [SQLite3](https://www.sqlite.org/index.html)
 - [jq command line JSON parser: Needs at least jq-1.6](https://stedolan.github.io/jq/)
-
+#### Running
-### Running
-
 ```
 git clone https://gitlab.com/dessalines/torrents.csv
 cd torrents.csv/scripts
@@ -36,7 +37,7 @@ cd torrents.csv/scripts
 ```
 and goto http://localhost:8080
 
-If running on a different host, run `export TORRENTS_CSV_ENDPOINT=http://whatever.com` to change the hostname.
+If running on a different host, run `export TORRENTS_CSV_ENDPOINT=http://whatever.com` to change the hostname, or use a reverse proxy with nginx or apache2.
 
 The torrent data is updated daily, and to do so, run, or place this in a crontab:
 
@@ -44,14 +45,6 @@ The torrent data is updated daily, and to do so, run, or place this in a crontab
 
 This updates the repository, and rebuilds the sqlite cache necessary for searching.
 
-### Docker
-
-```
-git clone https://gitlab.com/dessalines/torrents.csv
-cd torrents.csv
-docker-compose up
-```
-
 ## Command Line Searching
 
 ### Requirements
@@ -68,13 +61,10 @@ bleh season 1 (1993-)
 	size: 13GiB
 	link: magnet:?xt=urn:btih:INFO_HASH_HERE
 ```
-
 ## Uploading / Adding Torrents from a Directory
-
 An *upload*, consists of making a pull request after running the `scan_torrents.sh` script, which adds torrents from a directory you choose to the `.csv` file, after checking that they aren't already there, and that they have seeders. It also adds their files to `torrent_files.json`.
 
 ### Requirements
-
 - [Torrent-Tracker-Health Dessalines branch](https://github.com/dessalines/torrent-tracker-health)
   - `npm i -g dessalines/torrent-tracker-health`
 - [jq command line JSON parser: Needs at least jq-1.6](https://stedolan.github.io/jq/)
@@ -82,7 +72,6 @@ An *upload*, consists of making a pull request after running the `scan_torrents.
 - [Gnu Parallel](https://www.gnu.org/software/parallel/)
 
 ### Running
-
 [Click here](https://gitlab.com/dessalines/torrents.csv/forks/new) to fork this repo.
 ```sh
 git clone https://gitlab.com/[MY_USER]/torrents.csv
@@ -93,19 +82,15 @@ git push
 ```
 
 Then [click here](https://gitlab.com/dessalines/torrents.csv/merge_requests/new) to do a pull/merge request to my branch.
-
 ## Web scraping torrents
-
 `Torrents.csv` has a `Rust` repository for scraping new and top torrents from some torrent sites in the `new_torrents_fetcher` folder. It currently scrapes skytorrents, magnetdl, and leetx.
 
 ### Requirements
-
 - Rust
 - [Cloudflare Scrape](https://github.com/Anorov/cloudflare-scrape)
   - `sudo pip install cfscrape`
 
 ### Running
-
 ```
 git clone https://gitlab.com/dessalines/torrents.csv
 cd torrents.csv/scripts
@@ -113,20 +98,17 @@ cd torrents.csv/scripts
 ```
 
 ## API
-
 A JSON output of search results is available at:
 
 http://localhost:8080/service/search?q=[QUERY]&size=[NUMBER_OF_RESULTS]&offset=[PAGE]
 
 ## How the torrents.csv file looks
-
 ```sh
 infohash;name;size_bytes;created_unix;seeders;leechers;completed;scraped_date
 # torrents here...
 ```
 
 ## How the torrent_files.json looks
-
 ```
 {
   "012ae083ec82bf911f4fe503b9f6df1effaad9ac": [

docker-compose.yml

@@ -4,12 +4,9 @@ services:
   torrents-csv:
     build: 
       context: .
-      args:
-        TORRENTS_CSV_ENDPOINT: http://0.0.0.0:8080
     command: /bin/sh -c /app/torrents-csv-service
     ports:
     - "8080:8080"
     environment:
       TORRENTS_CSV_DB_FILE: /app/torrents.db 
       TORRENTS_CSV_FRONT_END_DIR: /app/dist
-      TORRENTS_CSV_ENDPOINT: 0.0.0.0:8080

scripts/build_sqlite.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
-csv_file="${TORRENTS_CSV_FILE:-../torrents.csv}"
+csv_file="../torrents.csv"
+torrent_files_json="../torrent_files.json"
 db_file="${TORRENTS_CSV_DB_FILE:-../torrents.db}"
-torrent_files_json="`pwd`/../torrent_files.json"
 
 echo "Creating temporary torrents.db file from $csv_file ..."
 
@@ -40,8 +40,17 @@ if [ -f $torrent_files_json ]; then
   rg "^([^;]*;){3}[^;]+$" torrent_files_temp > torrent_files_temp_2
   mv torrent_files_temp_2 torrent_files_temp
 
-  sqlite3 -batch db_tmp 2>/dev/null <<"EOF"
+sqlite3 -batch db_tmp<<EOF
-drop table if exists files;
+create table files_tmp(
+"infohash" TEXT,
+"index_" INTEGER,
+"path" TEXT,
+"size_bytes" INTEGER
+);
+.separator ";"
+.import torrent_files_temp files_tmp
+
+-- Filling the extra columns
 create table files(
 "infohash" TEXT,
 "index_" INTEGER,
@@ -51,24 +60,23 @@ create table files(
 "seeders" INTEGER,
 "leechers" INTEGER,
 "completed" INTEGER,
-  "scraped_date" INTEGER);
+"scraped_date" INTEGER
-.separator ";"
+);
-.import torrent_files_temp files
--- Filling the extra columns
 insert into files
-select files.infohash,
+select files_tmp.infohash,
-files.index_,
+files_tmp.index_,
-files.path,
+files_tmp.path,
-files.size_bytes,
+files_tmp.size_bytes,
 torrents.created_unix,
 torrents.seeders,
 torrents.leechers,
 torrents.completed,
 torrents.scraped_date
-from files
+from files_tmp
-inner join torrents on files.infohash = torrents.infohash
+inner join torrents on files_tmp.infohash = torrents.infohash
-order by torrents.seeders desc, files.size_bytes desc;
+order by torrents.seeders desc, files_tmp.size_bytes desc;
 delete from files where seeders is null;
+drop table files_tmp;
 EOF
   rm torrent_files_temp
 fi

scripts/scan_torrents.sh

@@ -76,5 +76,5 @@ popd
 rm -rf "$tmp_torrent_dir"
 
 # Scan the torrent dir for new files, and add them
-node --max-old-space-size=2096 scan_torrent_files.js --dir "$torrents_dir"
+node --max-old-space-size=4096 scan_torrent_files.js --dir "$torrents_dir"
 

scripts/webserver.sh

@@ -1,7 +1,6 @@
 # Optionally use environment variables
 
 # export TORRENTS_CSV_ENDPOINT=http://0.0.0.0:8080
-# export TORRENTS_CSV_FILE=`pwd`/../../torrents.db
 # export TORRENTS_CSV_DB_FILE=`pwd`/../../torrents.db
 # export TORRENTS_CSV_FRONT_END_DIR=`pwd`/../ui/dist
 

server/ui/.gitignore

@@ -28,4 +28,3 @@ test/data/result.json
 package-lock.json
 *.orig
 
-src/env.ts