Fixing docker build. #19

This commit is contained in:
Dessalines 2019-03-18 12:31:32 -07:00
parent e790b7af12
commit e1894a602c
7 changed files with 51 additions and 62 deletions

View File

@ -1,6 +1,5 @@
FROM node:10-jessie as node FROM node:10-jessie as node
#If encounter Invalid cross-device error -run on host 'echo N | sudo tee /sys/module/overlay/parameters/metacopy' #If encounter Invalid cross-device error -run on host 'echo N | sudo tee /sys/module/overlay/parameters/metacopy'
ARG TORRENTS_CSV_ENDPOINT
COPY server/ui /app/server/ui COPY server/ui /app/server/ui
RUN cd /app/server/ui && yarn && yarn build RUN cd /app/server/ui && yarn && yarn build
@ -9,22 +8,27 @@ COPY server/service /app/server/service
RUN cd /app/server/service && cargo build --release RUN cd /app/server/service && cargo build --release
FROM debian:jessie-slim as volume FROM debian:jessie-slim as volume
COPY torrents.csv /db/ COPY torrents.csv /db/
COPY torrent_files.json /db/
FROM debian:jessie-slim FROM debian:jessie-slim
RUN apt update && apt install -y sqlite3 curl
RUN apt update && apt install -y sqlite3 RUN curl -LO https://github.com/BurntSushi/ripgrep/releases/download/0.10.0/ripgrep_0.10.0_amd64.deb
RUN dpkg -i ripgrep_0.10.0_amd64.deb
RUN curl -LO https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
RUN chmod +x jq-linux64
RUN mv jq-linux64 /usr/bin/jq
COPY --from=node /app/server/ui/dist /app/dist COPY --from=node /app/server/ui/dist /app/dist
COPY --from=rust /app/server/service/target/release/torrents-csv-service /app/ COPY --from=rust /app/server/service/target/release/torrents-csv-service /app/
COPY --from=volume /db/torrents.csv /app/ COPY --from=volume /db/torrents.csv /app/
COPY --from=volume /db/torrent_files.json /app/
COPY scripts /app/scripts COPY scripts /app/scripts
RUN cd /app/scripts && . ./build_sqlite.sh WORKDIR /app/scripts/
RUN ["/bin/bash", "-c", "/app/scripts/build_sqlite.sh"]
EXPOSE 8080 EXPOSE 8080
WORKDIR /app/ WORKDIR /app/
ARG TORRENTS_CSV_DB_FILE ARG TORRENTS_CSV_DB_FILE
RUN sqlite3 ${TORRENTS_CSV_DB_FILE} 'select * from torrents limit 10' RUN sqlite3 ${TORRENTS_CSV_DB_FILE} 'select * from torrents limit 10'
CMD /app/torrents-csv-service

View File

@ -15,20 +15,21 @@ Its initially populated with a January 2017 backup of the pirate bay, and new to
To request more torrents, or add your own, go [here](https://gitlab.com/dessalines/torrents.csv/issues). To request more torrents, or add your own, go [here](https://gitlab.com/dessalines/torrents.csv/issues).
Made with [Rust](https://www.rust-lang.org), [ripgrep](https://github.com/BurntSushi/ripgrep), [Actix](https://actix.rs/), [Inferno](https://www.infernojs.org), [Typescript](https://www.typescriptlang.org/). Made with [Rust](https://www.rust-lang.org), [ripgrep](https://github.com/BurntSushi/ripgrep), [Actix](https://actix.rs/), [Inferno](https://www.infernojs.org), [Typescript](https://www.typescriptlang.org/).
## Webserver ## Webserver
`Torrents.csv` comes with a simple webserver. [Demo Server](https://torrents-csv.ml) `Torrents.csv` comes with a simple webserver. [Demo Server](https://torrents-csv.ml)
### Docker
### Requirements ```
git clone https://gitlab.com/dessalines/torrents.csv
cd torrents.csv
docker-compose up
```
### Local
#### Requirements
- [Rust](https://www.rust-lang.org/) - [Rust](https://www.rust-lang.org/)
- [Yarn](https://yarnpkg.com/en/) - [Yarn](https://yarnpkg.com/en/)
- [SQLite3](https://www.sqlite.org/index.html) - [SQLite3](https://www.sqlite.org/index.html)
- [jq command line JSON parser: Needs at least jq-1.6](https://stedolan.github.io/jq/) - [jq command line JSON parser: Needs at least jq-1.6](https://stedolan.github.io/jq/)
#### Running
### Running
``` ```
git clone https://gitlab.com/dessalines/torrents.csv git clone https://gitlab.com/dessalines/torrents.csv
cd torrents.csv/scripts cd torrents.csv/scripts
@ -36,7 +37,7 @@ cd torrents.csv/scripts
``` ```
and goto http://localhost:8080 and goto http://localhost:8080
If running on a different host, run `export TORRENTS_CSV_ENDPOINT=http://whatever.com` to change the hostname. If running on a different host, run `export TORRENTS_CSV_ENDPOINT=http://whatever.com` to change the hostname, or use a reverse proxy with nginx or apache2.
The torrent data is updated daily, and to do so, run, or place this in a crontab: The torrent data is updated daily, and to do so, run, or place this in a crontab:
@ -44,14 +45,6 @@ The torrent data is updated daily, and to do so, run, or place this in a crontab
This updates the repository, and rebuilds the sqlite cache necessary for searching. This updates the repository, and rebuilds the sqlite cache necessary for searching.
### Docker
```
git clone https://gitlab.com/dessalines/torrents.csv
cd torrents.csv
docker-compose up
```
## Command Line Searching ## Command Line Searching
### Requirements ### Requirements
@ -68,13 +61,10 @@ bleh season 1 (1993-)
size: 13GiB size: 13GiB
link: magnet:?xt=urn:btih:INFO_HASH_HERE link: magnet:?xt=urn:btih:INFO_HASH_HERE
``` ```
## Uploading / Adding Torrents from a Directory ## Uploading / Adding Torrents from a Directory
An *upload*, consists of making a pull request after running the `scan_torrents.sh` script, which adds torrents from a directory you choose to the `.csv` file, after checking that they aren't already there, and that they have seeders. It also adds their files to `torrent_files.json`. An *upload*, consists of making a pull request after running the `scan_torrents.sh` script, which adds torrents from a directory you choose to the `.csv` file, after checking that they aren't already there, and that they have seeders. It also adds their files to `torrent_files.json`.
### Requirements ### Requirements
- [Torrent-Tracker-Health Dessalines branch](https://github.com/dessalines/torrent-tracker-health) - [Torrent-Tracker-Health Dessalines branch](https://github.com/dessalines/torrent-tracker-health)
- `npm i -g dessalines/torrent-tracker-health` - `npm i -g dessalines/torrent-tracker-health`
- [jq command line JSON parser: Needs at least jq-1.6](https://stedolan.github.io/jq/) - [jq command line JSON parser: Needs at least jq-1.6](https://stedolan.github.io/jq/)
@ -82,7 +72,6 @@ An *upload*, consists of making a pull request after running the `scan_torrents.
- [Gnu Parallel](https://www.gnu.org/software/parallel/) - [Gnu Parallel](https://www.gnu.org/software/parallel/)
### Running ### Running
[Click here](https://gitlab.com/dessalines/torrents.csv/forks/new) to fork this repo. [Click here](https://gitlab.com/dessalines/torrents.csv/forks/new) to fork this repo.
```sh ```sh
git clone https://gitlab.com/[MY_USER]/torrents.csv git clone https://gitlab.com/[MY_USER]/torrents.csv
@ -93,19 +82,15 @@ git push
``` ```
Then [click here](https://gitlab.com/dessalines/torrents.csv/merge_requests/new) to do a pull/merge request to my branch. Then [click here](https://gitlab.com/dessalines/torrents.csv/merge_requests/new) to do a pull/merge request to my branch.
## Web scraping torrents ## Web scraping torrents
`Torrents.csv` has a `Rust` repository for scraping new and top torrents from some torrent sites in the `new_torrents_fetcher` folder. It currently scrapes skytorrents, magnetdl, and leetx. `Torrents.csv` has a `Rust` repository for scraping new and top torrents from some torrent sites in the `new_torrents_fetcher` folder. It currently scrapes skytorrents, magnetdl, and leetx.
### Requirements ### Requirements
- Rust - Rust
- [Cloudflare Scrape](https://github.com/Anorov/cloudflare-scrape) - [Cloudflare Scrape](https://github.com/Anorov/cloudflare-scrape)
- `sudo pip install cfscrape` - `sudo pip install cfscrape`
### Running ### Running
``` ```
git clone https://gitlab.com/dessalines/torrents.csv git clone https://gitlab.com/dessalines/torrents.csv
cd torrents.csv/scripts cd torrents.csv/scripts
@ -113,20 +98,17 @@ cd torrents.csv/scripts
``` ```
## API ## API
A JSON output of search results is available at: A JSON output of search results is available at:
http://localhost:8080/service/search?q=[QUERY]&size=[NUMBER_OF_RESULTS]&offset=[PAGE] http://localhost:8080/service/search?q=[QUERY]&size=[NUMBER_OF_RESULTS]&offset=[PAGE]
## How the torrents.csv file looks ## How the torrents.csv file looks
```sh ```sh
infohash;name;size_bytes;created_unix;seeders;leechers;completed;scraped_date infohash;name;size_bytes;created_unix;seeders;leechers;completed;scraped_date
# torrents here... # torrents here...
``` ```
## How the torrent_files.json looks ## How the torrent_files.json looks
``` ```
{ {
"012ae083ec82bf911f4fe503b9f6df1effaad9ac": [ "012ae083ec82bf911f4fe503b9f6df1effaad9ac": [

View File

@ -4,12 +4,9 @@ services:
torrents-csv: torrents-csv:
build: build:
context: . context: .
args:
TORRENTS_CSV_ENDPOINT: http://0.0.0.0:8080
command: /bin/sh -c /app/torrents-csv-service command: /bin/sh -c /app/torrents-csv-service
ports: ports:
- "8080:8080" - "8080:8080"
environment: environment:
TORRENTS_CSV_DB_FILE: /app/torrents.db TORRENTS_CSV_DB_FILE: /app/torrents.db
TORRENTS_CSV_FRONT_END_DIR: /app/dist TORRENTS_CSV_FRONT_END_DIR: /app/dist
TORRENTS_CSV_ENDPOINT: 0.0.0.0:8080

View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
csv_file="${TORRENTS_CSV_FILE:-../torrents.csv}" csv_file="../torrents.csv"
torrent_files_json="../torrent_files.json"
db_file="${TORRENTS_CSV_DB_FILE:-../torrents.db}" db_file="${TORRENTS_CSV_DB_FILE:-../torrents.db}"
torrent_files_json="`pwd`/../torrent_files.json"
echo "Creating temporary torrents.db file from $csv_file ..." echo "Creating temporary torrents.db file from $csv_file ..."
@ -40,35 +40,43 @@ if [ -f $torrent_files_json ]; then
rg "^([^;]*;){3}[^;]+$" torrent_files_temp > torrent_files_temp_2 rg "^([^;]*;){3}[^;]+$" torrent_files_temp > torrent_files_temp_2
mv torrent_files_temp_2 torrent_files_temp mv torrent_files_temp_2 torrent_files_temp
sqlite3 -batch db_tmp 2>/dev/null <<"EOF" sqlite3 -batch db_tmp<<EOF
drop table if exists files; create table files_tmp(
create table files( "infohash" TEXT,
"infohash" TEXT, "index_" INTEGER,
"index_" INTEGER, "path" TEXT,
"path" TEXT, "size_bytes" INTEGER
"size_bytes" INTEGER, );
"created_unix" INTEGER,
"seeders" INTEGER,
"leechers" INTEGER,
"completed" INTEGER,
"scraped_date" INTEGER);
.separator ";" .separator ";"
.import torrent_files_temp files .import torrent_files_temp files_tmp
-- Filling the extra columns -- Filling the extra columns
create table files(
"infohash" TEXT,
"index_" INTEGER,
"path" TEXT,
"size_bytes" INTEGER,
"created_unix" INTEGER,
"seeders" INTEGER,
"leechers" INTEGER,
"completed" INTEGER,
"scraped_date" INTEGER
);
insert into files insert into files
select files.infohash, select files_tmp.infohash,
files.index_, files_tmp.index_,
files.path, files_tmp.path,
files.size_bytes, files_tmp.size_bytes,
torrents.created_unix, torrents.created_unix,
torrents.seeders, torrents.seeders,
torrents.leechers, torrents.leechers,
torrents.completed, torrents.completed,
torrents.scraped_date torrents.scraped_date
from files from files_tmp
inner join torrents on files.infohash = torrents.infohash inner join torrents on files_tmp.infohash = torrents.infohash
order by torrents.seeders desc, files.size_bytes desc; order by torrents.seeders desc, files_tmp.size_bytes desc;
delete from files where seeders is null; delete from files where seeders is null;
drop table files_tmp;
EOF EOF
rm torrent_files_temp rm torrent_files_temp
fi fi

View File

@ -76,5 +76,5 @@ popd
rm -rf "$tmp_torrent_dir" rm -rf "$tmp_torrent_dir"
# Scan the torrent dir for new files, and add them # Scan the torrent dir for new files, and add them
node --max-old-space-size=2096 scan_torrent_files.js --dir "$torrents_dir" node --max-old-space-size=4096 scan_torrent_files.js --dir "$torrents_dir"

View File

@ -1,7 +1,6 @@
# Optionally use environment variables # Optionally use environment variables
# export TORRENTS_CSV_ENDPOINT=http://0.0.0.0:8080 # export TORRENTS_CSV_ENDPOINT=http://0.0.0.0:8080
# export TORRENTS_CSV_FILE=`pwd`/../../torrents.db
# export TORRENTS_CSV_DB_FILE=`pwd`/../../torrents.db # export TORRENTS_CSV_DB_FILE=`pwd`/../../torrents.db
# export TORRENTS_CSV_FRONT_END_DIR=`pwd`/../ui/dist # export TORRENTS_CSV_FRONT_END_DIR=`pwd`/../ui/dist

View File

@ -28,4 +28,3 @@ test/data/result.json
package-lock.json package-lock.json
*.orig *.orig
src/env.ts