diff --git a/Dockerfile b/Dockerfile index 4ccf20a..ae33b6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ FROM node:10-jessie as node #If encounter Invalid cross-device error -run on host 'echo N | sudo tee /sys/module/overlay/parameters/metacopy' -ARG TORRENTS_CSV_ENDPOINT COPY server/ui /app/server/ui RUN cd /app/server/ui && yarn && yarn build @@ -9,22 +8,27 @@ COPY server/service /app/server/service RUN cd /app/server/service && cargo build --release FROM debian:jessie-slim as volume - COPY torrents.csv /db/ +COPY torrent_files.json /db/ FROM debian:jessie-slim - -RUN apt update && apt install -y sqlite3 +RUN apt update && apt install -y sqlite3 curl +RUN curl -LO https://github.com/BurntSushi/ripgrep/releases/download/0.10.0/ripgrep_0.10.0_amd64.deb +RUN dpkg -i ripgrep_0.10.0_amd64.deb +RUN curl -LO https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 +RUN chmod +x jq-linux64 +RUN mv jq-linux64 /usr/bin/jq COPY --from=node /app/server/ui/dist /app/dist COPY --from=rust /app/server/service/target/release/torrents-csv-service /app/ COPY --from=volume /db/torrents.csv /app/ +COPY --from=volume /db/torrent_files.json /app/ COPY scripts /app/scripts -RUN cd /app/scripts && . ./build_sqlite.sh +WORKDIR /app/scripts/ +RUN ["/bin/bash", "-c", "/app/scripts/build_sqlite.sh"] EXPOSE 8080 WORKDIR /app/ ARG TORRENTS_CSV_DB_FILE RUN sqlite3 ${TORRENTS_CSV_DB_FILE} 'select * from torrents limit 10' -CMD /app/torrents-csv-service diff --git a/README.md b/README.md index 0ee7839..793c1bb 100644 --- a/README.md +++ b/README.md @@ -15,20 +15,21 @@ Its initially populated with a January 2017 backup of the pirate bay, and new to To request more torrents, or add your own, go [here](https://gitlab.com/dessalines/torrents.csv/issues). Made with [Rust](https://www.rust-lang.org), [ripgrep](https://github.com/BurntSushi/ripgrep), [Actix](https://actix.rs/), [Inferno](https://www.infernojs.org), [Typescript](https://www.typescriptlang.org/). - ## Webserver - `Torrents.csv` comes with a simple webserver. [Demo Server](https://torrents-csv.ml) - -### Requirements - +### Docker +``` +git clone https://gitlab.com/dessalines/torrents.csv +cd torrents.csv +docker-compose up +``` +### Local +#### Requirements - [Rust](https://www.rust-lang.org/) - [Yarn](https://yarnpkg.com/en/) - [SQLite3](https://www.sqlite.org/index.html) - [jq command line JSON parser: Needs at least jq-1.6](https://stedolan.github.io/jq/) - -### Running - +#### Running ``` git clone https://gitlab.com/dessalines/torrents.csv cd torrents.csv/scripts @@ -36,7 +37,7 @@ cd torrents.csv/scripts ``` and goto http://localhost:8080 -If running on a different host, run `export TORRENTS_CSV_ENDPOINT=http://whatever.com` to change the hostname. +If running on a different host, run `export TORRENTS_CSV_ENDPOINT=http://whatever.com` to change the hostname, or use a reverse proxy with nginx or apache2. The torrent data is updated daily, and to do so, run, or place this in a crontab: @@ -44,14 +45,6 @@ The torrent data is updated daily, and to do so, run, or place this in a crontab This updates the repository, and rebuilds the sqlite cache necessary for searching. -### Docker - -``` -git clone https://gitlab.com/dessalines/torrents.csv -cd torrents.csv -docker-compose up -``` - ## Command Line Searching ### Requirements @@ -68,13 +61,10 @@ bleh season 1 (1993-) size: 13GiB link: magnet:?xt=urn:btih:INFO_HASH_HERE ``` - ## Uploading / Adding Torrents from a Directory - An *upload*, consists of making a pull request after running the `scan_torrents.sh` script, which adds torrents from a directory you choose to the `.csv` file, after checking that they aren't already there, and that they have seeders. It also adds their files to `torrent_files.json`. ### Requirements - - [Torrent-Tracker-Health Dessalines branch](https://github.com/dessalines/torrent-tracker-health) - `npm i -g dessalines/torrent-tracker-health` - [jq command line JSON parser: Needs at least jq-1.6](https://stedolan.github.io/jq/) @@ -82,7 +72,6 @@ An *upload*, consists of making a pull request after running the `scan_torrents. - [Gnu Parallel](https://www.gnu.org/software/parallel/) ### Running - [Click here](https://gitlab.com/dessalines/torrents.csv/forks/new) to fork this repo. ```sh git clone https://gitlab.com/[MY_USER]/torrents.csv @@ -93,19 +82,15 @@ git push ``` Then [click here](https://gitlab.com/dessalines/torrents.csv/merge_requests/new) to do a pull/merge request to my branch. - ## Web scraping torrents - `Torrents.csv` has a `Rust` repository for scraping new and top torrents from some torrent sites in the `new_torrents_fetcher` folder. It currently scrapes skytorrents, magnetdl, and leetx. ### Requirements - - Rust - [Cloudflare Scrape](https://github.com/Anorov/cloudflare-scrape) - `sudo pip install cfscrape` ### Running - ``` git clone https://gitlab.com/dessalines/torrents.csv cd torrents.csv/scripts @@ -113,20 +98,17 @@ cd torrents.csv/scripts ``` ## API - A JSON output of search results is available at: http://localhost:8080/service/search?q=[QUERY]&size=[NUMBER_OF_RESULTS]&offset=[PAGE] ## How the torrents.csv file looks - ```sh infohash;name;size_bytes;created_unix;seeders;leechers;completed;scraped_date # torrents here... ``` ## How the torrent_files.json looks - ``` { "012ae083ec82bf911f4fe503b9f6df1effaad9ac": [ diff --git a/docker-compose.yml b/docker-compose.yml index e712cea..30f538e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,12 +4,9 @@ services: torrents-csv: build: context: . - args: - TORRENTS_CSV_ENDPOINT: http://0.0.0.0:8080 command: /bin/sh -c /app/torrents-csv-service ports: - "8080:8080" environment: TORRENTS_CSV_DB_FILE: /app/torrents.db TORRENTS_CSV_FRONT_END_DIR: /app/dist - TORRENTS_CSV_ENDPOINT: 0.0.0.0:8080 diff --git a/scripts/build_sqlite.sh b/scripts/build_sqlite.sh index 7a9bafa..99af315 100755 --- a/scripts/build_sqlite.sh +++ b/scripts/build_sqlite.sh @@ -1,7 +1,7 @@ #!/bin/bash -csv_file="${TORRENTS_CSV_FILE:-../torrents.csv}" +csv_file="../torrents.csv" +torrent_files_json="../torrent_files.json" db_file="${TORRENTS_CSV_DB_FILE:-../torrents.db}" -torrent_files_json="`pwd`/../torrent_files.json" echo "Creating temporary torrents.db file from $csv_file ..." @@ -40,35 +40,43 @@ if [ -f $torrent_files_json ]; then rg "^([^;]*;){3}[^;]+$" torrent_files_temp > torrent_files_temp_2 mv torrent_files_temp_2 torrent_files_temp - sqlite3 -batch db_tmp 2>/dev/null <<"EOF" -drop table if exists files; -create table files( - "infohash" TEXT, - "index_" INTEGER, - "path" TEXT, - "size_bytes" INTEGER, - "created_unix" INTEGER, - "seeders" INTEGER, - "leechers" INTEGER, - "completed" INTEGER, - "scraped_date" INTEGER); +sqlite3 -batch db_tmp<