Initial commit for file searching support

scripts/.gitignore

@@ -0,0 +1 @@
+node_modules

scripts/build_sqlite.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 csv_file="${TORRENTS_CSV_FILE:-../torrents.csv}"
 db_file="${TORRENTS_CSV_DB_FILE:-../torrents.db}"
+torrent_files_json="`pwd`/../torrent_files.json"
 
 echo "Creating temporary torrents.db file..."
 
@@ -10,9 +11,11 @@ sed 's/\"//g' $csv_file > torrents_removed_quotes.csv
 # Sort by seeders desc before insert
 sort --field-separator=';' --key=5 -nr -o torrents_removed_quotes.csv torrents_removed_quotes.csv
 
-rm $db_file
+rm db_tmp
+touch db_tmp
 
-sqlite3 -batch $db_file <<"EOF"
+sqlite3 -batch db_tmp <<"EOF"
+drop table if exists torrents;
 create table torrents(
   "infohash" TEXT,
   "name" TEXT,
@@ -27,7 +30,51 @@ create table torrents(
 .import torrents_removed_quotes.csv torrents
 UPDATE torrents SET completed=NULL WHERE completed = '';
 EOF
-
 rm torrents_removed_quotes.csv
 
+# Cache torrent files if they exist
+if [ -f $torrent_files_json ]; then
+  echo "Building files DB from $torrent_files_json ..."
+  jq -r 'to_entries[] | {hash: .key, val: .value[]} | [.hash, .val.i, .val.p, .val.l] | join(";")' $torrent_files_json > torrent_files_temp
+
+  # Removing those with too many ;
+  rg "^([^;]*;){3}[^;]+$" torrent_files_temp > torrent_files_temp_2
+  mv torrent_files_temp_2 torrent_files_temp
+
+  sqlite3 -batch db_tmp <<"EOF"
+drop table if exists files;
+create table files(
+  "infohash" TEXT,
+  "index_" INTEGER,
+  "path" TEXT,
+  "size_bytes" INTEGER,
+  "created_unix" INTEGER,
+  "seeders" INTEGER,
+  "leechers" INTEGER,
+  "completed" INTEGER,
+  "scraped_date" INTEGER);
+.separator ";"
+.import torrent_files_temp files
+-- Filling the extra columns
+insert into files
+select files.infohash,
+files.index_,
+files.path,
+files.size_bytes,
+torrents.created_unix,
+torrents.seeders,
+torrents.leechers,
+torrents.completed,
+torrents.scraped_date
+from files
+inner join torrents on files.infohash = torrents.infohash
+order by torrents.seeders desc, files.size_bytes desc;
+delete from files where seeders is null;
+EOF
+  rm torrent_files_temp
+fi
+
+mv db_tmp $db_file
+
+echo "Done."
 

scripts/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "torrent-file-reader",
+  "version": "1.1.6",
+  "main": "scan_torrent_files.js",
+  "engines": {
+    "node": ">= 8.0.0"
+  },
+  "dependencies": {
+    "minimist": "^1.2.0",
+    "read-torrent": "^1.3.1"
+  }
+}

scripts/scan_torrent_files.js

@@ -0,0 +1,73 @@
+// jq -r 'to_entries[] | {hash: .key, val: .value[]} | {hash: .hash, i: .val.i, p: .val.p, l: .val.l}' torrent_files.json
+// jq -r 'to_entries[] | {hash: .key, val: .value[]} | [.hash, .val.i, .val.p, .val.l] | join(";")' torrent_files.json
+var fs = require('fs'),
+  path = require('path'),
+  readTorrent = require('read-torrent'),
+  argv = require('minimist')(process.argv.slice(2));
+
+var torrentFiles = {};
+var jsonFile = '../torrent_files.json';
+main();
+
+async function main() {
+  await fillTorrentFiles();
+  await scanFolder();
+  writeFile();
+}
+
+
+async function fillTorrentFiles() {
+  if (fs.existsSync(jsonFile)) {
+    var fileContents = await fs.promises.readFile(jsonFile, 'utf8');
+    torrentFiles = JSON.parse(fileContents);
+  }
+}
+
+async function scanFolder() {
+  console.log('Scanning dir: ' + argv.dir + '...');
+  var files = fs.readdirSync(argv.dir).filter(f => {
+    var f = f.split('.');
+    var ext = f[1];
+    var hash = f[0];
+    return (ext == 'torrent' && !Object.keys(torrentFiles).includes(hash));
+  });
+  for (const file of files) {
+    var fullPath = argv.dir + '/' + file;
+    console.log(`Scanning File ${fullPath}`);
+    var torrent = await read(fullPath).catch(e => console.log('Read error'));
+    torrentFiles = { ...torrentFiles, ...torrent }; // concat them
+  };
+  console.log('Done scanning.')
+}
+
+function writeFile() {
+  torrentFiles = Object.keys(torrentFiles).sort().reduce((r, k) => (r[k] = torrentFiles[k], r), {});
+  fs.writeFileSync(jsonFile, JSON.stringify(torrentFiles));
+  console.log(`${jsonFile} written.`);
+}
+
+function read(uri, options) {
+  return new Promise((resolve, reject) => {
+    readTorrent(uri, (err, info) => {
+      if (!err) {
+        // Removing some extra fields from files
+        if (info.files) {
+          info.files.forEach((f, i) => {
+            f.i = i;
+            f.p = f.path;
+            f.l = f.length;
+            delete f.name;
+            delete f.offset;
+            delete f.path;
+            delete f.length;
+          });
+        }
+
+        resolve({ [info.infoHash]: info.files });
+      } else {
+        console.error('Error in read-torrent: ' + err.message + ' for torrent uri: ' + uri);
+        reject(err);
+      }
+    });
+  });
+}

scripts/scan_torrent_files.sh

@@ -0,0 +1,10 @@
+torrent_files_json="`pwd`/../torrent_files.json"
+
+# Scan the torrent_files.json for already scanned torrents
+
+# Disjoint the ones there with the ones in your torrent scan dir
+
+# Run the js read-torrent in that dir, and update the torrent_files.json with the new ones
+
+node scan_torrent_files.js --dir "$1"
+

scripts/yarn.lock

@@ -0,0 +1,405 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+ajv@^6.5.5:
+  version "6.7.0"
+  resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.7.0.tgz#e3ce7bb372d6577bb1839f1dfdfcbf5ad2948d96"
+  integrity sha512-RZXPviBTtfmtka9n9sy1N5M5b82CbxWIR6HIis4s3WQTXDJamc/0gpCWNGz6EWdWp4DOfjzJfhz/AS9zVPjjWg==
+  dependencies:
+    fast-deep-equal "^2.0.1"
+    fast-json-stable-stringify "^2.0.0"
+    json-schema-traverse "^0.4.1"
+    uri-js "^4.2.2"
+
+asn1@~0.2.3:
+  version "0.2.4"
+  resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136"
+  integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==
+  dependencies:
+    safer-buffer "~2.1.0"
+
+assert-plus@1.0.0, assert-plus@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525"
+  integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=
+
+asynckit@^0.4.0:
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
+  integrity sha1-x57Zf380y48robyXkLzDZkdLS3k=
+
+aws-sign2@~0.7.0:
+  version "0.7.0"
+  resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
+  integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=
+
+aws4@^1.8.0:
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f"
+  integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==
+
+bcrypt-pbkdf@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e"
+  integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=
+  dependencies:
+    tweetnacl "^0.14.3"
+
+bencode@^0.7.0:
+  version "0.7.0"
+  resolved "https://registry.yarnpkg.com/bencode/-/bencode-0.7.0.tgz#811ed647c0118945e41bb4bbbdea9a2c78a17083"
+  integrity sha1-gR7WR8ARiUXkG7S7veqaLHihcIM=
+
+caseless@~0.12.0:
+  version "0.12.0"
+  resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
+  integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=
+
+combined-stream@^1.0.6, combined-stream@~1.0.6:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.7.tgz#2d1d24317afb8abe95d6d2c0b07b57813539d828"
+  integrity sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==
+  dependencies:
+    delayed-stream "~1.0.0"
+
+core-util-is@1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
+  integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
+
+dashdash@^1.12.0:
+  version "1.14.1"
+  resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0"
+  integrity sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=
+  dependencies:
+    assert-plus "^1.0.0"
+
+delayed-stream@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
+  integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk=
+
+ecc-jsbn@~0.1.1:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
+  integrity sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=
+  dependencies:
+    jsbn "~0.1.0"
+    safer-buffer "^2.1.0"
+
+extend@~3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa"
+  integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==
+
+extsprintf@1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05"
+  integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=
+
+extsprintf@^1.2.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f"
+  integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8=
+
+fast-deep-equal@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz#7b05218ddf9667bf7f370bf7fdb2cb15fdd0aa49"
+  integrity sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=
+
+fast-json-stable-stringify@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2"
+  integrity sha1-1RQsDK7msRifh9OnYREGT4bIu/I=
+
+flatten@0.0.1:
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/flatten/-/flatten-0.0.1.tgz#554440766da0a0d603999f433453f6c2fc6a75c1"
+  integrity sha1-VURAdm2goNYDmZ9DNFP2wvxqdcE=
+
+forever-agent@~0.6.1:
+  version "0.6.1"
+  resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
+  integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=
+
+form-data@~2.3.2:
+  version "2.3.3"
+  resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6"
+  integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==
+  dependencies:
+    asynckit "^0.4.0"
+    combined-stream "^1.0.6"
+    mime-types "^2.1.12"
+
+getpass@^0.1.1:
+  version "0.1.7"
+  resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa"
+  integrity sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=
+  dependencies:
+    assert-plus "^1.0.0"
+
+har-schema@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
+  integrity sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=
+
+har-validator@~5.1.0:
+  version "5.1.3"
+  resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.3.tgz#1ef89ebd3e4996557675eed9893110dc350fa080"
+  integrity sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==
+  dependencies:
+    ajv "^6.5.5"
+    har-schema "^2.0.0"
+
+http-signature@~1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
+  integrity sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=
+  dependencies:
+    assert-plus "^1.0.0"
+    jsprim "^1.2.2"
+    sshpk "^1.7.0"
+
+is-typedarray@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
+  integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=
+
+isstream@~0.1.2:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a"
+  integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=
+
+jsbn@~0.1.0:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513"
+  integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM=
+
+json-schema-traverse@^0.4.1:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660"
+  integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==
+
+json-schema@0.2.3:
+  version "0.2.3"
+  resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13"
+  integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=
+
+json-stringify-safe@~5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
+  integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=
+
+jsprim@^1.2.2:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2"
+  integrity sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=
+  dependencies:
+    assert-plus "1.0.0"
+    extsprintf "1.3.0"
+    json-schema "0.2.3"
+    verror "1.10.0"
+
+magnet-uri@^4.0.0:
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/magnet-uri/-/magnet-uri-4.2.3.tgz#79cc6d65a00bb5b7ef5c25ae60ebbb5d9a7681a8"
+  integrity sha1-ecxtZaALtbfvXCWuYOu7XZp2gag=
+  dependencies:
+    flatten "0.0.1"
+    thirty-two "^0.0.2"
+    xtend "^4.0.0"
+
+magnet-uri@~2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/magnet-uri/-/magnet-uri-2.0.1.tgz#d331d3dfcd3836565ade0fc3ca315e39217bb209"
+  integrity sha1-0zHT3804NlZa3g/DyjFeOSF7sgk=
+  dependencies:
+    thirty-two "~0.0.1"
+
+mime-db@~1.37.0:
+  version "1.37.0"
+  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.37.0.tgz#0b6a0ce6fdbe9576e25f1f2d2fde8830dc0ad0d8"
+  integrity sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==
+
+mime-types@^2.1.12, mime-types@~2.1.19:
+  version "2.1.21"
+  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.21.tgz#28995aa1ecb770742fe6ae7e58f9181c744b3f96"
+  integrity sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==
+  dependencies:
+    mime-db "~1.37.0"
+
+minimist@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284"
+  integrity sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=
+
+oauth-sign@~0.9.0:
+  version "0.9.0"
+  resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455"
+  integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==
+
+parse-torrent-file@^2.0.0:
+  version "2.1.4"
+  resolved "https://registry.yarnpkg.com/parse-torrent-file/-/parse-torrent-file-2.1.4.tgz#32d4b6afde631420e5f415919a222b774b575707"
+  integrity sha1-MtS2r95jFCDl9BWRmiIrd0tXVwc=
+  dependencies:
+    bencode "^0.7.0"
+    simple-sha1 "^2.0.0"
+
+parse-torrent@^4.0.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/parse-torrent/-/parse-torrent-4.1.0.tgz#a814bd8505e8b58e88eb8ff3e2daff5d19a711b7"
+  integrity sha1-qBS9hQXotY6I64/z4tr/XRmnEbc=
+  dependencies:
+    magnet-uri "^4.0.0"
+    parse-torrent-file "^2.0.0"
+
+performance-now@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
+  integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
+
+psl@^1.1.24:
+  version "1.1.31"
+  resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.31.tgz#e9aa86d0101b5b105cbe93ac6b784cd547276184"
+  integrity sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==
+
+punycode@^1.4.1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"
+  integrity sha1-wNWmOycYgArY4esPpSachN1BhF4=
+
+punycode@^2.1.0:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
+  integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==
+
+qs@~6.5.2:
+  version "6.5.2"
+  resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
+  integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==
+
+read-torrent@^1.3.1:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/read-torrent/-/read-torrent-1.3.1.tgz#de7e721af6f7d94e0e7a1f23a94b71edc073b1ae"
+  integrity sha512-TzPdVpK3xEnvsS1yCy94CiOmfzG2/MNuZjpZi64+HJxb9fVZD4nIfFFGZ9T2N/dgwOFumfacw3xCD6rXtgwn2g==
+  dependencies:
+    magnet-uri "~2.0.0"
+    parse-torrent "^4.0.0"
+    request "^2.83.0"
+    xtend "^4.0.0"
+
+request@^2.83.0:
+  version "2.88.0"
+  resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef"
+  integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==
+  dependencies:
+    aws-sign2 "~0.7.0"
+    aws4 "^1.8.0"
+    caseless "~0.12.0"
+    combined-stream "~1.0.6"
+    extend "~3.0.2"
+    forever-agent "~0.6.1"
+    form-data "~2.3.2"
+    har-validator "~5.1.0"
+    http-signature "~1.2.0"
+    is-typedarray "~1.0.0"
+    isstream "~0.1.2"
+    json-stringify-safe "~5.0.1"
+    mime-types "~2.1.19"
+    oauth-sign "~0.9.0"
+    performance-now "^2.1.0"
+    qs "~6.5.2"
+    safe-buffer "^5.1.2"
+    tough-cookie "~2.4.3"
+    tunnel-agent "^0.6.0"
+    uuid "^3.3.2"
+
+rusha@^0.8.1:
+  version "0.8.13"
+  resolved "https://registry.yarnpkg.com/rusha/-/rusha-0.8.13.tgz#9a084e7b860b17bff3015b92c67a6a336191513a"
+  integrity sha1-mghOe4YLF7/zAVuSxnpqM2GRUTo=
+
+safe-buffer@^5.0.1, safe-buffer@^5.1.2:
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
+  integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
+
+safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
+  integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==
+
+simple-sha1@^2.0.0:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/simple-sha1/-/simple-sha1-2.1.1.tgz#93f3b7f2e8dfdc056c32793e5d47b58d311b140d"
+  integrity sha512-pFMPd+I/lQkpf4wFUeS/sED5IqdIG1lUlrQviBMV4u4mz8BRAcB5fvUx5Ckfg3kBigEglAjHg7E9k/yy2KlCqA==
+  dependencies:
+    rusha "^0.8.1"
+
+sshpk@^1.7.0:
+  version "1.16.1"
+  resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877"
+  integrity sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==
+  dependencies:
+    asn1 "~0.2.3"
+    assert-plus "^1.0.0"
+    bcrypt-pbkdf "^1.0.0"
+    dashdash "^1.12.0"
+    ecc-jsbn "~0.1.1"
+    getpass "^0.1.1"
+    jsbn "~0.1.0"
+    safer-buffer "^2.0.2"
+    tweetnacl "~0.14.0"
+
+thirty-two@^0.0.2, thirty-two@~0.0.1:
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/thirty-two/-/thirty-two-0.0.2.tgz#4253e29d8cb058f0480267c5698c0e4927e54b6a"
+  integrity sha1-QlPinYywWPBIAmfFaYwOSSflS2o=
+
+tough-cookie@~2.4.3:
+  version "2.4.3"
+  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781"
+  integrity sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==
+  dependencies:
+    psl "^1.1.24"
+    punycode "^1.4.1"
+
+tunnel-agent@^0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"
+  integrity sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=
+  dependencies:
+    safe-buffer "^5.0.1"
+
+tweetnacl@^0.14.3, tweetnacl@~0.14.0:
+  version "0.14.5"
+  resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"
+  integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=
+
+uri-js@^4.2.2:
+  version "4.2.2"
+  resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.2.2.tgz#94c540e1ff772956e2299507c010aea6c8838eb0"
+  integrity sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==
+  dependencies:
+    punycode "^2.1.0"
+
+uuid@^3.3.2:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"
+  integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==
+
+verror@1.10.0:
+  version "1.10.0"
+  resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400"
+  integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=
+  dependencies:
+    assert-plus "^1.0.0"
+    core-util-is "1.0.2"
+    extsprintf "^1.2.0"
+
+xtend@^4.0.0:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"
+  integrity sha1-pcbVMr5lbiPbgg77lDofBJmNY68=

server/service/src/main.rs

@@ -8,6 +8,7 @@ extern crate time;
 
 use actix_web::{fs, fs::NamedFile, http, server, App, HttpRequest, HttpResponse, Query};
 use std::env;
+use std::ops::Deref;
 
 use rusqlite::{Connection, NO_PARAMS};
 
@@ -45,6 +46,7 @@ struct SearchQuery {
   q: String,
   page: Option<usize>,
   size: Option<usize>,
+  type_: Option<String>
 }
 
 fn search(query: Query<SearchQuery>) -> HttpResponse {
@@ -57,13 +59,18 @@ fn search(query: Query<SearchQuery>) -> HttpResponse {
 fn search_query(query: Query<SearchQuery>) -> String {
   let page = query.page.unwrap_or(1);
   let size = query.size.unwrap_or(10);
+  let type_ = query.type_.as_ref().map_or("torrent", String::deref);
   let offset = size * (page - 1);
 
-  println!("query = {} , page = {}, size = {}", query.q, page, size);
+  println!("query = {}, type = {}, page = {}, size = {}", query.q, type_, page, size);
 
-  let results = sql_search(&query.q, size, offset);
+  if type_ == "file" {
-
+    let results = torrent_file_search(&query.q, size, offset);
-  serde_json::to_string(&results).unwrap()
+    serde_json::to_string(&results).unwrap()
+  } else {
+    let results = torrent_search(&query.q, size, offset);
+    serde_json::to_string(&results).unwrap()
+  }
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -78,7 +85,7 @@ struct Torrent {
   scraped_date: u32,
 }
 
-fn sql_search(query: &str, size: usize, offset: usize) -> Vec<Torrent> {
+fn torrent_search(query: &str, size: usize, offset: usize) -> Vec<Torrent> {
   let stmt_str = format!(
     "select * from torrents where name like '%{}%' limit {} offset {}",
     query.replace(" ", "%").replace("\'","''"),
@@ -109,6 +116,52 @@ fn sql_search(query: &str, size: usize, offset: usize) -> Vec<Torrent> {
   torrents
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+struct File {
+  infohash: String,
+  index_: u32,
+  path: String,
+  size_bytes: isize,
+  created_unix: u32,
+  seeders: u32,
+  leechers: u32,
+  completed: Option<u32>,
+  scraped_date: u32,
+}
+
+fn torrent_file_search(query: &str, size: usize, offset: usize) -> Vec<File> {
+  let stmt_str = format!(
+    "select * from files where path like '%{}%' limit {} offset {}",
+    query.replace(" ", "%").replace("\'","''"),
+    size,
+    offset
+  );
+
+  let conn = Connection::open(torrents_db_file()).unwrap();
+
+  let mut stmt = conn.prepare(&stmt_str).unwrap();
+  let file_iter = stmt
+    .query_map(NO_PARAMS, |row| File {
+      infohash: row.get(0),
+      index_: row.get(1),
+      path: row.get(2),
+      size_bytes: row.get(3),
+      created_unix: row.get(4),
+      seeders: row.get(5),
+      leechers: row.get(6),
+      completed: row.get(7),
+      scraped_date: row.get(8),
+    })
+    .unwrap();
+
+  let mut files = Vec::new();
+  for file in file_iter {
+    files.push(file.unwrap());
+  }
+  files
+}
+
+
 #[cfg(test)]
 mod tests {
   use time::PreciseTime;
@@ -117,7 +170,7 @@ mod tests {
   fn test() {
     let start = PreciseTime::now();
     let results =
-      super::sql_search("sherlock", 10, 0);
+      super::torrent_search("sherlock", 10, 0);
     assert!(results.len() > 2);
     let end = PreciseTime::now();
     println!("Query took {} seconds.", start.to(end));

server/ui/src/Main.css

@@ -58,3 +58,11 @@ a::-moz-focus-inner {
   box-shadow: none !important;
   outline: none !important;
 }
+
+.path_column {
+  white-space: -o-pre-wrap;
+  word-wrap: break-word;
+  white-space: pre-wrap;
+  white-space: -moz-pre-wrap;
+  white-space: -pre-wrap;
+}

server/ui/src/components/navbar.tsx

@@ -12,7 +12,8 @@ export class Navbar extends Component<any, State> {
   state: State = {
     searchParams: {
       page: 1,
-      q: ""
+      q: "",
+      type_: "torrent"
     }
   }
 
@@ -49,9 +50,18 @@ export class Navbar extends Component<any, State> {
           <input class="form-control border-0 no-outline" type="search" placeholder="Search..." aria-label="Search..." required
                  value={this.state.searchParams.q}
                  onInput={linkEvent(this, this.searchChange)}></input>
-          <button class="btn btn-light bg-white border-0 rounded-right no-outline" type="submit">
+          <div class="input-group-append">
-            <i className="fas fa-fw fa-search"></i>
+            <select value={this.state.searchParams.type_} 
-          </button>
+              onInput={linkEvent(this, this.searchTypeChange)}
+              class="custom-select rounded-0">
+              <option disabled>Type</option>
+              <option value="torrent">Torrent</option>
+              <option value="file">File</option>
+            </select>
+            <button class="btn btn-light bg-white border-0 rounded-right no-outline" type="submit">
+              <i className="fas fa-fw fa-search"></i>
+            </button>
+          </div>
         </div>
       </form>
     );
@@ -59,23 +69,33 @@ export class Navbar extends Component<any, State> {
 
   search(i: Navbar, event) {
     event.preventDefault();
-    i.context.router.history.push(`/search/${i.state.searchParams.q}/${i.state.searchParams.page}`);
+    i.context.router.history.push(`/search/${i.state.searchParams.type_}/${i.state.searchParams.q}/${i.state.searchParams.page}`);
   }
 
   searchChange(i: Navbar, event) {
     let searchParams: SearchParams = {
       q: event.target.value,
-      page: 1
+      page: 1,
+      type_: i.state.searchParams.type_
     }
     i.setState({ searchParams: searchParams });
   }
 
+  searchTypeChange(i: Navbar, event) {
+    let searchParams: SearchParams = {
+      q: i.state.searchParams.q,
+      page: 1,
+      type_: event.target.value
+    }
+    i.setState({ searchParams: searchParams });
+  }
   fillSearchField() {
     let splitPath: Array<string> = this.context.router.route.location.pathname.split("/");
-    if (splitPath.length == 4 && splitPath[1] == 'search')
+    if (splitPath.length == 5 && splitPath[1] == 'search')
       this.state.searchParams = {
-        page: Number(splitPath[3]),
+        page: Number(splitPath[4]),
-        q: splitPath[2]
+        q: splitPath[3],
+        type_: splitPath[2]
       };
   }
 }

server/ui/src/components/search.tsx

@@ -3,7 +3,7 @@ import * as moment from 'moment';
 
 import { endpoint } from '../env';
 import { SearchParams, Results, Torrent } from '../interfaces';
-import { humanFileSize, magnetLink } from '../utils';
+import { humanFileSize, magnetLink, getFileName } from '../utils';
 
 interface State {
   results: Results;
@@ -19,7 +19,8 @@ export class Search extends Component<any, State> {
     },
     searchParams: {
       q: "",
-      page: 1
+      page: 1,
+      type_: 'torrent'
     },
     searching: false
   };
@@ -31,7 +32,8 @@ export class Search extends Component<any, State> {
   componentDidMount() {
     this.state.searchParams = {
       page: Number(this.props.match.params.page),
-      q: this.props.match.params.q
+      q: this.props.match.params.q,
+      type_: this.props.match.params.type_
     }
     this.search();
   }
@@ -41,7 +43,8 @@ export class Search extends Component<any, State> {
     if (lastProps.match && lastProps.match.params !== this.props.match.params) {
       this.state.searchParams = {
         page: Number(this.props.match.params.page),
-        q: this.props.match.params.q
+        q: this.props.match.params.q,
+        type_: this.props.match.params.type_
       }
       this.search();
     }
@@ -70,7 +73,7 @@ export class Search extends Component<any, State> {
 
   fetchData(searchParams: SearchParams): Promise<Array<Torrent>> {
     let q = encodeURI(searchParams.q);
-    return fetch(`${endpoint}/service/search?q=${q}&page=${searchParams.page}`)
+    return fetch(`${endpoint}/service/search?q=${q}&page=${searchParams.page}&type_=${searchParams.type_}`)
       .then(data => data.json());
   }
 
@@ -80,7 +83,8 @@ export class Search extends Component<any, State> {
         {
           this.state.searching ?
             this.spinner() : this.state.results.torrents[0] ?
-              this.table() : this.noResults()
+              this.torrentsTable()
+                : this.noResults()
         }
       </div>
     );
@@ -102,7 +106,7 @@ export class Search extends Component<any, State> {
     )
   }
 
-  table() {
+  torrentsTable() {
     return (
       <div>
         <table class="table table-fixed table-hover table-sm table-striped table-hover-purple table-padding">
@@ -119,8 +123,12 @@ export class Search extends Component<any, State> {
           <tbody>
             {this.state.results.torrents.map(torrent => (
               <tr>
-                <td class="search-name-cell">{torrent.name}</td>
+                { !torrent.name ? (
-                <td class="text-right text-muted">{humanFileSize(torrent.size_bytes, true)}</td>
+                    <td class="path_column">{getFileName(torrent.path)}</td>
+                ) : (
+                    <td class="search-name-cell">{torrent.name}</td>
+                )}
+                  <td class="text-right text-muted">{humanFileSize(torrent.size_bytes, true)}</td>
                 <td class="text-right text-success"><i class="fas fa-fw fa-arrow-up d-none d-sm-inline mr-1"></i>{torrent.seeders}</td>
                 <td class="text-right text-danger d-none d-md-table-cell"><i class="fas fa-fw fa-arrow-down mr-1"></i>{torrent.leechers}</td>
                 <td class="text-right text-muted d-none d-md-table-cell"
@@ -130,7 +138,7 @@ export class Search extends Component<any, State> {
                 </td>
                 <td class="text-right">
                   <a class="btn btn-sm no-outline p-1"
-                    href={magnetLink(torrent.infohash, torrent.name)}
+                    href={magnetLink(torrent.infohash, torrent.name, torrent.index_)}
                     data-balloon="Magnet link"
                     data-balloon-pos="left">
                     <i class="fas fa-fw fa-magnet"></i>
@@ -176,6 +184,6 @@ export class Search extends Component<any, State> {
   switchPage(a: { i: Search, nextPage: boolean }, event) {
     let newSearch = a.i.state.searchParams;
     newSearch.page += (a.nextPage) ? 1 : -1;
-    a.i.props.history.push(`/search/${newSearch.q}/${newSearch.page}`);
+    a.i.props.history.push(`/search/${newSearch.type_}/${newSearch.q}/${newSearch.page}`);
   }
 }

server/ui/src/index.tsx

@@ -17,7 +17,7 @@ class Index extends Component<any, any> {
         <div class="mt-3 p-0">
           <Switch>
             <Route exact path="/" component={Home} />
-            <Route path={`/search/:q/:page`} component={Search} />
+            <Route path={`/search/:type_/:q/:page`} component={Search} />
           </Switch>
         </div>
       </HashRouter>

server/ui/src/interfaces.ts

@@ -2,6 +2,7 @@ export interface SearchParams {
   q: string;
   page: number;
   size?: number;
+  type_: string;
 }
 
 export interface Results {
@@ -17,4 +18,6 @@ export interface Torrent {
   leechers: number;
   completed: number;
   scraped_date: number;
-}
+  index_: number;
+  path: string;
+}

server/ui/src/utils.ts

@@ -1,5 +1,9 @@
-export function magnetLink(infohash: string, name: string): string {
+export function magnetLink(infohash: string, name: string, index?: number): string {
-  return `magnet:?xt=urn:btih:${infohash}&dn=${name}${trackerListToUrl(trackerList)}`;
+  let link = `magnet:?xt=urn:btih:${infohash}&dn=${name}${trackerListToUrl(trackerList)}`;
+  if (index != undefined) {
+    link+=`&so=${index}`;
+  }
+  return link;
 }
 
 let trackerList: Array<string> = [
@@ -13,6 +17,19 @@ let trackerList: Array<string> = [
   "udp://explodie.org:6969/announce"
 ];
 
+export function getFileName(path: string): string {
+  let lines = path.split('/');
+  let out: string = lines[0];
+
+  for (let i = 1; i < lines.length; i++) {
+    let tabs = Array(i + 1).join('  ');
+    out += '\n' + tabs + '└─ ' + lines[i];
+  }
+
+  return out;
+
+}
+
 function trackerListToUrl(trackerList: Array<string>): string {
   return trackerList.map(t => "&tr=" + t).join("");
 }