diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 0000000..b512c09 --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file diff --git a/scripts/build_sqlite.sh b/scripts/build_sqlite.sh index f5c2566..9772a4e 100755 --- a/scripts/build_sqlite.sh +++ b/scripts/build_sqlite.sh @@ -1,6 +1,7 @@ #!/bin/bash csv_file="${TORRENTS_CSV_FILE:-../torrents.csv}" db_file="${TORRENTS_CSV_DB_FILE:-../torrents.db}" +torrent_files_json="`pwd`/../torrent_files.json" echo "Creating temporary torrents.db file..." @@ -10,9 +11,11 @@ sed 's/\"//g' $csv_file > torrents_removed_quotes.csv # Sort by seeders desc before insert sort --field-separator=';' --key=5 -nr -o torrents_removed_quotes.csv torrents_removed_quotes.csv -rm $db_file +rm db_tmp +touch db_tmp -sqlite3 -batch $db_file <<"EOF" +sqlite3 -batch db_tmp <<"EOF" +drop table if exists torrents; create table torrents( "infohash" TEXT, "name" TEXT, @@ -27,7 +30,51 @@ create table torrents( .import torrents_removed_quotes.csv torrents UPDATE torrents SET completed=NULL WHERE completed = ''; EOF - rm torrents_removed_quotes.csv +# Cache torrent files if they exist +if [ -f $torrent_files_json ]; then + echo "Building files DB from $torrent_files_json ..." + jq -r 'to_entries[] | {hash: .key, val: .value[]} | [.hash, .val.i, .val.p, .val.l] | join(";")' $torrent_files_json > torrent_files_temp + + # Removing those with too many ; + rg "^([^;]*;){3}[^;]+$" torrent_files_temp > torrent_files_temp_2 + mv torrent_files_temp_2 torrent_files_temp + + sqlite3 -batch db_tmp <<"EOF" +drop table if exists files; +create table files( + "infohash" TEXT, + "index_" INTEGER, + "path" TEXT, + "size_bytes" INTEGER, + "created_unix" INTEGER, + "seeders" INTEGER, + "leechers" INTEGER, + "completed" INTEGER, + "scraped_date" INTEGER); +.separator ";" +.import torrent_files_temp files +-- Filling the extra columns +insert into files +select files.infohash, +files.index_, +files.path, +files.size_bytes, +torrents.created_unix, +torrents.seeders, +torrents.leechers, +torrents.completed, +torrents.scraped_date +from files +inner join torrents on files.infohash = torrents.infohash +order by torrents.seeders desc, files.size_bytes desc; +delete from files where seeders is null; +EOF + rm torrent_files_temp +fi + +mv db_tmp $db_file + +echo "Done." diff --git a/scripts/package.json b/scripts/package.json new file mode 100644 index 0000000..c5bc01a --- /dev/null +++ b/scripts/package.json @@ -0,0 +1,12 @@ +{ + "name": "torrent-file-reader", + "version": "1.1.6", + "main": "scan_torrent_files.js", + "engines": { + "node": ">= 8.0.0" + }, + "dependencies": { + "minimist": "^1.2.0", + "read-torrent": "^1.3.1" + } +} diff --git a/scripts/scan_torrent_files.js b/scripts/scan_torrent_files.js new file mode 100644 index 0000000..9de981e --- /dev/null +++ b/scripts/scan_torrent_files.js @@ -0,0 +1,73 @@ +// jq -r 'to_entries[] | {hash: .key, val: .value[]} | {hash: .hash, i: .val.i, p: .val.p, l: .val.l}' torrent_files.json +// jq -r 'to_entries[] | {hash: .key, val: .value[]} | [.hash, .val.i, .val.p, .val.l] | join(";")' torrent_files.json +var fs = require('fs'), + path = require('path'), + readTorrent = require('read-torrent'), + argv = require('minimist')(process.argv.slice(2)); + +var torrentFiles = {}; +var jsonFile = '../torrent_files.json'; +main(); + +async function main() { + await fillTorrentFiles(); + await scanFolder(); + writeFile(); +} + + +async function fillTorrentFiles() { + if (fs.existsSync(jsonFile)) { + var fileContents = await fs.promises.readFile(jsonFile, 'utf8'); + torrentFiles = JSON.parse(fileContents); + } +} + +async function scanFolder() { + console.log('Scanning dir: ' + argv.dir + '...'); + var files = fs.readdirSync(argv.dir).filter(f => { + var f = f.split('.'); + var ext = f[1]; + var hash = f[0]; + return (ext == 'torrent' && !Object.keys(torrentFiles).includes(hash)); + }); + for (const file of files) { + var fullPath = argv.dir + '/' + file; + console.log(`Scanning File ${fullPath}`); + var torrent = await read(fullPath).catch(e => console.log('Read error')); + torrentFiles = { ...torrentFiles, ...torrent }; // concat them + }; + console.log('Done scanning.') +} + +function writeFile() { + torrentFiles = Object.keys(torrentFiles).sort().reduce((r, k) => (r[k] = torrentFiles[k], r), {}); + fs.writeFileSync(jsonFile, JSON.stringify(torrentFiles)); + console.log(`${jsonFile} written.`); +} + +function read(uri, options) { + return new Promise((resolve, reject) => { + readTorrent(uri, (err, info) => { + if (!err) { + // Removing some extra fields from files + if (info.files) { + info.files.forEach((f, i) => { + f.i = i; + f.p = f.path; + f.l = f.length; + delete f.name; + delete f.offset; + delete f.path; + delete f.length; + }); + } + + resolve({ [info.infoHash]: info.files }); + } else { + console.error('Error in read-torrent: ' + err.message + ' for torrent uri: ' + uri); + reject(err); + } + }); + }); +} diff --git a/scripts/scan_torrent_files.sh b/scripts/scan_torrent_files.sh new file mode 100755 index 0000000..2a04a29 --- /dev/null +++ b/scripts/scan_torrent_files.sh @@ -0,0 +1,10 @@ +torrent_files_json="`pwd`/../torrent_files.json" + +# Scan the torrent_files.json for already scanned torrents + +# Disjoint the ones there with the ones in your torrent scan dir + +# Run the js read-torrent in that dir, and update the torrent_files.json with the new ones + +node scan_torrent_files.js --dir "$1" + diff --git a/scripts/yarn.lock b/scripts/yarn.lock new file mode 100644 index 0000000..8878584 --- /dev/null +++ b/scripts/yarn.lock @@ -0,0 +1,405 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +ajv@^6.5.5: + version "6.7.0" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.7.0.tgz#e3ce7bb372d6577bb1839f1dfdfcbf5ad2948d96" + integrity sha512-RZXPviBTtfmtka9n9sy1N5M5b82CbxWIR6HIis4s3WQTXDJamc/0gpCWNGz6EWdWp4DOfjzJfhz/AS9zVPjjWg== + dependencies: + fast-deep-equal "^2.0.1" + fast-json-stable-stringify "^2.0.0" + json-schema-traverse "^0.4.1" + uri-js "^4.2.2" + +asn1@~0.2.3: + version "0.2.4" + resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" + integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg== + dependencies: + safer-buffer "~2.1.0" + +assert-plus@1.0.0, assert-plus@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" + integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU= + +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= + +aws-sign2@~0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" + integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg= + +aws4@^1.8.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f" + integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ== + +bcrypt-pbkdf@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" + integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4= + dependencies: + tweetnacl "^0.14.3" + +bencode@^0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/bencode/-/bencode-0.7.0.tgz#811ed647c0118945e41bb4bbbdea9a2c78a17083" + integrity sha1-gR7WR8ARiUXkG7S7veqaLHihcIM= + +caseless@~0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" + integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= + +combined-stream@^1.0.6, combined-stream@~1.0.6: + version "1.0.7" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.7.tgz#2d1d24317afb8abe95d6d2c0b07b57813539d828" + integrity sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w== + dependencies: + delayed-stream "~1.0.0" + +core-util-is@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= + +dashdash@^1.12.0: + version "1.14.1" + resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" + integrity sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA= + dependencies: + assert-plus "^1.0.0" + +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= + +ecc-jsbn@~0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" + integrity sha1-OoOpBOVDUyh4dMVkt1SThoSamMk= + dependencies: + jsbn "~0.1.0" + safer-buffer "^2.1.0" + +extend@~3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" + integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== + +extsprintf@1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" + integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU= + +extsprintf@^1.2.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" + integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= + +fast-deep-equal@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz#7b05218ddf9667bf7f370bf7fdb2cb15fdd0aa49" + integrity sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk= + +fast-json-stable-stringify@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2" + integrity sha1-1RQsDK7msRifh9OnYREGT4bIu/I= + +flatten@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/flatten/-/flatten-0.0.1.tgz#554440766da0a0d603999f433453f6c2fc6a75c1" + integrity sha1-VURAdm2goNYDmZ9DNFP2wvxqdcE= + +forever-agent@~0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" + integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE= + +form-data@~2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" + integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.6" + mime-types "^2.1.12" + +getpass@^0.1.1: + version "0.1.7" + resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" + integrity sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo= + dependencies: + assert-plus "^1.0.0" + +har-schema@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" + integrity sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI= + +har-validator@~5.1.0: + version "5.1.3" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.3.tgz#1ef89ebd3e4996557675eed9893110dc350fa080" + integrity sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g== + dependencies: + ajv "^6.5.5" + har-schema "^2.0.0" + +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + integrity sha1-muzZJRFHcvPZW2WmCruPfBj7rOE= + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + +is-typedarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" + integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= + +isstream@~0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" + integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= + +jsbn@~0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" + integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM= + +json-schema-traverse@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" + integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== + +json-schema@0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" + integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM= + +json-stringify-safe@~5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" + integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= + +jsprim@^1.2.2: + version "1.4.1" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" + integrity sha1-MT5mvB5cwG5Di8G3SZwuXFastqI= + dependencies: + assert-plus "1.0.0" + extsprintf "1.3.0" + json-schema "0.2.3" + verror "1.10.0" + +magnet-uri@^4.0.0: + version "4.2.3" + resolved "https://registry.yarnpkg.com/magnet-uri/-/magnet-uri-4.2.3.tgz#79cc6d65a00bb5b7ef5c25ae60ebbb5d9a7681a8" + integrity sha1-ecxtZaALtbfvXCWuYOu7XZp2gag= + dependencies: + flatten "0.0.1" + thirty-two "^0.0.2" + xtend "^4.0.0" + +magnet-uri@~2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/magnet-uri/-/magnet-uri-2.0.1.tgz#d331d3dfcd3836565ade0fc3ca315e39217bb209" + integrity sha1-0zHT3804NlZa3g/DyjFeOSF7sgk= + dependencies: + thirty-two "~0.0.1" + +mime-db@~1.37.0: + version "1.37.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.37.0.tgz#0b6a0ce6fdbe9576e25f1f2d2fde8830dc0ad0d8" + integrity sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg== + +mime-types@^2.1.12, mime-types@~2.1.19: + version "2.1.21" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.21.tgz#28995aa1ecb770742fe6ae7e58f9181c744b3f96" + integrity sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg== + dependencies: + mime-db "~1.37.0" + +minimist@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" + integrity sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ= + +oauth-sign@~0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" + integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== + +parse-torrent-file@^2.0.0: + version "2.1.4" + resolved "https://registry.yarnpkg.com/parse-torrent-file/-/parse-torrent-file-2.1.4.tgz#32d4b6afde631420e5f415919a222b774b575707" + integrity sha1-MtS2r95jFCDl9BWRmiIrd0tXVwc= + dependencies: + bencode "^0.7.0" + simple-sha1 "^2.0.0" + +parse-torrent@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/parse-torrent/-/parse-torrent-4.1.0.tgz#a814bd8505e8b58e88eb8ff3e2daff5d19a711b7" + integrity sha1-qBS9hQXotY6I64/z4tr/XRmnEbc= + dependencies: + magnet-uri "^4.0.0" + parse-torrent-file "^2.0.0" + +performance-now@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" + integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= + +psl@^1.1.24: + version "1.1.31" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.31.tgz#e9aa86d0101b5b105cbe93ac6b784cd547276184" + integrity sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw== + +punycode@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" + integrity sha1-wNWmOycYgArY4esPpSachN1BhF4= + +punycode@^2.1.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" + integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== + +qs@~6.5.2: + version "6.5.2" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" + integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== + +read-torrent@^1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/read-torrent/-/read-torrent-1.3.1.tgz#de7e721af6f7d94e0e7a1f23a94b71edc073b1ae" + integrity sha512-TzPdVpK3xEnvsS1yCy94CiOmfzG2/MNuZjpZi64+HJxb9fVZD4nIfFFGZ9T2N/dgwOFumfacw3xCD6rXtgwn2g== + dependencies: + magnet-uri "~2.0.0" + parse-torrent "^4.0.0" + request "^2.83.0" + xtend "^4.0.0" + +request@^2.83.0: + version "2.88.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef" + integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg== + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.8.0" + caseless "~0.12.0" + combined-stream "~1.0.6" + extend "~3.0.2" + forever-agent "~0.6.1" + form-data "~2.3.2" + har-validator "~5.1.0" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.19" + oauth-sign "~0.9.0" + performance-now "^2.1.0" + qs "~6.5.2" + safe-buffer "^5.1.2" + tough-cookie "~2.4.3" + tunnel-agent "^0.6.0" + uuid "^3.3.2" + +rusha@^0.8.1: + version "0.8.13" + resolved "https://registry.yarnpkg.com/rusha/-/rusha-0.8.13.tgz#9a084e7b860b17bff3015b92c67a6a336191513a" + integrity sha1-mghOe4YLF7/zAVuSxnpqM2GRUTo= + +safe-buffer@^5.0.1, safe-buffer@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + +safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== + +simple-sha1@^2.0.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/simple-sha1/-/simple-sha1-2.1.1.tgz#93f3b7f2e8dfdc056c32793e5d47b58d311b140d" + integrity sha512-pFMPd+I/lQkpf4wFUeS/sED5IqdIG1lUlrQviBMV4u4mz8BRAcB5fvUx5Ckfg3kBigEglAjHg7E9k/yy2KlCqA== + dependencies: + rusha "^0.8.1" + +sshpk@^1.7.0: + version "1.16.1" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877" + integrity sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg== + dependencies: + asn1 "~0.2.3" + assert-plus "^1.0.0" + bcrypt-pbkdf "^1.0.0" + dashdash "^1.12.0" + ecc-jsbn "~0.1.1" + getpass "^0.1.1" + jsbn "~0.1.0" + safer-buffer "^2.0.2" + tweetnacl "~0.14.0" + +thirty-two@^0.0.2, thirty-two@~0.0.1: + version "0.0.2" + resolved "https://registry.yarnpkg.com/thirty-two/-/thirty-two-0.0.2.tgz#4253e29d8cb058f0480267c5698c0e4927e54b6a" + integrity sha1-QlPinYywWPBIAmfFaYwOSSflS2o= + +tough-cookie@~2.4.3: + version "2.4.3" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781" + integrity sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ== + dependencies: + psl "^1.1.24" + punycode "^1.4.1" + +tunnel-agent@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" + integrity sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0= + dependencies: + safe-buffer "^5.0.1" + +tweetnacl@^0.14.3, tweetnacl@~0.14.0: + version "0.14.5" + resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" + integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q= + +uri-js@^4.2.2: + version "4.2.2" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.2.2.tgz#94c540e1ff772956e2299507c010aea6c8838eb0" + integrity sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ== + dependencies: + punycode "^2.1.0" + +uuid@^3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + +verror@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" + integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA= + dependencies: + assert-plus "^1.0.0" + core-util-is "1.0.2" + extsprintf "^1.2.0" + +xtend@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af" + integrity sha1-pcbVMr5lbiPbgg77lDofBJmNY68= diff --git a/server/service/src/main.rs b/server/service/src/main.rs index 78635d7..1e927fd 100644 --- a/server/service/src/main.rs +++ b/server/service/src/main.rs @@ -8,6 +8,7 @@ extern crate time; use actix_web::{fs, fs::NamedFile, http, server, App, HttpRequest, HttpResponse, Query}; use std::env; +use std::ops::Deref; use rusqlite::{Connection, NO_PARAMS}; @@ -45,6 +46,7 @@ struct SearchQuery { q: String, page: Option, size: Option, + type_: Option } fn search(query: Query) -> HttpResponse { @@ -57,13 +59,18 @@ fn search(query: Query) -> HttpResponse { fn search_query(query: Query) -> String { let page = query.page.unwrap_or(1); let size = query.size.unwrap_or(10); + let type_ = query.type_.as_ref().map_or("torrent", String::deref); let offset = size * (page - 1); - println!("query = {} , page = {}, size = {}", query.q, page, size); + println!("query = {}, type = {}, page = {}, size = {}", query.q, type_, page, size); - let results = sql_search(&query.q, size, offset); - - serde_json::to_string(&results).unwrap() + if type_ == "file" { + let results = torrent_file_search(&query.q, size, offset); + serde_json::to_string(&results).unwrap() + } else { + let results = torrent_search(&query.q, size, offset); + serde_json::to_string(&results).unwrap() + } } #[derive(Debug, Serialize, Deserialize)] @@ -78,7 +85,7 @@ struct Torrent { scraped_date: u32, } -fn sql_search(query: &str, size: usize, offset: usize) -> Vec { +fn torrent_search(query: &str, size: usize, offset: usize) -> Vec { let stmt_str = format!( "select * from torrents where name like '%{}%' limit {} offset {}", query.replace(" ", "%").replace("\'","''"), @@ -109,6 +116,52 @@ fn sql_search(query: &str, size: usize, offset: usize) -> Vec { torrents } +#[derive(Debug, Serialize, Deserialize)] +struct File { + infohash: String, + index_: u32, + path: String, + size_bytes: isize, + created_unix: u32, + seeders: u32, + leechers: u32, + completed: Option, + scraped_date: u32, +} + +fn torrent_file_search(query: &str, size: usize, offset: usize) -> Vec { + let stmt_str = format!( + "select * from files where path like '%{}%' limit {} offset {}", + query.replace(" ", "%").replace("\'","''"), + size, + offset + ); + + let conn = Connection::open(torrents_db_file()).unwrap(); + + let mut stmt = conn.prepare(&stmt_str).unwrap(); + let file_iter = stmt + .query_map(NO_PARAMS, |row| File { + infohash: row.get(0), + index_: row.get(1), + path: row.get(2), + size_bytes: row.get(3), + created_unix: row.get(4), + seeders: row.get(5), + leechers: row.get(6), + completed: row.get(7), + scraped_date: row.get(8), + }) + .unwrap(); + + let mut files = Vec::new(); + for file in file_iter { + files.push(file.unwrap()); + } + files +} + + #[cfg(test)] mod tests { use time::PreciseTime; @@ -117,7 +170,7 @@ mod tests { fn test() { let start = PreciseTime::now(); let results = - super::sql_search("sherlock", 10, 0); + super::torrent_search("sherlock", 10, 0); assert!(results.len() > 2); let end = PreciseTime::now(); println!("Query took {} seconds.", start.to(end)); diff --git a/server/ui/src/Main.css b/server/ui/src/Main.css index 85653a9..4a2a01b 100644 --- a/server/ui/src/Main.css +++ b/server/ui/src/Main.css @@ -58,3 +58,11 @@ a::-moz-focus-inner { box-shadow: none !important; outline: none !important; } + +.path_column { + white-space: -o-pre-wrap; + word-wrap: break-word; + white-space: pre-wrap; + white-space: -moz-pre-wrap; + white-space: -pre-wrap; +} diff --git a/server/ui/src/components/navbar.tsx b/server/ui/src/components/navbar.tsx index 6b9e0e0..d355955 100644 --- a/server/ui/src/components/navbar.tsx +++ b/server/ui/src/components/navbar.tsx @@ -12,7 +12,8 @@ export class Navbar extends Component { state: State = { searchParams: { page: 1, - q: "" + q: "", + type_: "torrent" } } @@ -49,9 +50,18 @@ export class Navbar extends Component { - +
+ + +
); @@ -59,23 +69,33 @@ export class Navbar extends Component { search(i: Navbar, event) { event.preventDefault(); - i.context.router.history.push(`/search/${i.state.searchParams.q}/${i.state.searchParams.page}`); + i.context.router.history.push(`/search/${i.state.searchParams.type_}/${i.state.searchParams.q}/${i.state.searchParams.page}`); } searchChange(i: Navbar, event) { let searchParams: SearchParams = { q: event.target.value, - page: 1 + page: 1, + type_: i.state.searchParams.type_ } i.setState({ searchParams: searchParams }); } + searchTypeChange(i: Navbar, event) { + let searchParams: SearchParams = { + q: i.state.searchParams.q, + page: 1, + type_: event.target.value + } + i.setState({ searchParams: searchParams }); + } fillSearchField() { let splitPath: Array = this.context.router.route.location.pathname.split("/"); - if (splitPath.length == 4 && splitPath[1] == 'search') + if (splitPath.length == 5 && splitPath[1] == 'search') this.state.searchParams = { - page: Number(splitPath[3]), - q: splitPath[2] + page: Number(splitPath[4]), + q: splitPath[3], + type_: splitPath[2] }; } } diff --git a/server/ui/src/components/search.tsx b/server/ui/src/components/search.tsx index 6f68de0..e59a9e1 100644 --- a/server/ui/src/components/search.tsx +++ b/server/ui/src/components/search.tsx @@ -3,7 +3,7 @@ import * as moment from 'moment'; import { endpoint } from '../env'; import { SearchParams, Results, Torrent } from '../interfaces'; -import { humanFileSize, magnetLink } from '../utils'; +import { humanFileSize, magnetLink, getFileName } from '../utils'; interface State { results: Results; @@ -19,7 +19,8 @@ export class Search extends Component { }, searchParams: { q: "", - page: 1 + page: 1, + type_: 'torrent' }, searching: false }; @@ -31,7 +32,8 @@ export class Search extends Component { componentDidMount() { this.state.searchParams = { page: Number(this.props.match.params.page), - q: this.props.match.params.q + q: this.props.match.params.q, + type_: this.props.match.params.type_ } this.search(); } @@ -41,7 +43,8 @@ export class Search extends Component { if (lastProps.match && lastProps.match.params !== this.props.match.params) { this.state.searchParams = { page: Number(this.props.match.params.page), - q: this.props.match.params.q + q: this.props.match.params.q, + type_: this.props.match.params.type_ } this.search(); } @@ -70,7 +73,7 @@ export class Search extends Component { fetchData(searchParams: SearchParams): Promise> { let q = encodeURI(searchParams.q); - return fetch(`${endpoint}/service/search?q=${q}&page=${searchParams.page}`) + return fetch(`${endpoint}/service/search?q=${q}&page=${searchParams.page}&type_=${searchParams.type_}`) .then(data => data.json()); } @@ -80,7 +83,8 @@ export class Search extends Component { { this.state.searching ? this.spinner() : this.state.results.torrents[0] ? - this.table() : this.noResults() + this.torrentsTable() + : this.noResults() } ); @@ -102,7 +106,7 @@ export class Search extends Component { ) } - table() { + torrentsTable() { return (
@@ -119,8 +123,12 @@ export class Search extends Component { {this.state.results.torrents.map(torrent => ( - - + { !torrent.name ? ( + + ) : ( + + )} +
{torrent.name}{humanFileSize(torrent.size_bytes, true)}{getFileName(torrent.path)}{torrent.name}{humanFileSize(torrent.size_bytes, true)} {torrent.seeders} {torrent.leechers} { @@ -176,6 +184,6 @@ export class Search extends Component { switchPage(a: { i: Search, nextPage: boolean }, event) { let newSearch = a.i.state.searchParams; newSearch.page += (a.nextPage) ? 1 : -1; - a.i.props.history.push(`/search/${newSearch.q}/${newSearch.page}`); + a.i.props.history.push(`/search/${newSearch.type_}/${newSearch.q}/${newSearch.page}`); } } diff --git a/server/ui/src/index.tsx b/server/ui/src/index.tsx index 2c169cb..774880e 100644 --- a/server/ui/src/index.tsx +++ b/server/ui/src/index.tsx @@ -17,7 +17,7 @@ class Index extends Component {
- +
diff --git a/server/ui/src/interfaces.ts b/server/ui/src/interfaces.ts index e43eeb7..2fbf4e2 100644 --- a/server/ui/src/interfaces.ts +++ b/server/ui/src/interfaces.ts @@ -2,6 +2,7 @@ export interface SearchParams { q: string; page: number; size?: number; + type_: string; } export interface Results { @@ -17,4 +18,6 @@ export interface Torrent { leechers: number; completed: number; scraped_date: number; -} \ No newline at end of file + index_: number; + path: string; +} diff --git a/server/ui/src/utils.ts b/server/ui/src/utils.ts index a06d09b..6e9ed65 100644 --- a/server/ui/src/utils.ts +++ b/server/ui/src/utils.ts @@ -1,5 +1,9 @@ -export function magnetLink(infohash: string, name: string): string { - return `magnet:?xt=urn:btih:${infohash}&dn=${name}${trackerListToUrl(trackerList)}`; +export function magnetLink(infohash: string, name: string, index?: number): string { + let link = `magnet:?xt=urn:btih:${infohash}&dn=${name}${trackerListToUrl(trackerList)}`; + if (index != undefined) { + link+=`&so=${index}`; + } + return link; } let trackerList: Array = [ @@ -13,6 +17,19 @@ let trackerList: Array = [ "udp://explodie.org:6969/announce" ]; +export function getFileName(path: string): string { + let lines = path.split('/'); + let out: string = lines[0]; + + for (let i = 1; i < lines.length; i++) { + let tabs = Array(i + 1).join(' '); + out += '\n' + tabs + '└─ ' + lines[i]; + } + + return out; + +} + function trackerListToUrl(trackerList: Array): string { return trackerList.map(t => "&tr=" + t).join(""); }