rob
/
torrents.csv
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Reformatting files.json. Streaming to it instead of serializing the whole thing at once. Fixes #51

This commit is contained in:
Dessalines 2019-02-20 18:02:16 -08:00
parent a2af1c2fc4
commit c460c958dd
3 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scripts/rescan_olds.sh
View File

@ -1,4 +1,5 @@
# Rescan everything that hasn't been scanned in a while # Rescan everything that hasn't been scanned in a while
cd ../
awk -F';' '$8 < $(date -d "6 months ago" "+%s")' torrents.csv | cut -d ';' -f1 > hashes_to_rescan awk -F';' '$8 < $(date -d "6 months ago" "+%s")' torrents.csv | cut -d ';' -f1 > hashes_to_rescan
grep -vFf hashes_to_rescan infohashes_scanned.txt > new_infohashes_scanned grep -vFf hashes_to_rescan infohashes_scanned.txt > new_infohashes_scanned
rm hashes_to_rescan rm hashes_to_rescan

11
scripts/scan_torrent_files.js
View File

@ -1,4 +1,4 @@
var fs = require('fs'), const fs = require('fs'),
path = require('path'), path = require('path'),
readTorrent = require('read-torrent'), readTorrent = require('read-torrent'),
argv = require('minimist')(process.argv.slice(2)); argv = require('minimist')(process.argv.slice(2));
@ -74,7 +74,14 @@ function writeFile() {
.sort() .sort()
.filter(hash => torrentCsvHashes.has(hash)) .filter(hash => torrentCsvHashes.has(hash))
.reduce((r, k) => (r[k] = torrentFiles[k], r), {}); .reduce((r, k) => (r[k] = torrentFiles[k], r), {});
fs.writeFileSync(jsonFile, JSON.stringify(torrentFiles, null, 2)); fs.writeFileSync(jsonFile, "{\n");
var first = true;
for (let [key, value] of Object.entries(torrentFiles)) {
if(first) first = false;
else fs.appendFileSync(jsonFile, ",\n");
fs.appendFileSync(jsonFile, `${JSON.stringify(key)}:${JSON.stringify(value)}`);
}
fs.appendFileSync(jsonFile, "\n}");
console.log(`${jsonFile} written.`); console.log(`${jsonFile} written.`);
process.exit(); process.exit();
} }

2
scripts/scan_torrents.sh
View File

@ -76,5 +76,5 @@ popd
rm -rf "$tmp_torrent_dir" rm -rf "$tmp_torrent_dir"
# Scan the torrent dir for new files, and add them # Scan the torrent dir for new files, and add them
node --max-old-space-size=8192 scan_torrent_files.js --dir "$torrents_dir" node --max-old-space-size=2096 scan_torrent_files.js --dir "$torrents_dir"