Reformatting files.json. Streaming to it instead of serializing the whole thing at once. Fixes #51

scripts/rescan_olds.sh

@@ -1,4 +1,5 @@
 # Rescan everything that hasn't been scanned in a while
+cd ../
 awk -F';' '$8 < $(date -d "6 months ago" "+%s")' torrents.csv | cut -d ';' -f1 > hashes_to_rescan
 grep -vFf hashes_to_rescan infohashes_scanned.txt > new_infohashes_scanned
 rm hashes_to_rescan

scripts/scan_torrent_files.js

@@ -1,4 +1,4 @@
-var fs = require('fs'),
+const fs = require('fs'),
   path = require('path'),
   readTorrent = require('read-torrent'),
   argv = require('minimist')(process.argv.slice(2));
@@ -74,7 +74,14 @@ function writeFile() {
     .sort()
     .filter(hash => torrentCsvHashes.has(hash))
     .reduce((r, k) => (r[k] = torrentFiles[k], r), {});
-  fs.writeFileSync(jsonFile, JSON.stringify(torrentFiles, null, 2));
+  fs.writeFileSync(jsonFile, "{\n");
+  var first = true;
+  for (let [key, value] of Object.entries(torrentFiles)) {
+    if(first) first = false;
+    else fs.appendFileSync(jsonFile, ",\n");
+    fs.appendFileSync(jsonFile, `${JSON.stringify(key)}:${JSON.stringify(value)}`);
+  }
+  fs.appendFileSync(jsonFile, "\n}");
   console.log(`${jsonFile} written.`);
   process.exit();
 }

scripts/scan_torrents.sh

@@ -76,5 +76,5 @@ popd
 rm -rf "$tmp_torrent_dir"
 
 # Scan the torrent dir for new files, and add them
-node --max-old-space-size=8192 scan_torrent_files.js --dir "$torrents_dir"
+node --max-old-space-size=2096 scan_torrent_files.js --dir "$torrents_dir"