torrents.csv/scripts/scan_torrent_files.js

114 lines
3.2 KiB
JavaScript
Raw Normal View History

const fs = require('fs'),
path = require('path'),
readTorrent = require('read-torrent'),
argv = require('minimist')(process.argv.slice(2));
var torrentFiles = {};
var torrentCsvHashes = new Set();
var jsonFile = '../torrent_files.json';
var torrentsCsvFile = '../torrents.csv';
2019-02-06 18:34:20 +00:00
console.log(`Scanning torrent files from ${argv.dir} into ${jsonFile} ...`);
main();
async function main() {
await fillTorrentFiles();
await fillTorrentCsvHashes();
await scanFolder();
writeFile();
}
async function fillTorrentFiles() {
if (fs.existsSync(jsonFile)) {
var fileContents = await fs.promises.readFile(jsonFile, 'utf8');
torrentFiles = JSON.parse(fileContents);
}
}
async function fillTorrentCsvHashes() {
var fileContents = await fs.promises.readFile(torrentsCsvFile, 'utf8');
var lines = fileContents.split('\n');
for (const line of lines) {
var hash = line.split(';')[0];
torrentCsvHashes.add(hash);
}
torrentCsvHashes.delete('infohash');
}
async function scanFolder() {
console.log('Scanning dir: ' + argv.dir + '...');
2019-02-07 01:48:49 +00:00
var fileHashes = new Set(Object.keys(torrentFiles));
var files = fs.readdirSync(argv.dir).filter(f => {
2019-02-07 01:48:49 +00:00
var sp = f.split('.');
var ext = sp[1];
var hash = sp[0];
var fullPath = argv.dir + '/' + f;
// It must be a torrent file,
// NOT in the torrent_files.json
// must be in the CSV file
// must have a file size
return (ext == 'torrent' &&
2019-02-07 01:48:49 +00:00
!fileHashes.has(hash) &&
torrentCsvHashes.has(hash) &&
getFilesizeInBytes(fullPath) > 0);
});
for (const file of files) {
var fullPath = argv.dir + '/' + file;
console.log(`Scanning File ${fullPath}`);
var torrent = await read(fullPath).catch(e => console.log('Read error'));
torrentFiles = { ...torrentFiles, ...torrent }; // concat them
};
2019-02-07 03:44:07 +00:00
console.log('Done scanning.');
}
2019-02-07 01:48:49 +00:00
function getFilesizeInBytes(filename) {
2019-02-07 03:44:07 +00:00
var stats = fs.statSync(filename);
var fileSizeInBytes = stats["size"];
return fileSizeInBytes;
2019-02-07 01:48:49 +00:00
}
function writeFile() {
torrentFiles = Object.keys(torrentFiles)
.sort()
.filter(hash => torrentCsvHashes.has(hash))
.reduce((r, k) => (r[k] = torrentFiles[k], r), {});
fs.writeFileSync(jsonFile, "{\n");
var first = true;
for (let [key, value] of Object.entries(torrentFiles)) {
if(first) first = false;
else fs.appendFileSync(jsonFile, ",\n");
fs.appendFileSync(jsonFile, `${JSON.stringify(key)}:${JSON.stringify(value)}`);
}
fs.appendFileSync(jsonFile, "\n}");
console.log(`${jsonFile} written.`);
2019-02-20 02:04:09 +00:00
process.exit();
}
function read(uri, options) {
return new Promise((resolve, reject) => {
readTorrent(uri, (err, info) => {
if (!err) {
// Removing some extra fields from files
if (info.files) {
info.files.forEach((f, i) => {
f.i = i;
f.p = f.path;
f.l = f.length;
delete f.name;
delete f.offset;
delete f.path;
delete f.length;
});
}
resolve({ [info.infoHash]: info.files });
} else {
console.error('Error in read-torrent: ' + err.message + ' for torrent uri: ' + uri);
reject(err);
}
});
});
}