From 559a55324cade494135bd2aba196e846b8960104 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Wed, 6 Nov 2019 18:14:50 -0700 Subject: [PATCH] Updating prune to work for files too. --- scripts/prune.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/prune.sh b/scripts/prune.sh index 9709f9c..158bbd4 100755 --- a/scripts/prune.sh +++ b/scripts/prune.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # This prunes torrents.csv, removing those with too many columns, and sorts it echo "Pruning torrents.csv ..." pushd .. @@ -50,6 +52,9 @@ cp $torrent_files_csv $torrent_files_csv_tmp header=$(head -n1 $torrent_files_csv_tmp) sed -i '1d' $torrent_files_csv_tmp +# Remove dups, keeping the last ones +tac $torrent_files_csv_tmp | sort -u -t';' -k1,1 -k2,2 -o $torrent_files_csv_tmp + # Same for the infohashes scanned sort --field-separator=';' --key=1,2 -o $torrent_files_csv_tmp $torrent_files_csv_tmp