torrents.csv/scripts/prune.sh

45 lines
1.2 KiB
Bash
Raw Normal View History

# This prunes torrents.csv, removing those with too many columns, and sorts it
echo "Pruning torrents.csv ..."
cd ..
torrents_csv="`pwd`/torrents.csv"
2019-01-28 23:01:03 +00:00
torrents_csv_tmp="`pwd`/torrents_prune_tmp.csv"
scanned_out="`pwd`/infohashes_scanned.txt"
2019-01-28 23:01:03 +00:00
cp $torrents_csv $torrents_csv_tmp
# Remove lines that don't have exactly 7 ';'
2019-01-28 23:01:03 +00:00
rg "^([^;]*;){7}[^;]+$" $torrents_csv_tmp > tmp_adds
mv tmp_adds $torrents_csv_tmp
# Remove random newlines
2019-01-28 23:01:03 +00:00
sed -i '/^$/d' $torrents_csv_tmp
# Extract the header
2019-01-28 23:01:03 +00:00
header=$(head -n1 $torrents_csv_tmp)
sed -i '1d' $torrents_csv_tmp
# Sort by seeders desc (so when we remove dups it removes the lower seeder counts)
2019-01-28 23:01:03 +00:00
# Remove dups, keeping the last ones
sort -r -t';' -k1,1 -o $torrents_csv_tmp $torrents_csv_tmp
sort -r -u -t';' -k1,1 -o $torrents_csv_tmp $torrents_csv_tmp
# Same for the infohashes scanned
sort -u -o $scanned_out $scanned_out
# Remove torrents with zero seeders
2019-01-28 23:01:03 +00:00
awk -F';' '$5>=1' $torrents_csv_tmp> tmp
mv tmp $torrents_csv_tmp
# Sort by infohash asc
2019-01-28 23:01:03 +00:00
sort --field-separator=';' --key=1 -o $torrents_csv_tmp $torrents_csv_tmp
# Add the header back in
2019-01-28 23:01:03 +00:00
sed -i "1i $header" $torrents_csv_tmp
2018-10-15 21:02:47 +00:00
#truncate -s -1 $torrents_csv # Removing last newline
2019-01-28 23:01:03 +00:00
mv $torrents_csv_tmp $torrents_csv
2019-01-29 00:49:29 +00:00
echo "Pruning done."