rob
/
torrents.csv
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adding a resort based on infohash, not seeders descending, since were 

sorting by seeders descending before caching to sqlite.
This commit is contained in:
Dessalines 2018-12-02 11:39:57 -07:00
parent f864295cf6
commit 0b570e6249
2 changed files with 17 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
scripts/build_sqlite.sh
View File

@ -1,6 +1,11 @@
# Remove quotes echo "Creating temporary torrents.db file..."
# Remove double quotes for csv import
sed 's/\"//g' ../torrents.csv > torrents_removed_quotes.csv sed 's/\"//g' ../torrents.csv > torrents_removed_quotes.csv
# Sort by seeders desc before insert
sort --field-separator=';' --key=5 -nr -o torrents_removed_quotes.csv torrents_removed_quotes.csv
rm ../torrents.db rm ../torrents.db
sqlite3 -batch ../torrents.db <<"EOF" sqlite3 -batch ../torrents.db <<"EOF"
@ -17,7 +22,11 @@ create table torrents(
.separator ";" .separator ";"
.import torrents_removed_quotes.csv torrents .import torrents_removed_quotes.csv torrents
UPDATE torrents SET completed=NULL WHERE completed = ''; UPDATE torrents SET completed=NULL WHERE completed = '';
create index name_index on torrents (name); # create index idx_name_seeders on torrents (name, seeders desc);
# create index idx_name on torrents (name);
# create index idx_seeders on torrents (seeders desc);
# create index idx_name on torrents (name collate nocase);
EOF EOF
rm torrents_removed_quotes.csv rm torrents_removed_quotes.csv

8
scripts/prune.sh
View File

@ -14,12 +14,16 @@ sed -i '/^$/d' $torrents_csv
header=$(head -n1 $torrents_csv) header=$(head -n1 $torrents_csv)
sed -i '1d' $torrents_csv sed -i '1d' $torrents_csv
# Sort by seeders desc (so when we remove dups it removes the lower seeder counts)
# TODO this should actually probably do it by scraped date
sort --field-separator=';' --key=5 -nr -o $torrents_csv $torrents_csv
# Remove dups # Remove dups
sort -u -t';' -k1,1 -o $torrents_csv $torrents_csv sort -u -t';' -k1,1 -o $torrents_csv $torrents_csv
sort -u -t';' -k2,2 -k3,3 -o $torrents_csv $torrents_csv sort -u -t';' -k2,2 -k3,3 -o $torrents_csv $torrents_csv
# Sort by seeders desc # Sort by infohash asc
sort --field-separator=';' --key=5 -nr -o $torrents_csv $torrents_csv sort --field-separator=';' --key=1 -o $torrents_csv $torrents_csv
# Add the header back in # Add the header back in
sed -i "1i $header" $torrents_csv sed -i "1i $header" $torrents_csv