Don't collect up add lines, just add them within the loop.

This commit is contained in:
Dessalines 2018-10-04 14:05:50 -07:00
parent 1e54bfa717
commit 0af1db917d
2 changed files with 16 additions and 21 deletions

View File

@ -2,7 +2,6 @@ export LC_ALL=C
# Checking arguments # Checking arguments
# Help line # Help line
startdir="`pwd`"
torrent_csv_file="`pwd`/torrent.csv" torrent_csv_file="`pwd`/torrent.csv"
help="Run ./add_torrents.sh [TORRENTS_DIR] \nor goto https://gitlab.com/dessalines/torrent.csv for more help" help="Run ./add_torrents.sh [TORRENTS_DIR] \nor goto https://gitlab.com/dessalines/torrent.csv for more help"
@ -38,7 +37,6 @@ else
fi fi
# Loop over all torrents # Loop over all torrents
add_lines=""
cd $torrents_dir cd $torrents_dir
for torrent_file in *.torrent; do for torrent_file in *.torrent; do
@ -61,7 +59,7 @@ for torrent_file in *.torrent; do
created_date=$(date -d "${date_string}" +"%s") created_date=$(date -d "${date_string}" +"%s")
# Scrape for seeder counts # Scrape for seeder counts
scrape_text=$(python -m torrent_tracker_scraper.scraper \ scrape_text=$(timeout 20 python -m torrent_tracker_scraper.scraper \
-i "$infohash" \ -i "$infohash" \
-t tracker.coppersurfer.tk -p 6969) -t tracker.coppersurfer.tk -p 6969)
# -t tracker.internetwarriors.net -p 1337 # -t tracker.internetwarriors.net -p 1337
@ -71,21 +69,24 @@ for torrent_file in *.torrent; do
completed=$(grep -Po 'Completed: \K[0-9]+' <<< $scrape_text) completed=$(grep -Po 'Completed: \K[0-9]+' <<< $scrape_text)
scraped_date=$(date +%s) scraped_date=$(date +%s)
# Construct add line
add_line="$infohash,$name,$size_bytes,$created_date,$seeders,$leechers,$completed,$scraped_date" add_line="$infohash,$name,$size_bytes,$created_date,$seeders,$leechers,$completed,$scraped_date"
# Only add the line if there are seeds, and the infohash doesn't already exist # Only add the line if there are seeds, and the infohash doesn't already exist
if (( $seeders >= 0 )) && ! grep -q $infohash $torrent_csv_file; then if (( $seeders > 0 )) && ! grep -q $infohash $torrent_csv_file; then
add_lines+="\n$add_line"
# TODO edit this logic to do a line replace where its already there
# Append the add lines to the torrent.csv file
echo -e "\n$add_line" >> $torrent_csv_file
truncate -s -1 $torrent_csv_file # Removing last newline
echo -e "Added $name to $torrent_csv_file"
# head -n1 $torrent_csv_file && sort <(tail -n+2 $torrent_csv_file) > $torrent_csv_file # Sort it
else
echo -e "$name had no seeders, or was already added."
fi fi
done done
# Append the add lines to the torrent.csv file
cd $startdir
if [ ! -z "$add_lines" ]; then
echo -e $add_lines >> $torrent_csv_file
truncate -s -1 $torrent_csv_file # Removing last newline from file
echo -e "Added the lines to $torrent_csv_file\nSubmit a Pull Request now."
else
echo "No new torrents added."
fi

View File

@ -1,7 +1 @@
infohash,name,size_bytes,created_unix,seeders,leechers,completed,scraped_date infohash,name,size_bytes,created_unix,seeders,leechers,completed,scraped_date
ff4b5ebbf63bdd76e6d8b4e44d3300af6b03eef1,Commas - Future Ft.Lil Wayne \, Rick Ross \, Big Sean (Remix)-tv7DEYN6U2U.mp4,29330000,1538675508,0,0,0,1538684319
4a79db8f63cca3c4d99f173843fe6ce16b93ed5b,Evanescence - Wake me up Inside [Lyrics]-wXcdYBh3hgg.webm,7390000,1538666209,0,0,0,1538684319
60cc499f94c393c02e264f1f4d59ad90a37417a9,MGMT- Kids-w7JthgTMHDU.webm,5900000,1538666351,0,0,0,1538684320
9d9957b87b879e2530f1cea7dd1f53f8a09eb683,Presidential Alert System To Test On Wednesday-xE8XZU2TD5k.mp4,13960000,1538595191,0,0,0,1538684496
d6acad5792e2ee5ac680c73260acb773a2df6f24,Puerto Rico Forward - Episode 11 Promo 2-wtYDN8jAErA.mkv,14500000,1538595192,0,0,0,1538684496
4c799ac80793e2ba17adc2216368410d79e0ac4b,Seether- Hang On Lyrics-uYFut05BWrg.mkv,9110000,1538666352,0,0,0,1538684497
1 infohash,name,size_bytes,created_unix,seeders,leechers,completed,scraped_date infohash name size_bytes created_unix seeders leechers completed scraped_date
ff4b5ebbf63bdd76e6d8b4e44d3300af6b03eef1,Commas - Future Ft.Lil Wayne \, Rick Ross \, Big Sean (Remix)-tv7DEYN6U2U.mp4,29330000,1538675508,0,0,0,1538684319
4a79db8f63cca3c4d99f173843fe6ce16b93ed5b,Evanescence - Wake me up Inside [Lyrics]-wXcdYBh3hgg.webm,7390000,1538666209,0,0,0,1538684319
60cc499f94c393c02e264f1f4d59ad90a37417a9,MGMT- Kids-w7JthgTMHDU.webm,5900000,1538666351,0,0,0,1538684320
9d9957b87b879e2530f1cea7dd1f53f8a09eb683,Presidential Alert System To Test On Wednesday-xE8XZU2TD5k.mp4,13960000,1538595191,0,0,0,1538684496
d6acad5792e2ee5ac680c73260acb773a2df6f24,Puerto Rico Forward - Episode 11 Promo 2-wtYDN8jAErA.mkv,14500000,1538595192,0,0,0,1538684496
4c799ac80793e2ba17adc2216368410d79e0ac4b,Seether- Hang On Lyrics-uYFut05BWrg.mkv,9110000,1538666352,0,0,0,1538684497