pricecharts

track prices of consumer electronics
Log | Files | Refs | README

commit 8e30e89265d8de1dab0b0875e66042219ea0dbf2
parent 440364db6fb93516c5dc4ea4c3126a0d9da472c3
Author: Kyle Milz <kyle@getaddrinfo.net>
Date:   Mon, 13 Oct 2014 13:09:58 -0600

price_scraper: filter and update last_seen field

Use this field to not scrape out of date products.

Diffstat:
Mprice_scraper.pl | 5++++-
1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/price_scraper.pl b/price_scraper.pl @@ -29,8 +29,9 @@ if ($args{p}) { $part_num = $args{p}; } else { + my $cutoff = time - (30 * 24 * 60 * 60); my $results = $dbh->selectcol_arrayref("select part_num from products " . - "order by last_scraped asc"); + "where last_seen > $cutoff order by last_scraped asc"); if (scalar $results == 0) { print "Product table empty, run product_scraper.pl\n"; exit; @@ -99,6 +100,8 @@ for (sort keys $cfg->{vendors}) { $dbh->do("insert into prices(date, part_num, vendor, price, duration)" . "values (?, ?, ?, ?, ?)", undef, $date, $part_num, $_, $price, time - $start); + $dbh->do("update products set last_seen = ? where part_num = ?", + undef, $date, $part_num); } my $duration = time - $date;