commit 8e30e89265d8de1dab0b0875e66042219ea0dbf2
parent 440364db6fb93516c5dc4ea4c3126a0d9da472c3
Author: Kyle Milz <kyle@getaddrinfo.net>
Date: Mon, 13 Oct 2014 13:09:58 -0600
price_scraper: filter and update last_seen field
Use this field to not scrape out of date products.
Diffstat:
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/price_scraper.pl b/price_scraper.pl
@@ -29,8 +29,9 @@ if ($args{p}) {
$part_num = $args{p};
}
else {
+ my $cutoff = time - (30 * 24 * 60 * 60);
my $results = $dbh->selectcol_arrayref("select part_num from products " .
- "order by last_scraped asc");
+ "where last_seen > $cutoff order by last_scraped asc");
if (scalar $results == 0) {
print "Product table empty, run product_scraper.pl\n";
exit;
@@ -99,6 +100,8 @@ for (sort keys $cfg->{vendors}) {
$dbh->do("insert into prices(date, part_num, vendor, price, duration)" .
"values (?, ?, ?, ?, ?)",
undef, $date, $part_num, $_, $price, time - $start);
+ $dbh->do("update products set last_seen = ? where part_num = ?",
+ undef, $date, $part_num);
}
my $duration = time - $date;