pricecharts

commit 20683a56ca80638aae900fc67d5bf72bd7f3486d
parent 7ee98c25b29cb6107f6b1385e54e4f9b17c6ad7f
Author: Kyle Milz <kyle@getaddrinfo.net>
Date:   Thu,  2 Apr 2015 19:46:27 -0600

price_scraper: general cleanup

- use qq
- remove -m -p args
- select both manufacturer and part num
- add product type to log

Diffstat:
M
price_scraper
 | 
54
++++++++++++++++++++++++++++--------------------------

1 file changed, 28 insertions(+), 26 deletions(-)
diff --git a/price_scraper b/price_scraper
@@ -16,7 +16,7 @@ use URI::Escape;
 
 
 my %args;
-getopts("m:np:v", \%args);
+getopts("nv", \%args);
 
 $| = 1 if ($args{v});
 
@@ -30,19 +30,18 @@ my $log = get_log($log_path, $args{v});
 # allow products to go out of stock. if we haven't seen them for > 30 days
 # chances are retailers aren't carrying them anymore
 my $cutoff = time - (30 * 24 * 60 * 60);
-my $sql = "select part_num, manufacturer from products " .
+my $sql = "select part_num, manufacturer, type from products " .
 	"where last_seen > $cutoff order by last_scraped asc";
-my ($part_num, $manufacturer) = $dbh->selectrow_array($sql);
+my ($part_num, $manufacturer, $type) = $dbh->selectrow_array($sql);
 
-# prevent races with other scrapers, claim ownership as soon as possible
-$dbh->do("update products set last_scraped = ? where part_num = ?",
-	undef, time, $part_num);
-
-if ($args{p} && $args{m}) {
-	$part_num = $args{p};
-	$manufacturer = $args{m};
+unless (defined $part_num && defined $manufacturer) {
+	print "error: no parts have been seen in the last $cutoff s";
+	exit 1;
 }
-exit unless (defined $part_num);
+
+# prevent races with other scrapers, claim ownership as soon as possible
+$dbh->do("update products set last_scraped = ? where part_num = ? and manufacturer = ?",
+	undef, time, $part_num, $manufacturer);
 
 $dbh->do(qq{
 	create table if not exists retailers(
@@ -51,17 +50,18 @@ $dbh->do(qq{
 		url text not null)
 }) or die $DBI::errstr;
 
-$dbh->do("create table if not exists prices(" .
-	"date int not null, " .
-	"manufacturer text not null, " .
-	"part_num text not null, " .
-	"retailer text not null, " .
-	"price int not null, " .
-	"duration int, " .
-	"primary key(date, part_num, retailer, price), " .
-	"foreign key(manufacturer, part_num) references products(manufacturer, part_num), " .
-	"foreign key(retailer) references retailers(name))"
-) or die $DBI::errstr;
+$dbh->do(qq{
+	create table if not exists prices(
+	date int not null,
+	manufacturer text not null,
+	part_num text not null,
+	retailer text not null,
+	price int not null,
+	duration int,
+	primary key(date, part_num, retailer, price),
+	foreign key(manufacturer, part_num) references products(manufacturer, part_num),
+	foreign key(retailer) references retailers(name))
+}) or die $DBI::errstr;
 
 print "info: scraping $manufacturer $part_num\n" if ($args{v});
 
@@ -69,7 +69,7 @@ $sql = "insert into prices(date, manufacturer, part_num, retailer, price, durati
 	"values (?, ?, ?, ?, ?, ?)";
 my $prices_sth = $dbh->prepare($sql);
 
-$sql = "update products set last_seen = ? where part_num = ?";
+$sql = "update products set last_seen = ? where part_num = ? and manufacturer = ?";
 my $products_sth = $dbh->prepare($sql);
 
 $sql = "insert or replace into retailers(name, color, url) values (?, ?, ?)";
@@ -135,18 +135,20 @@ for my $retailer (sort keys %{$cfg->{retailers}}) {
 	$retailer_sth->execute($retailer, $color, $url);
 	$prices_sth->execute($start, $manufacturer, $part_num, $retailer, $price,
 		time - $retailer_start);
-	$products_sth->execute($start, $part_num);
+	$products_sth->execute($start, $part_num, $manufacturer);
 	$descriptions_sth->execute($manufacturer, $part_num, $retailer, $desc, time);
 
 	print "info: $retailer: db: inserted \$$price\n" if ($args{v});
 }
 
-printf $log "%s %-10s %-20s [%s] (%i s)\n", $timestamp, $manufacturer,
-	$part_num, join("", @status), time - $start;
+printf $log "%s %-10s %-10s %-20s [%s] (%i s)\n", $timestamp, $type,
+	$manufacturer, $part_num, join("", @status), time - $start;
 
 $log->close();
+$retailer_sth = undef;
 $prices_sth = undef;
 $products_sth = undef;
+$descriptions_sth = undef;
 $dbh->disconnect();
 
 exit 0;