pricecharts

commit 75d1489a4eac9a03ed0009a73b8932d3725e4927
parent a26c35f5aece23dc8edbbd291d3f4b0ec7e75875
Author: Kyle Milz <kyle@getaddrinfo.net>
Date:   Tue, 11 Nov 2014 16:24:38 -0700

don't parse vendors from config anymore

Instead, they must be inserted into the database first using a helper script.

Diffstat:
M
price_scraper.pl
 | 
24
++++++++++++++----------
M
shared.pm
 | 
13
+------------
M
update_vendors.pl
 | 
35
+++++++++++++++++++++--------------

3 files changed, 36 insertions(+), 36 deletions(-)
diff --git a/price_scraper.pl b/price_scraper.pl
@@ -50,22 +50,26 @@ my $prices_sth = $dbh->prepare($sql);
 $sql = "update products set last_seen = ? where part_num = ?";
 my $products_sth = $dbh->prepare($sql);
 
+$sql = "select * from vendors";
+my $vendor_sth = $dbh->prepare($sql);
+
 my $date = time;
-for (sort keys $cfg->{vendors}) {
-	my $start = time;
-	my $vendor = $cfg->{vendors}{$_};
+$vendor_sth->execute();
+while (my ($vendor, $url, $price_tag, $sale_tag) = $vendor_sth->fetchrow_array) {
 
-	print "$_:\n" if ($args{v});
+	my $start = time;
+	print "$vendor:\n" if ($args{v});
 
-	my $dom = get_dom("$vendor->{search_uri}$part_num", $ua);
+	my $dom = get_dom($url . $part_num, $ua);
 	if (!defined $dom) {
 		msg("e", "error: dom");
 		next;
 	}
+	print "\turl GET ok\n" if ($args{v});
 
-	my $price = get_price($vendor->{"reg_price"}, $dom);
-	if ($vendor->{sale_price}) {
-		my $sale_price = get_price($vendor->{"sale_price"}, $dom);
+	my $price = get_price($price_tag, $dom);
+	if ($sale_tag) {
+		my $sale_price = get_price($sale_tag, $dom);
 		$price = $sale_price if (defined $sale_price);
 	}
 	if (! $price) {
@@ -86,11 +90,11 @@ for (sort keys $cfg->{vendors}) {
 		next;
 	}
 
-	msg(substr($_, 0, 1), "price = \$$price");
+	msg(substr($vendor, 0, 1), "price = \$$price");
 
 	next if ($args{n});
 
-	$prices_sth->execute($date, $part_num, $_, $price, time - $start);
+	$prices_sth->execute($date, $part_num, $vendor, $price, time - $start);
 	$products_sth->execute($date, $part_num);
 
 	print "\tdb updated\n" if ($args{v});
diff --git a/shared.pm b/shared.pm
@@ -12,7 +12,7 @@ use POSIX;
 sub get_config
 {
 	my $parser = Config::Grammar->new({
-		_sections => ['general', 'vendors'],
+		_sections => ['general'],
 		general => {
 			_vars => [
 				'user_agent',
@@ -20,17 +20,6 @@ sub get_config
 				'smtp',
 			],
 		},
-		vendors	=> {
-			_sections => ['/[A-Za-z ]+/'],
-			'/[A-Za-z ]+/' => {
-				_vars => [
-					'search_uri',
-					'reg_price',
-					'sale_price',
-					'color'
-				],
-			},
-		},
 	});
 	my $cfg_file = "/etc/pricechart.cfg";
 	return $parser->parse($cfg_file) or die "error: $parser->{err}\n";
diff --git a/update_vendors.pl b/update_vendors.pl
@@ -3,35 +3,42 @@
 use strict;
 use warnings;
 
-use Config::Grammar;
+use Getopt::Std;
 
 use shared;
 
 
-my $cfg = get_config();
+my %args;
+getopt("v:u:r:s:c:", \%args);
+
+if (!$args{v}) {
+	print "Argument -v must be present\n";
+	exit
+}
+
 my $dbh = get_dbh();
 
 $dbh->do("create table if not exists vendors(" .
 	"name text not null primary key, " .
 	"search_url not null, " .
+	"price_tag not null, " .
+	"sale_tag, " .
 	"color text not null)") or die $DBI::errstr;
 
-my $sql = "update vendors set search_url = ?, color = ? where name = ?";
+my $sql = "update vendors set search_url = ?, price_tag = ?, sale_tag = ?, " .
+	"color = ? where name = ?";
 my $update_sth = $dbh->prepare($sql);
 
-$sql = "insert into vendors(name, search_url, color) values (?, ?, ?)";
+$sql = "insert into vendors(name, search_url, price_tag, sale_tag, color) " .
+	"values (?, ?, ?, ?, ?)";
 my $insert_sth = $dbh->prepare($sql);
 
-for (sort keys $cfg->{vendors}) {
-	$sql = "select * from vendors where name = ?";
-	if ($dbh->selectrow_arrayref($sql, undef, $_)) {
-		$update_sth->execute($cfg->{vendors}{$_}{search_uri},
-			"#$cfg->{vendors}{$_}{color}", $_);
-	}
-	else {
-		$insert_sth->execute($_, $cfg->{vendors}{$_}{search_uri},
-		 	"#$cfg->{vendors}{$_}{color}");
-	}
+$sql = "select * from vendors where name = ?";
+if ($dbh->selectrow_arrayref($sql, undef, $args{v})) {
+	$update_sth->execute($args{u}, $args{r}, $args{s}, $args{c}, $args{v});
+}
+else {
+	$insert_sth->execute($args{v}, $args{u}, $args{r}, $args{s}, $args{c});
 }
 
 $dbh->disconnect();