pricecharts

commit 653ad438b548aa353274c0c199cac00454127390
parent b90fdd53ba1da4dcdc3e0495a9e24898c9fc4fc5
Author: Kyle Milz <kyle@getaddrinfo.net>
Date:   Sun, 19 Oct 2014 16:15:51 -0600

product_scraper: sleep in between products

Diffstat:
M
product_scraper.pl
 | 
10
+++++++---

1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/product_scraper.pl b/product_scraper.pl
@@ -22,6 +22,7 @@ my $dbh = get_dbh($cfg);
 my $ua  = get_ua($cfg);
 
 $| = 1 if ($args{v});
+srand;
 
 $dbh->do("create table if not exists products(" .
 	"part_num text not null primary key, " .
@@ -45,8 +46,8 @@ print "$vendor:\n" if ($args{v});
 
 my $email;
 $email .= "*** $vendor ***\n\n";
-$email .= "type            scraped total new\n";
-$email .= "------------    ------- ----- ---\n";
+$email .= "type            scraped total new time\n";
+$email .= "------------    ------- ----- --- ----\n";
 
 my @new = ();
 for (keys %product_map) {
@@ -78,7 +79,9 @@ for (keys %product_map) {
 
 	my $new = 0;
 	my $old = 0;
+	my $start = time;
 	for my $node (@results) {
+		sleep rand % 10;
 		my $product = HTML::Grabber->new(html => $node);
 
 		# title is easier to parse from general results page
@@ -129,7 +132,8 @@ for (keys %product_map) {
 		print "($part_num) $brand $title\n" if ($args{v});
 	}
 
-	$email .= sprintf("%7s %5s %3s\n", $new + $old, scalar @results, $new);
+	$email .= sprintf("%7s %5s %3s %4s\n",
+		$new + $old, scalar @results, $new, time - $start);
 }
 
 $email .= "\nNew products:\n" if (@new);