pricecharts

commit e35118120309ccb54d23a1b3630de9534a0fbe6a
parent 6d93133318c769b7580ab8101f491d335a0126f8
Author: Kyle Milz <kyle@getaddrinfo.net>
Date:   Thu, 26 Feb 2015 01:34:37 -0700

PriceChart: add some real sounding headers

Also while here correct a mistake where I was not using the values from the
config file. Change get_ua to new_ua.

Diffstat:
M
PriceChart.pm
 | 
16
+++++++++++++---
M
price_scraper.pl
 | 
2
+-

2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/PriceChart.pm b/PriceChart.pm
@@ -4,7 +4,7 @@ use DBI;
 use Exporter;
 
 @ISA = ("Exporter");
-@EXPORT = qw(get_config get_dom get_ua get_log get_dbh);
+@EXPORT = qw(get_config get_dom new_ua get_log get_dbh);
 
 
 sub get_config
@@ -85,12 +85,22 @@ sub get_dom
 	return undef;
 }
 
-sub get_ua
+sub new_ua
 {
 	my $cfg = shift;
+	my $verbose = shift || 0;
 
-	my $ua = LWP::UserAgent->new(agent => $cfg->{"general"}{"user_agent"});
+	my $ua = LWP::UserAgent->new();
 	$ua->default_header("Accept" => "*/*");
+	$ua->default_header("Accept-Encoding" => scalar HTTP::Message::decodable());
+	$ua->default_header("Accept-Charset" => "utf-8");
+	$ua->default_header("Accept-Language" => "en-US");
+	$ua->default_header("Host" => "localhost:8177");
+	$ua->default_header("User-Agent" => $cfg->{"user_agent"});
+
+	while (my ($name, $value) = each %{$ua->default_headers}) {
+		print "info: new_ua: $name: $value\n";
+	}
 
 	return $ua;
 }
diff --git a/price_scraper.pl b/price_scraper.pl
@@ -20,7 +20,7 @@ $| = 1 if ($args{v});
 
 my $log = get_log("scrapes", $args{v});
 my $cfg = get_config();
-my $ua  = get_ua($cfg->{"general"});
+my $ua  = new_ua($cfg->{"general"}, $args{v});
 my $dbh = get_dbh($cfg->{"general"});
 
 # allow products to go out of stock. if we haven't seen them for > 30 days