commit e35118120309ccb54d23a1b3630de9534a0fbe6a
parent 6d93133318c769b7580ab8101f491d335a0126f8
Author: Kyle Milz <kyle@getaddrinfo.net>
Date: Thu, 26 Feb 2015 01:34:37 -0700
PriceChart: add some real sounding headers
Also while here correct a mistake where I was not using the values from the
config file. Change get_ua to new_ua.
Diffstat:
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/PriceChart.pm b/PriceChart.pm
@@ -4,7 +4,7 @@ use DBI;
use Exporter;
@ISA = ("Exporter");
-@EXPORT = qw(get_config get_dom get_ua get_log get_dbh);
+@EXPORT = qw(get_config get_dom new_ua get_log get_dbh);
sub get_config
@@ -85,12 +85,22 @@ sub get_dom
return undef;
}
-sub get_ua
+sub new_ua
{
my $cfg = shift;
+ my $verbose = shift || 0;
- my $ua = LWP::UserAgent->new(agent => $cfg->{"general"}{"user_agent"});
+ my $ua = LWP::UserAgent->new();
$ua->default_header("Accept" => "*/*");
+ $ua->default_header("Accept-Encoding" => scalar HTTP::Message::decodable());
+ $ua->default_header("Accept-Charset" => "utf-8");
+ $ua->default_header("Accept-Language" => "en-US");
+ $ua->default_header("Host" => "localhost:8177");
+ $ua->default_header("User-Agent" => $cfg->{"user_agent"});
+
+ while (my ($name, $value) = each %{$ua->default_headers}) {
+ print "info: new_ua: $name: $value\n";
+ }
return $ua;
}
diff --git a/price_scraper.pl b/price_scraper.pl
@@ -20,7 +20,7 @@ $| = 1 if ($args{v});
my $log = get_log("scrapes", $args{v});
my $cfg = get_config();
-my $ua = get_ua($cfg->{"general"});
+my $ua = new_ua($cfg->{"general"}, $args{v});
my $dbh = get_dbh($cfg->{"general"});
# allow products to go out of stock. if we haven't seen them for > 30 days