pricecharts

track prices of consumer electronics
Log | Files | Refs | README

commit e35118120309ccb54d23a1b3630de9534a0fbe6a
parent 6d93133318c769b7580ab8101f491d335a0126f8
Author: Kyle Milz <kyle@getaddrinfo.net>
Date:   Thu, 26 Feb 2015 01:34:37 -0700

PriceChart: add some real sounding headers

Also while here correct a mistake where I was not using the values from the
config file. Change get_ua to new_ua.

Diffstat:
MPriceChart.pm | 16+++++++++++++---
Mprice_scraper.pl | 2+-
2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/PriceChart.pm b/PriceChart.pm @@ -4,7 +4,7 @@ use DBI; use Exporter; @ISA = ("Exporter"); -@EXPORT = qw(get_config get_dom get_ua get_log get_dbh); +@EXPORT = qw(get_config get_dom new_ua get_log get_dbh); sub get_config @@ -85,12 +85,22 @@ sub get_dom return undef; } -sub get_ua +sub new_ua { my $cfg = shift; + my $verbose = shift || 0; - my $ua = LWP::UserAgent->new(agent => $cfg->{"general"}{"user_agent"}); + my $ua = LWP::UserAgent->new(); $ua->default_header("Accept" => "*/*"); + $ua->default_header("Accept-Encoding" => scalar HTTP::Message::decodable()); + $ua->default_header("Accept-Charset" => "utf-8"); + $ua->default_header("Accept-Language" => "en-US"); + $ua->default_header("Host" => "localhost:8177"); + $ua->default_header("User-Agent" => $cfg->{"user_agent"}); + + while (my ($name, $value) = each %{$ua->default_headers}) { + print "info: new_ua: $name: $value\n"; + } return $ua; } diff --git a/price_scraper.pl b/price_scraper.pl @@ -20,7 +20,7 @@ $| = 1 if ($args{v}); my $log = get_log("scrapes", $args{v}); my $cfg = get_config(); -my $ua = get_ua($cfg->{"general"}); +my $ua = new_ua($cfg->{"general"}, $args{v}); my $dbh = get_dbh($cfg->{"general"}); # allow products to go out of stock. if we haven't seen them for > 30 days