UserAgent.pm (1235B)
1 package PS::UserAgent; 2 3 use LWP::UserAgent; 4 use Log::Log4perl qw(:easy); 5 use File::Path qw(make_path); 6 7 use PS::Config; 8 9 my $logger = Log::Log4perl->get_logger('pricesloth.useragent'); 10 11 sub new { 12 my $class = shift; 13 my $self = {}; 14 bless ($self, $class); 15 16 my $config = PS::Config->new(); 17 my $cfg = $config->{cfg}->{general}; 18 19 # it's optional to list ip addresses to scrape on 20 my $ua; 21 if ($cfg->{addrs}) { 22 my @addresses = split(" ", $cfg->{addrs}); 23 my $addr = $addresses[rand @addresses]; 24 $logger->debug("new_ua: using ip $addr\n"); 25 $ua = LWP::UserAgent->new(local_address => $addr); 26 } 27 else { 28 $ua = LWP::UserAgent->new(); 29 } 30 31 $ua->default_header("Accept" => "*/*"); 32 $ua->default_header("Accept-Encoding" => scalar HTTP::Message::decodable()); 33 $ua->default_header("Accept-Charset" => "utf-8"); 34 $ua->default_header("Accept-Language" => "en-US"); 35 $ua->default_header("User-Agent" => $cfg->{agent}); 36 37 $self->{ua} = $ua; 38 return $self; 39 } 40 41 sub get_dom { 42 my ($self, $url) = @_; 43 my $ua = $self->{ua}; 44 45 my $resp = $ua->get($url); 46 if ($resp->is_success) { 47 $logger->debug("get_dom: " . $resp->status_line . " $url\n"); 48 return $resp; 49 } 50 51 $logger->error("get_dom: " . $resp->status_line . " $url\n"); 52 return; 53 } 54 55 1;