Day 6 & 7 – Moose
Packages
|—Utils
| |—Counter.pm
package Utils::Counter;
# FOR PERFORMANCE, DO NOT USE MOOSE ON THIS CLASS
# TESTS WITH MOOSE
#Files=4, Tests=18, 20 wallclock secs ( 0.07 usr 0.02 sys + 16.08 cusr 0.30 csys = 16.47 CPU)
# TESTS WITHOUT MOOSE
#Files=4, Tests=18, 6 wallclock secs ( 0.06 usr 0.03 sys + 5.20 cusr 0.28 csys = 5.57 CPU)
use strict;
use warnings;
sub new {
my ( $class, $param1 ) = @_;
my $self = {
_startWith => undef,
_current => undef,
_increment => 1,
_maxIterations => 100,
_currIteration => 0,
};
if(defined($param1)) {
$self->{_startWith} = $param1->{start} if exists($param1->{startWith});
$self->{_increment} = $param1->{increment} if exists($param1->{increment});
$self->{_maxIterations} = $param1->{maxIterations} if exists($param1->{maxIterations});
}
bless $self, $class;
return $self;
}
sub increment {
my ( $self, $param1 ) = @_;
$self->{_increment} = $param1 if defined($param1);
return $self->{_increment};
}
sub startWith {
my ( $self, $param1 ) = @_;
$self->{_startWith} = $param1 if defined($param1);
return $self->{_startWith};
}
sub maxIterations {
my ( $self, $param1 ) = @_;
$self->{_maxIterations} = $param1 if defined($param1);
return $self->{_maxIterations};
}
sub currIteration {
my ( $self, $param1 ) = @_;
$self->{_currIteration} = $param1 if defined($param1);
return $self->{_currIteration};
}
sub _init {
my ( $self ) = @_;
if(!defined($self->{_startWith})) {
$self->{_startWith} = 1;
}
$self->{_currIteration} = 0;
$self->{_current} = $self->{_startWith};
}
sub reset {
my ( $self ) = @_;
$self->{_currIteration} = 0;
$self->{_current} = undef;
}
sub hasNext {
my ( $self ) = @_;
my $hasNext = 0;
if($self->{_currIteration} < $self->{_maxIterations}) {
$hasNext = 1;
}
return $hasNext;
}
sub getNext {
my ( $self ) = @_;
if(!defined($self->{_current})) {
$self->_init();
}
else {
$self->{_current} += $self->{_increment};
}
$self->{_currIteration}++;
return $self->{_current};
}
1;
| |—Web.pm
package Utils::Web;
use base 'Class::Singleton';
use HTTP::Request;
use HTTP::Response;
use LWP::UserAgent;
# this only gets called the first time instance() is called
sub new_instance {
my $class = shift;
my $self = bless { }, $class;
$self->{_userAgent} = LWP::UserAgent->new;
$self->{_userAgent}->agent("Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0)");
return $self;
}
sub setProxy {
my ( $self, $url, $user, $password ) = @_;
$self->{_userAgent}->credentials( $url, '', $user, $password );
$self->{_userAgent}->proxy(['http', 'ftp'], $url);
}
sub getPage{
my ( $self, $url ) = @_;
my $response = $self->{_userAgent}->request(HTTP::Request->new('GET', $url, [ 'Content-length' => 0]));
return $response->content;
}
1;
|—Job
| |—Site
| |—Seek.pm
package Job::Site::Seek;
use Moose;
extends 'Job::Site';
use Utils::Web;
sub BUILD {
my ($self) = @_;
$self->displayName("Seek - Melbourne");
$self->url("http://www.seek.com.au");
$self->searchString("/JobSearch?DateRange=31&location=1002&Keywords=<<KEYWORDS>>&page=<<COUNTER>>");
$self->adString("/job/<<JOB_ID>>");
}
# Download and process a website
sub processSite {
my ( $self ) = @_;
my $content = undef;
my $web = Utils::Web->new_instance();
while($self->counter->hasNext()) {
$content .= $web->getPage($self->_getNextPageUrl());
}
return $content;
}
no Moose;
__PACKAGE__->meta->make_immutable;
| |—JobAd.pm
package Job::JobAd;
use Moose;
use Moose::Helper;
my $attrHash = {
site => { isa => 'Job::Site' },
id => {},
url => {},
shortDesc => {},
description => {},
};
createAccessors($attrHash, &has);
no Moose;
__PACKAGE__->meta->make_immutable;
| |—Site.pm
package Job::Site;
use Moose;
use Moose::Helper;
use Utils::Web;
use Utils::Counter;
my $attrHash = {
displayName => {},
url => {},
searchString => {},
adString => {},
keywords => {},
counter => {
isa => 'Utils::Counter',
lazy => 1,
builder => 1,
},
pagesToDownload => {
isa => 'Num',
trigger => &_pagesToDownload_set,
},
};
createAccessors($attrHash, &has);
#builder
sub _build_counter {
my $self = shift;
return Utils::Counter->new({startWith => 1, maxIterations => 10});
}
#trigger
sub _pagesToDownload_set {
my ( $self, $new_value, $old_value ) = @_;
# Update counter with new value
$self->counter->maxIterations($new_value);
}
# Builds the url
sub _getNextPageUrl {
my ( $self ) = @_;
my $keywords = $self->keywords;
my $counter = $self->counter->getNext();
my $searchString = $self->searchString;
$searchString =~ s/<<KEYWORDS>>/$keywords/;
$searchString =~ s/<<COUNTER>>/$counter/;
return $self->url . $searchString;
}
# Download and process a website
# Override this method on each subclass
sub processSite {
my ( $self ) = @_;
my $content = undef;
my $web = Utils::Web->new_instance();
while($self->counter->hasNext()) {
my $nextp = $self->_getNextPageUrl();
my $result = $web->getPage($nextp);
$result = "ERROR - Could not get the page [$nextp]n" if !defined($result);
$content .= $result;
}
return $content;
}
no Moose;
__PACKAGE__->meta->make_immutable;
|—Moose
| |—Helper.pm
package Moose::Helper;
use base "Exporter";
@EXPORT = qw(createAccessors);
sub createAccessors {
my ($attrHash, $has, $default) = @_;
$default = {} if(!defined($default));
$default->{is} = 'rw' if(!defined($default->{is}));
$default->{isa} = 'Str' if(!defined($default->{isa}));
foreach my $attr (keys %$attrHash) {
my $values = $attrHash->{$attr};
$values->{is} = $default->{is} if(!defined($values->{is}));
$values->{isa} = $default->{isa} if(!defined($values->{isa}));
$values->{builder} = '_build_' . $attr if(defined($values->{builder}) && $values->{builder} == 1);
$has->($attr => %$values );
}
}
1;
||—Makefile.PL
use ExtUtils::MakeMaker;
WriteMakefile(
test => {TESTS => 't/*.t t/*/*.t t/*/*/*.t t/*/*/*/*.t',}
);
||—run_tests.sh
perl Makefile.PL make test # Clean make clean >>/dev/null
t
|—Test
| |—Utils
| |—Counter.t
#!/usr/bin/perl
use lib 'Packages';
use strict;
use Test::More 0.88;
require Utils::Counter;
my $counter = new_ok('Utils::Counter');
$counter->increment(0.3567);
$counter->startWith(.325);
$counter->maxIterations(11);
my $c = $counter->startWith();
while($counter->hasNext()) {
is($counter->getNext(), $c, "Counter $c OK");
$c += $counter->increment();
}
# Test a big limit
my $i = 0;
my $infinit = 1000000;
$counter->reset();
$counter->maxIterations($infinit);
while($counter->hasNext()) {
$i++;
$counter->getNext();
}
is($i, $infinit, "Iterator limiter works OK");
my $expected = $counter->startWith() + ($counter->currIteration() * $counter->increment() );
is(sprintf("%.4f", $counter->getNext()), sprintf("%.4f", $expected), "Counter counted until $expected OK");
done_testing;
| |—MockWeb.pm
package Test::Utils::MockWeb;
use strict;
use warnings;
sub new {
my ($class) = @_;
my $self = {};
bless $self, $class;
return $self;
}
sub setPageResponse {
my ( $self, $url, $response ) = @_;
$self->{$url} = $response;
}
sub getPage{
my ( $self, $url ) = @_;
return $self->{$url};
}
1;
| |—Builders
| |—Counter.pm
Package Builder::Counter;
use lib "../../../Packages";
use strict;
use base 'Utils::Counter';
sub new {
my ( $class ) = @_;
my $self = {
_counter => Utils::Counter->new(),
};
bless $self, $class;
return $self;
}
sub build {
my $self = shift;
return $self->{_counter};
}
sub withIncrement {
my $self = shift;
my $params = @_;
$self->{_counter}->increment($params);
return $self;
}
sub withStart {
my $self = shift;
my $params = @_;
$self->{_counter}-startWith($params);
return $self;
}
sub withMaxIterations {
my $self = shift;
my $params = @_;
$self->{_counter}->maxIterations($params);
return $self;
}
1;
| |—Job
| |—Site
| |—Seek.t
#!/usr/bin/perl
use lib 'Packages';
use lib 't';
use strict;
use Test::More 0.88;
use Test::MockModule;
use Test::Utils::MockWeb;
require Job::Site::Seek;
######################################################################
# BEGIN MOCK OBJECTS DEFINITION
my $customWeb = Test::Utils::MockWeb->new();
my $module = new Test::MockModule('Utils::Web');
$module->mock(getPage => sub {
my ($self, $url) = @_ ;
return $customWeb->getPage($url)
});
# END MOCK OBJECTS DEFINITION
######################################################################
######################################################################
# BEGIN TESTS
# Test
subtest 'Test Processing Pages' => sub {
my $seek = new_ok( 'Job::Site::Seek' );
# Get getNextPageUrl
my $keywords = "TEST THIS";
my $url = $seek->url();
my $searchString = $seek->searchString();
$searchString =~ s/<<KEYWORDS>>/$keywords/;
my $expected_url = [
$url . $searchString =~ s/<<COUNTER>>/1/r,
$url . $searchString =~ s/<<COUNTER>>/2/r,
];
my $expected_content = [
"PAGE1",
"PAGE2",
];
# Setup Site
$seek->pagesToDownload(2);
$seek->keywords($keywords);
$customWeb->setPageResponse($expected_url->[0], $expected_content->[0]);
$customWeb->setPageResponse($expected_url->[1], $expected_content->[1]);
my $expected_result = $expected_content->[0] . $expected_content->[1];
is($seek->processSite(), $expected_result, "processSite ok");
};
# END TESTS
######################################################################
done_testing;
| |—JobAd.t
#!/usr/bin/perl
use lib "Packages";
use strict;
use Test::More 0.88;
use Job::Site;
require Job::JobAd;
######################################################################
# BEGIN TESTS
subtest 'Testing accessors' => sub {
my $jobAd = new_ok( 'Job::JobAd' );
my $test_value = "test_value";
# Test accessors
my $test_site = Job::Site->new();
$jobAd->site($test_site);
is($jobAd->site(), $test_site, "Site accessor ok");
$jobAd->id($test_value);
is($jobAd->id(), $test_value, "id accessor ok");
$jobAd->url($test_value);
is($jobAd->url(), $test_value, "url accessor ok");
$jobAd->shortDesc($test_value);
is($jobAd->shortDesc(), $test_value, "shortDesc accessor ok");
$jobAd->description($test_value);
is($jobAd->description(), $test_value, "description accessor ok");
};
# END TESTS
######################################################################
done_testing;
| |—Site.t
#!/usr/bin/perl
use lib 'Packages';
use lib 't';
use strict;
use Test::More 0.88;
use Test::MockModule;
use Test::Utils::MockWeb;
require Job::Site;
######################################################################
# BEGIN MOCK OBJECTS DEFINITION
my $customWeb = Test::Utils::MockWeb->new();
my $module = new Test::MockModule('Utils::Web');
$module->mock(getPage => sub {
my ($self, $url) = @_ ;
return $customWeb->getPage($url)
});
# END MOCK OBJECTS DEFINITION
######################################################################
######################################################################
# BEGIN TESTS
# Test
subtest 'Testing accessors' => sub {
my $site = new_ok( 'Job::Site' );
my $test_value = "test_value";
# Test accessors
$site->displayName($test_value);
is($site->displayName(), $test_value, "displayName accessor ok");
$site->url($test_value);
is($site->url(), $test_value, "url accessor ok");
$site->searchString($test_value);
is($site->searchString(), $test_value, "searchString accessor ok");
$site->adString($test_value);
is($site->adString(), $test_value, "adString accessor ok");
$site->keywords($test_value);
is($site->keywords(), $test_value, "keywords accessor ok");
$test_value = 100;
$site->pagesToDownload($test_value);
is($site->pagesToDownload(), $test_value, "pagesToDownload accessor ok");
my $test_counter = Utils::Counter->new();
$site->counter($test_counter);
is($site->counter(), $test_counter, "counter accessor ok");
};
subtest 'Test Processing Pages' => sub {
my $site = new_ok( 'Job::Site' );
my $keywords = "TEST THIS";
my $url = "my.com";
my $searchString = "/search=<<KEYWORDS>>&page=<<COUNTER>>";
my $expected_url = [
$url . "/search=" . $keywords . "&page=1",
$url . "/search=" . $keywords . "&page=2",
];
my $expected_content = [
"PAGE1",
"PAGE2",
];
# Setup Site
$site->url($url);
$site->searchString($searchString);
$site->pagesToDownload(2);
$site->keywords($keywords);
$customWeb->setPageResponse($expected_url->[0], $expected_content->[0]);
$customWeb->setPageResponse($expected_url->[1], $expected_content->[1]);
my $expected_result = $expected_content->[0] . $expected_content->[1];
is($site->processSite(), $expected_result, "processSite ok");
};
# END TESTS
######################################################################
done_testing;
Leave a Comment
My Developer Joy 