I wrote a code:
use LWP::UserAgent;
use HTTP::Cookies;
use threads;
use threads::shared;
$| = 1;
$threads = 50;
my @urls : shared = loadf('url.txt');
my @thread_list = ();
$thread_list[$_] = threads->create(\&thread) for 0 .. $threads - 1;
$_->join for @thread_list;
thread();
sub thread
{
my ($web, $ck) = browser();
while(1)
{
my $url = shift @urls;
if(!$url)
{
last;
}
$code = $web->get($url)->code;
print "[+] $url - code: $code\n";
if($code == 200)
{
open F, ">>200.txt";
print F $url."\n";
close F;
}
elsif($code == 301)
{
open F, ">>301.txt";
print F $url."\n";
close F;
}
else
{
open F, ">>else.txt";
print F "$url code - $code\n";
close F;
}
}
}
sub loadf {
open (F, "<".$_[0]) or erroropen($_[0]);
chomp(my @data = <F>);
close F;
return @data;
}
sub browser
{
my $web = new LWP::UserAgent;
my $ck = new HTTP::Cookies;
$web->cookie_jar($ck);
$web->agent('Opera/9.80 (Windows 7; U; en) Presto/2.9.168 Version/11.50');
$web->timeout(5);
return $web, $ck;
}
After its working for some time physical storage is full.
Can u help me to re-write it with AnyEvent. I tried but my code didn't work. I read that it will help me to safe some memory.
Thanks a lot to any helpers.
