0 ) {
foreach( $output as $item ) {
if( $item[1] ) { $urls[] = $item[1]; }
echo "Now Scraping : " . $item[1] . "\n";
scrapeit($item[1]);
//echo "$item[1]
";
}
}
}
}
function scrapeit($url)
{
$ch = curl_init();
$useragent="Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1";
curl_setopt($ch, CURLOPT_URL,$url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER,1);
curl_setopt($ch, CURLOPT_USERAGENT, $useragent);
$scrape = curl_exec ($ch);
$scrape = str_replace(chr(10), '', str_replace(chr(13), '', stripslashes($scrape)));
curl_close ($ch);
preg_match("%(.*)%", $scrape, $article, PREG_OFFSET_CAPTURE);
preg_match("%(.*)%",$scrape, $title, PREG_OFFSET_CAPTURE);
$article[0][0] = preg_replace('%%', "", $article[0][0]);
$newarticle = remove_html($article[0][0]);
$newtitle = remove_html($title[0][0]) . '.txt';
$newtitle = str_replace(chr(32), '-', str_replace(chr(63), '',$newtitle));
//echo "title is : " . $newtitle . "\n";
$fp = fopen($newtitle, "w+");
fwrite($fp, $newarticle);
}
function remove_html($html)
{
$search = array ("']*?>.*?'si", // Strip out javascript
"'<[\/\!]*?[^<>]*?>'si", // Strip out html tags
"'([\r\n])[\s]+'", // Strip out white space
"'&(quot|#34);'i", // Replace html entities
"'&(amp|#38);'i",
"'&(lt|#60);'i",
"'&(gt|#62);'i",
"'&(nbsp|#160);'i",
"'&(iexcl|#161);'i",
"'&(cent|#162);'i",
"'&(pound|#163);'i",
"'&(copy|#169);'i",
"'&#(\d+);'e"); // evaluate as php
$replace = array ("",
"",
"\\1",
"\"",
"&",
"<",
">",
" ",
chr(161),
chr(162),
chr(163),
chr(169),
"chr(\\1)");
$html = preg_replace ($search, $replace, $html);
return $html;
}
?>