0 ) {
foreach( $output as $item ) {
if( $item[1] ) { $urls[] = $item[1]; }
echo "Now Scraping : " . $item[1] . "\n";
scrapeit($item[1]);
//echo "$item[1]
";
}
}
}
}
function scrapeit($url)
{
$ch = curl_init();
$useragent="Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1";
curl_setopt($ch, CURLOPT_URL,$url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER,1);
curl_setopt($ch, CURLOPT_USERAGENT, $useragent);
$scrape = curl_exec ($ch);
$scrape = str_replace(chr(10), '', str_replace(chr(13), '', stripslashes($scrape)));
curl_close ($ch);
preg_match("%(.*)%", $scrape, $article, PREG_OFFSET_CAPTURE);
preg_match("%(.*)%",$scrape, $title, PREG_OFFSET_CAPTURE);
$article[0][0] = preg_replace('%%', "", $article[0][0]);
$newarticle = remove_html($article[0][0]);
$newtitle = remove_html($title[0][0]) . '.txt';
$newtitle = str_replace(chr(32), '-', str_replace(chr(63), '',$newtitle));
//echo "title is : " . $newtitle . "\n";
$fp = fopen($newtitle, "w+");
fwrite($fp, $newarticle);
}
function remove_html($html)
{
$search = array ("']*?>.*?'si", // Strip out javascript
"'<[\/\!]*?[^<>]*?>'si", // Strip out html tags
"'([\r\n])[\s]+'", // Strip out white space
"'&(quot|#34);