true, 'maxRedirects' => 5); $r = new HTTP_Request($start,$options); $r->setMethod(HTTP_REQUEST_METHOD_GET); $r->sendRequest(); $page = $r->getResponseBody(); $query = mysql_query("INSERT INTO urls (url_base,url_title,url_description,url_keywords,url_domain,url_crawled,url_added) VALUES('".$start."','".format_data(getTitle($page))."','".format_data(getDescription($page))."','".format_data(getKeywords($page))."','".format_data(strip_domain($url))."','Yes',now())") or die(mysql_error()); preg_match_all ("/\s+href\s*=\s*[\"\']?([^\s\"\']+)[\"\'\s]+/ims",$page,$matches); $url_array = $matches[0]; $url_count = count($url_array); for ($i=0;$i<$url_count;$i++) { if (preg_match('/http:\/\/(.*?)/i',$url_array[$i])) { $url = str_replace('href="',"",$url_array[$i]); $url = str_replace('"',"",$url); $url = trim($url); $url = strtolower($url); if (preg_match('/href=/i',$url)) { $url = str_replace('href=',"",$url); } $query = mysql_query("INSERT INTO urls (url_base,url_domain,url_crawled,url_added) VALUES('".format_data($url)."','".format_data(strip_domain($url))."','No',now())") or die(mysql_error()); } elseif (!preg_match('/mailto:(.*?)/i',$urlArray[$i])) { $url = str_replace('href="',"",$urlArray[$i]); $url = str_replace('"',"",$url); $url = trim(strtolower($url)); $current_URL = preg_replace("/([^\/])\/[^\/]+$/i", "\\1", $start); $url = $start . "/" . $url; $url = str_replace("/./", "/", $url); $url = preg_replace("/\/[\/]+/i", "/", $url); $url = str_replace("http:/", "http://", $url); $url = str_replace("&", "&", $url); $query = mysql_query("INSERT INTO urls (url_base,url_domain,url_crawled,url_added) VALUES('".format_data($url)."','".format_data(strip_domain($url))."','No',now())") or die(mysql_error()); } } function format_data($string) { return str_replace("'","\'",$string); } function strip_domain($url) { $pattern = '/^(http:\/\/)?([^\/]+)/i'; preg_match($pattern,$url,$domain); preg_match("/[^\.\/]+\.[^\.\/]+$/", $domain[2],$domain); return $domain[0]; } function getTitle($page) { $pgTitle = preg_match('/