$urls=array("http://google.com","http://yahoo.com"); // could be thousands in this list $mh = curl_multi_init(); // create queue of curl handles $handles=array(); $mh=curl_multi_setup_queue($urls,$handles); // execute requests curl_multi_perform($mh); // fetch data $data=array(); $failed=array(); curl_multi_data($mh,$handles,$data,$failed); foreach($data as $content) { if(isset($failed[$i])) { echo "ERROR $failed[$i]\n"; continue; } echo "$content\n"; } / * FUNCTIONS */ function curl_multi_setup_queue($urls,&$c) { $mh = curl_multi_init(); if(!$mh) die("CANT GET MULTIHANDLE\n"); // create work queue of curl handles $c=array(); foreach($urls as $i=>$url) { $c[$i]=curl_setup($url); if(!$c[$i]) { err(curl_log_error($c[$i],"curl_setup($url) failed | ")); continue; } $rc=curl_multi_add_handle($mh,$c[$i]); if($rc!=0) { err(curl_log_error($c[$i],"curl_multi_add_handle() failed | $rc | ")); continue; } } return $mh; } function curl_multi_perform(&$mh) { if(!$mh) return FALSE; // start performing the request while(($mrc=CURLM_CALL_MULTI_PERFORM==curl_multi_exec($mh, $active))); while ($active and $mrc == CURLM_OK) { // wait for network if (curl_multi_select($mh) != -1) { // read/write all data while (($mrc=CURLM_CALL_MULTI_PERFORM==curl_multi_exec($mh, $active))); } } if ($mrc != CURLM_OK) { err("Curl multi read error $mrc\n"); } return; } function curl_multi_data(&$mh,&$handles,&$data,&$failed) { // get data foreach ($handles as $i=>$h) { // look for err $err = curl_error($h); if ($err) { debug("CURL ERROR: $err"); $e=curl_log_error($h); $data[$i]=''; $failed[$i]=$e; } else { $data[$i]=curl_multi_getcontent($h); } curl_multi_remove_handle($mh,$h); curl_close($h); } return; } function curl_log_error($ch) { $r=curl_error($ch); return $r; }