Script #1
"http://www.stayinvisible.com/index.pl/proxy_list?order=type%20asc&offset=0" ,
'siteRoot' => "http://www.stayinvisible.com" ,
'nextPage' => '\[Next >>\]' ,
'sleepTime' => 3 ,
'proxiesFile' => "proxiesdb.txt" ,
'dbDelim' => "\t" ,
'stripChars' => "[^A-Za-z]" ,
'dataRow' =>
'
\s*' .
'| (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) | \s*' .
'(\d+) | \s*' .
'([\w ]+) | \s*' .
'([\w ]+) | \s*' .
'(\d{4}-\d{2}-\d{2}) | \s*' .
'.*? | \s*' .
'.*? | \s*' .
'
'
);
$url = $GLOBALS['INI']['URL'];
$proxyFile = fopen( $GLOBALS['INI']['proxiesFile'] , 'w' );
while( true ) {
$thisPage = file_get_contents( $url );
preg_match_all( "|{$GLOBALS['INI']['dataRow']}|" , $thisPage , $dataMatches );
//go through all matches
for ($i=0; $i < sizeof($dataMatches[0]); $i++) {
$resArray = array(
'host' => $dataMatches[1][$i] ,
'port' => $dataMatches[2][$i] ,
'type' => strtolower( preg_replace( "|{$GLOBALS['INI']['stripChars']}|" , "" ,$dataMatches[3][$i] ) ) ,
'country' => strtolower( preg_replace( "|{$GLOBALS['INI']['stripChars']}|" , "" ,$dataMatches[4][$i] ) ) ,
'date' => $dataMatches[5][$i] ,
);
fwrite( $proxyFile , implode( $GLOBALS['INI']['dbDelim'] , $resArray) . "\n" );
}
$matchCount = preg_match( "|{$GLOBALS['INI']['nextPage']}|" , $thisPage , $matches );
$thisPath = str_replace( " " , "%20" , $matches[1] );
$url = $GLOBALS['INI']['siteRoot'] . $thisPath;
if( !(bool)$matchCount ) {break;}
sleep( $GLOBALS['INI']['sleepTime'] );
}
fclose( $proxyFile );
?>
Script #2
"proxiesdb.txt" ,
'parsedFile' => "proxies.txt" ,
'dbDelim' => "\t" ,
'showFields' => array(0,1) ,
'fieldDelim' => ":" ,
'maxLineLength' => 4096 ,
'typeIndex' => 2 ,
'merge' => true ,
'locationIndex' => 3 ,
'showTypes' => array(
'anonymous' ,
'highanonymity'
) ,
'showLocations' => array(
'unitedstates'
)
);
$proxies = fopen( $GLOBALS['INI']['proxiesFile'] , 'r' );
while( !feof($proxies) ) {
$thisProxyLine = rtrim( fgets( $proxies , $GLOBALS['INI']['maxLineLength'] ) );
$thisProxy = explode( $GLOBALS['INI']['dbDelim'] , $thisProxyLine );
//check for the right fields
$typeKey = array_search( $thisProxy[$GLOBALS['INI']['typeIndex']] , $GLOBALS['INI']['showTypes']);
if( $typeKey === false ) {continue;}
$locationKey = array_search( $thisProxy[$GLOBALS['INI']['locationIndex']] , $GLOBALS['INI']['showLocations']);
if( $locationKey === false ) {continue;}
$tempArray = array();
for($j = 0; $j < sizeof($GLOBALS['INI']['showFields']); $j++) {
$thisProxyIndex = $GLOBALS['INI']['showFields'][$j];
$tempArray[] = $thisProxy[$thisProxyIndex];
}
$dataStr = implode( $GLOBALS['INI']['fieldDelim'] , $tempArray );
//open the results file
$openType = $GLOBALS['INI']['merge'] ? 'a+' : 'w+';
$pdb = fopen( $GLOBALS['INI']['parsedFile'] , $openType );
//compare
while( !feof($pdb) ) {
$thisParsedLine = rtrim( fgets( $pdb , $GLOBALS['INI']['maxLineLength'] ) );
//if the line already exists, then we go on to the next one
if( $thisParsedLine == $dataStr ) {
continue 2;
}
}
fwrite( $pdb , $dataStr . "\n" );
fclose( $pdb );
}
fclose( $proxies );
?>