Iltud
09-13-2003, 02:19 PM
This mod allows phpDid to index sites through proxies.
BUT, there are some constraints :
[list=1]
the database must run at localhost
FTP features of phpDig won't work
this proxy feature works in exclusive mode : Either you decide to use it, or not. The detection of local sites isn't automatic.
perhaps phpDig won't use cookies and authentication on remote sites (to check ?)
[/list=1]
The line numbers given here are approximative and based on the version 1.6.2 of phpDig.
In the file /includes/config.php :
[list=2]
Add these 3 lines (anywhere) :
define('PHPDIG_USE_PROXY',false); // Use proxy-mode ? true or false
define('PHPDIG_PROXY_URL','my.proxy.net'); // Proxy url
define('PHPDIG_PROXY_PORT',3128); // Proxy port
[/list=2]
In the file /admin/robot_functions.php :
[list=3]
Replace the line #264 :
$fp = @fsockopen($host,$port);
by this section :
if(PHPDIG_USE_PROXY==true && PHPDIG_PROXY_URL!="")
{
$fp = @fsockopen(PHPDIG_PROXY_URL,PHPDIG_PROXY_PORT);
} else {
$fp = @fsockopen($host,$port);
}
At lines #285-287, replace :
$request =
"HEAD $path HTTP/1.1\\n"
."Host: $host$sport\\n"
by :
$request =
"HEAD ".(PHPDIG_USE_PROXY==true?$url:$path)." HTTP/1.0\\r\\n"
.(PHPDIG_USE_PROXY==true?"":"Host: $host$sport\\r\\n")
At lines #352-353, replace :
$req1 = "HEAD $path HTTP/1.1\\n"
."Host: $host$sport\\n"
by :
$req1 = "HEAD ".(PHPDIG_USE_PROXY==true?$newpath:$path)." HTTP/1.0\\r\\n"
.(PHPDIG_USE_PROXY==true?"":"Host: $host$sport\\r\\n")
Delete the line #630 :
$file_content = @file($uri);
At lines #638-641, replace :
if (is_array($file_content)) {
fwrite($f_handler,implode('',$file_content));
}
fclose($f_handler);
by this section :
if(PHPDIG_USE_PROXY==true && PHPDIG_PROXY_URL!="")
{
$request = "GET $uri HTTP/1.0\\r\\n"
."Accept: */*\\r\\n"
."Accept-Charset: iso-8859-1\\r\\n"
."Accept-Encoding: identity\\r\\n"
."User-Agent: PhpDig/".PHPDIG_VERSION." (PHP; MySql)\\r\\n\\r\\n";
$fp = fsockopen(PHPDIG_PROXY_URL,PHPDIG_PROXY_PORT);
if ($fp) {
fputs($fp,$request);
$c = "\\0";
while (!feof($fp) && $c != "\\r\\n")
$c = fgets($fp, 512); // 512 can be changed
while (!feof($fp))
{
$answer = fgets($fp,255); // 255 can be changed
$answer = str_replace("\\r\\n","\\n",$answer);
fwrite($f_handler,$answer);
}
fclose($fp);
fclose($f_handler);
}
}
else
{
$file_content = @file($uri);
if (is_array($file_content)) {
fwrite($f_handler,implode('',$file_content));
}
fclose($f_handler);
}
[/list=3]
That's all folks :-)
I hope that I didn't forget anything.
Any comments or bugfixes/enhancements are welcome.
BUT, there are some constraints :
[list=1]
the database must run at localhost
FTP features of phpDig won't work
this proxy feature works in exclusive mode : Either you decide to use it, or not. The detection of local sites isn't automatic.
perhaps phpDig won't use cookies and authentication on remote sites (to check ?)
[/list=1]
The line numbers given here are approximative and based on the version 1.6.2 of phpDig.
In the file /includes/config.php :
[list=2]
Add these 3 lines (anywhere) :
define('PHPDIG_USE_PROXY',false); // Use proxy-mode ? true or false
define('PHPDIG_PROXY_URL','my.proxy.net'); // Proxy url
define('PHPDIG_PROXY_PORT',3128); // Proxy port
[/list=2]
In the file /admin/robot_functions.php :
[list=3]
Replace the line #264 :
$fp = @fsockopen($host,$port);
by this section :
if(PHPDIG_USE_PROXY==true && PHPDIG_PROXY_URL!="")
{
$fp = @fsockopen(PHPDIG_PROXY_URL,PHPDIG_PROXY_PORT);
} else {
$fp = @fsockopen($host,$port);
}
At lines #285-287, replace :
$request =
"HEAD $path HTTP/1.1\\n"
."Host: $host$sport\\n"
by :
$request =
"HEAD ".(PHPDIG_USE_PROXY==true?$url:$path)." HTTP/1.0\\r\\n"
.(PHPDIG_USE_PROXY==true?"":"Host: $host$sport\\r\\n")
At lines #352-353, replace :
$req1 = "HEAD $path HTTP/1.1\\n"
."Host: $host$sport\\n"
by :
$req1 = "HEAD ".(PHPDIG_USE_PROXY==true?$newpath:$path)." HTTP/1.0\\r\\n"
.(PHPDIG_USE_PROXY==true?"":"Host: $host$sport\\r\\n")
Delete the line #630 :
$file_content = @file($uri);
At lines #638-641, replace :
if (is_array($file_content)) {
fwrite($f_handler,implode('',$file_content));
}
fclose($f_handler);
by this section :
if(PHPDIG_USE_PROXY==true && PHPDIG_PROXY_URL!="")
{
$request = "GET $uri HTTP/1.0\\r\\n"
."Accept: */*\\r\\n"
."Accept-Charset: iso-8859-1\\r\\n"
."Accept-Encoding: identity\\r\\n"
."User-Agent: PhpDig/".PHPDIG_VERSION." (PHP; MySql)\\r\\n\\r\\n";
$fp = fsockopen(PHPDIG_PROXY_URL,PHPDIG_PROXY_PORT);
if ($fp) {
fputs($fp,$request);
$c = "\\0";
while (!feof($fp) && $c != "\\r\\n")
$c = fgets($fp, 512); // 512 can be changed
while (!feof($fp))
{
$answer = fgets($fp,255); // 255 can be changed
$answer = str_replace("\\r\\n","\\n",$answer);
fwrite($f_handler,$answer);
}
fclose($fp);
fclose($f_handler);
}
}
else
{
$file_content = @file($uri);
if (is_array($file_content)) {
fwrite($f_handler,implode('',$file_content));
}
fclose($f_handler);
}
[/list=3]
That's all folks :-)
I hope that I didn't forget anything.
Any comments or bugfixes/enhancements are welcome.