here are new content for robots.php file
Quote:
<?php
header('Content-Type: text/plain');
echo "User-agent: IRLbot\n";
echo "Crawl-delay: 3600\n";
echo "\n";
echo "User-agent: *\n";
echo "Disallow: /next/\n";
echo "\n";
echo "# har har\n";
echo "User-agent: *\n";
echo "Disallow: /activate/\n";
echo "\n";
echo "User-agent: *\n";
echo "Disallow: /wp-login.php\n";
echo "\n";
echo "User-agent: *\n";
echo "Disallow: /wp-admin/\n";
echo "\n";
echo "User-agent: *\n";
echo "Disallow: /signup/\n";
echo "\n";
echo "User-agent: *\n";
echo "Disallow: /tag/\n";
echo "\n";
echo "# MT refugees\n";
echo "User-agent: *\n";
echo "Disallow: /cgi-bin/\n";
echo "\n";
echo "User-agent: *\n";
echo "Disallow:\n";
echo "\n";
echo "Sitemap: http://";
echo $_SERVER['HTTP_HOST'];
echo "/sitemap.xml";
?>
|