On our site, this error occurs every 5 minutes. And I don’t know where his name is from. can someone help us. we need robots.txt without this error.
our application automatically calls http://www.xyzName.com/content/images/thumbs/robots.txt
and show exceptions below
System.Web.Routing.UrlRoutingModule does not implement IHttpHandlerFactory or IHttpHandler.
Description: An unhandled exception occurred during the execution of the current web request. View the stack trace for error information and where it originated in the code.
This is a method that automatically generates a robots.txt file when this method is called. but the problem is where robots.txt is called automatically in the application.
public ActionResult RobotsTextFile()
{
if (_storeContext.CurrentStore.Url.Contains("tk"))
{
const string newLine = "\r\n";
var sb = new StringBuilder();
sb.Append("User-agent: *");
sb.Append(newLine);
sb.Append("Disallow: /");
Response.ContentType = "text/plain";
Response.Write(sb.ToString());
}
else
{
var disallowPaths = new List<string>
{
"/bin/",
"/content/files/",
"/content/files/exportimport/",
"/country/getstatesbycountryid",
"/install",
"/setproductreviewhelpfulness",
};
var localizableDisallowPaths = new List<string>
{
"/addproducttocart/catalog/",
"/addproducttocart/details/",
"/backinstocksubscriptions/manage",
"/boards/forumsubscriptions",
"/boards/forumwatch",
"/boards/postedit",
"/boards/postdelete",
"/boards/postcreate",
"/boards/topicedit",
"/boards/topicdelete",
"/boards/topiccreate",
"/boards/topicmove",
"/boards/topicwatch",
"/cart",
"/checkout",
"/checkout/billingaddress",
"/checkout/completed",
"/checkout/confirm",
"/checkout/shippingaddress",
"/checkout/shippingmethod",
"/checkout/paymentinfo",
"/checkout/paymentmethod",
"/clearcomparelist",
"/compareproducts",
"/customer/avatar",
"/customer/activation",
"/customer/addresses",
"/customer/changepassword",
"/customer/checkusernameavailability",
"/customer/downloadableproducts",
"/customer/info",
"/deletepm",
"/emailwishlist",
"/inboxupdate",
"/newsletter/subscriptionactivation",
"/onepagecheckout",
"/order/history",
"/orderdetails",
"/passwordrecovery/confirm",
"/poll/vote",
"/privatemessages",
"/returnrequest",
"/returnrequest/history",
"/rewardpoints/history",
"/sendpm",
"/sentupdate",
"/shoppingcart/productdetails_attributechange",
"/subscribenewsletter",
"/topic/authenticate",
"/viewpm",
"/uploadfileproductattribute",
"/uploadfilecheckoutattribute",
"/wishlist",
};
const string newLine = "\r\n";
var sb = new StringBuilder();
sb.Append("User-agent: *");
sb.Append(newLine);
if (_localizationSettings.SeoFriendlyUrlsForLanguagesEnabled)
{
foreach (var language in _languageService.GetAllLanguages(storeId: _storeContext.CurrentStore.Id))
{
sb.AppendFormat("Sitemap: {0}{1}/sitemap.xml", _storeContext.CurrentStore.Url, language.UniqueSeoCode);
sb.Append(newLine);
}
}
else
{
sb.AppendFormat("Sitemap: {0}sitemap.xml", _storeContext.CurrentStore.Url);
sb.Append(newLine);
}
foreach (var path in disallowPaths)
{
sb.AppendFormat("Disallow: {0}", path);
sb.Append(newLine);
}
foreach (var path in localizableDisallowPaths)
{
sb.AppendFormat("Disallow: {0}", path);
sb.Append(newLine);
}
if (_localizationSettings.SeoFriendlyUrlsForLanguagesEnabled)
{
foreach (var language in _languageService.GetAllLanguages(storeId: _storeContext.CurrentStore.Id))
{
foreach (var path in localizableDisallowPaths)
{
sb.AppendFormat("Disallow: {0}{1}", language.UniqueSeoCode, path);
sb.Append(newLine);
}
}
}
Response.ContentType = "text/plain";
Response.Write(sb.ToString());
}
return null;
}
RouteProvider , .
routes.MapRoute("robots.txt","robots.txt",new { controller = "Common", action ="RobotsTextFile" },new[] { "Nop.Web.Controllers" });
5 . Amazon CDN, .
, " http://www.xyzName.com/content/images/thumbs/robots.txt 'url?