Commit 2ee66e45 authored by Alon Salant's avatar Alon Salant

Add bot detection for Facebook crawler,...

Add bot detection for Facebook crawler, https://developers.facebook.com/docs/sharing/webmasters/crawler)
parent f0a10c4c
......@@ -12,6 +12,7 @@ bot Wotbox/2.01 (+http://www.wotbox.com/bot/)
bot Mozilla/5.0 (compatible; AhrefsBot/3.1; +http://ahrefs.com/robot/)
bot Baiduspider+(+http://www.baidu.com/search/spider.htm)
bot Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)
bot facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)
# from https://developers.google.com/webmasters/mobile-sites/references/googlebot
mobile-bot DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
......
......@@ -45,6 +45,7 @@ sub devicedetect {
set req.http.X-UA-Device = "mobile-bot"; }
elsif (req.http.User-Agent ~ "(?i)(ads|google|bing|msn|yandex|baidu|ro|career|seznam|)bot" ||
req.http.User-Agent ~ "(?i)(baidu|jike|symantec)spider" ||
req.http.User-Agent ~ "(?i)facebookexternalhit" ||
req.http.User-Agent ~ "(?i)scanner" ||
req.http.User-Agent ~ "(?i)(web)crawler") {
set req.http.X-UA-Device = "bot"; }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment