Sitemap: https://www.bengo4.com/topics/feed/googlenews
Sitemap: https://www.bengo4.com/times/sitemap.xml

User-agent: Googlebot-News
Disallow: /
Allow: /topics/
Allow: */n_*
Allow: /life/
Allow: */li_*

User-agent: Mediapartners-Google
Disallow:

User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Meta-ExternalAgent
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Perplexity-User
Disallow: /questions/
Disallow: /private/
Disallow: /sp/private/
Disallow: /lawyer/
Disallow: /login/
Disallow: /register/
Disallow: /*/contact/

User-agent: GPTBot
User-agent: ClaudeBot
User-agent: OAI-SearchBot
User-agent: PerplexityBot
Disallow: /questions/
Disallow: /private/
Disallow: /sp/private/
Disallow: /lawyer/
Disallow: /login/
Disallow: /register/
Disallow: /*/contact/
Crawl-delay: 10

User-agent: grapeshot
User-agent: SemrushBot
User-agent: YisouSpider
User-agent: MJ12bot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: TTD-Content
User-agent: dotbot
User-agent: AwarioBot
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: trendictionbot
User-agent: Yandex
User-agent: YandexCalendar
User-agent: YandexMobileBot
User-agent: ImagesiftBot
User-agent: barkrowler
User-agent: peer39_crawler
User-agent: Gensparkbot
User-agent: Grok
User-agent: Grok-DeepSearch
User-agent: GrokBot
User-agent: xAI-Grok
User-agent: Bytespider
User-agent: Diffbot
User-agent: Timpibot
User-agent: omgili
User-agent: omgilibot
User-agent: Webzio-Extended
User-agent: FacebookBot
User-agent: meta-externalfetcher
User-agent: AI2Bot
User-agent: AI2Bot-Dolma
User-agent: ICC-Crawler
User-agent: FriendlyCrawler
User-agent: magpie-crawler
User-agent: PanguBot
User-agent: SBIntuitionsBot
User-agent: img2dataset
User-agent: news-please
Disallow: /

User-agent: bingbot
Disallow: /74013581/
Disallow: /api/
Disallow: /c_3/api/
Disallow: /fudosan/api/
Allow: /api/bbs/rss
Allow: /api/img/
Disallow: /bbs/vote/
Disallow: /js/robotsDeny/
Disallow: /js/pc/robotsDeny/
Disallow: /js/sp/robotsDeny/
Disallow: /js/test/robotsDeny/
Disallow: /sp/search/result/*/
Disallow: /sp/user/
Disallow: /result/
Disallow: /topics/jump/
Disallow: /special/
Disallow: /module/
Disallow: /voice/*
Allow: /voice/$
Disallow: /sp/voice/*
Allow: /sp/voice/$
Disallow: /lawyer/private/*
Disallow: /*/contact/

User-agent: *
Disallow: /74013581/
Disallow: /api/
Disallow: /c_3/api/
Disallow: /fudosan/api/
Allow: /api/bbs/rss
Allow: /api/img/
Disallow: /bbs/vote/
Disallow: /js/robotsDeny/
Disallow: /js/pc/robotsDeny/
Disallow: /js/sp/robotsDeny/
Disallow: /js/test/robotsDeny/
Disallow: /sp/search/result/*/
Disallow: /sp/user/
Disallow: /result/
Disallow: /topics/jump/
Disallow: /special/
Disallow: /module/
Disallow: /voice/*
Allow: /voice/$
Disallow: /sp/voice/*
Allow: /sp/voice/$
Disallow: /lawyer/private/*
Disallow: /*/contact/
Crawl-delay: 5