### robots.txt for John Benjamins

# setting a delay although it is not repected by some crawlers
crawl-delay: 10

# tentatively reenabling semrush bot 2021-06-18 -p
#User-agent: SemrushBot
#Disallow: /

User-Agent: *
Disallow: /content*/catalog/
Disallow: /catalog*/content/
Disallow: /catalog*/catalog/
Disallow: /content*/content/
Disallow: */search
Disallow: /series
Disallow: */resources
Disallow: /feed
Disallow: /jbp/temp/
Disallow: /claire
Disallow: /online/login
Disallow: /online/*/articles*/articles
Disallow: /online/*/list*/list
Disallow: /*?*
Disallow: /cgi-bin
Disallow: /cbc/sc
Disallow: /content/about/team
# no crawling for bibs at the moment
Disallow: /online/ebop
Disallow: /online/etsb
Disallow: /online/emetbib