File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change 11baseURL = ' https://raim.dev/'
22languageCode = ' en-us'
33title = ' TIL of Raimbek'
4+ enableRobotsTXT = true
45
56theme = " til"
67
8+ [permalinks ]
9+ posts = ' /posts/:title/'
10+
711[caches ]
812 [caches .images ]
913 dir = ' :cacheDir/images'
Original file line number Diff line number Diff line change 1- {{- if (default false site .Params .robotstxt .blockCC ) }}
2- # Block Common Crawl (https://commoncrawl.org/ccbot)
3- User-agent: CCBot
4- Disallow: /
5-
6- {{ end - }}
7-
8- {{- if (default false site .Params .robotstxt .blockAI ) }}
9- # Block generative AI/LLM bots (source: https://github.com/ai-robots-txt/ai.robots.txt)
10- User-agent: AI 2 Bot
11- User-agent: Ai 2 Bot-Dolma
12- User-agent: Amazonbot
13- User-agent: anthropic-ai
14- User-agent: Applebot
15- User-agent: Applebot-Extended
16- User-agent: Bytespider
17- User-agent: ChatGPT-User
18- User-agent: Claude-Web
19- User-agent: ClaudeBot
20- User-agent: cohere-ai
21- User-agent: Diffbot
22- User-agent: FacebookBot
23- User-agent: facebookexternalhit
24- User-agent: FriendlyCrawler
25- User-agent: Google-Extended
26- User-agent: GoogleOther
27- User-agent: GoogleOther-Image
28- User-agent: GoogleOther-Video
29- User-agent: GPTBot
30- User-agent: iaskspider /2.0
31- User-agent: ICC-Crawler
32- User-agent: ImagesiftBot
33- User-agent: img 2 dataset
34- User-agent: ISSCyberRiskCrawler
35- User-agent: Kangaroo Bot
36- User-agent: Meta-ExternalAgent
37- User-agent: Meta-ExternalFetcher
38- User-agent: OAI-SearchBot
39- User-agent: omgili
40- User-agent: omgilibot
41- User-agent: PerplexityBot
42- User-agent: PetalBot
43- User-agent: Scrapy
44- User-agent: Sidetrade indexer bot
45- User-agent: Timpibot
46- User-agent: VelenPublicWebCrawler
47- User-agent: Webzio-Extended
48- User-agent: YouBot
49- Disallow: /
50- {{ end - }}
51-
521User-agent: *
532Allow: /
543
You can’t perform that action at this time.
0 commit comments