@@ -130,11 +130,13 @@ pub fn robotsTxt(
const robots_txt: []const u8 = brk: {
var compiled: []const u8 = "User-agent: *\n" ++
(if (options.delay > 0) delay else "") ++
(if (options.default_allow) "Allow: /\n\n" else "Disallow: /\n\n");
(if (options.default_allow) "Allow: /\n\n" else "Disallow: /\n\n") ++
(if (options.extra_rules) |er| er else "");
for (robots) |each| {
compiled = compiled ++
"User-agent: " ++ each.name ++ "\n" ++
(if (each.allow) "Allow" else "Disallow") ++ ": /\n" ++
(if (options.extra_rules) |er| er else "") ++
(if (each.delay) "Crawl-delay: 4\n\n" else "\n");
}