srctree

Gregory Mullen parent e94e203d 83faa963
add per bot rules

src/bot-detection.zig added: 5, removed: 2, total 3

@@ -118,6 +118,7 @@ pub fn robotsTxt(

compiled[i] = "User-agent: " ++ each.name ++ "\n" ++

(if (each.allow) "Allow" else "Disallow") ++ ": /\n" ++

(if (options.extra_rules) |er| er else "") ++

(if (each.extra) |ex| ex else "") ++

(if (each.delay) "Crawl-delay: 4\n\n" else "\n");

} else {

compiled[0] = "User-agent: *\n" ++

@@ -137,6 +138,7 @@ pub fn robotsTxt(

"User-agent: " ++ each.name ++ "\n" ++

(if (each.allow) "Allow" else "Disallow") ++ ": /\n" ++

(if (options.extra_rules) |er| er else "") ++

(if (each.extra) |ex| ex else "") ++

(if (each.delay) "Crawl-delay: 4\n\n" else "\n");

}

src/bot-detection/bots.zig added: 5, removed: 2, total 3

@@ -25,6 +25,7 @@ pub const TxtRules = struct {

name: []const u8,

allow: bool,

delay: bool = false,

extra: ?[]const u8 = null,

};

/// This isn't the final implementation, I'm just demoing some ideas