srctree

Gregory Mullen parent 05656209 a96d0414
add robotsTxt helper

I consider this partial, I would like to provide more defaults, but that will require a bit more research before I'm willing to commit it.
inlinesplit
src/bot-detection.zig added: 80, removed: 11, total 69
@@ -83,14 +83,60 @@ const rules = struct {
};
};
 
pub const browsers = @import("bot-detection/browsers.zig");
 
test BotDetection {
_ = std.testing.refAllDecls(@This());
_ = &browsers;
}
 
const std = @import("std");
pub fn robotsTxt(
comptime default_allow: bool,
delay_int: comptime_int,
comptime robots: []const bots.TxtRules,
) Router.Match {
const EP = struct {
const delay = std.fmt.comptimePrint("Crawl-delay: {}\n", .{delay_int});
const robots_txt: []const u8 = brk: {
var compiled: []const u8 = "User-agent: *\n" ++
(if (delay_int > 0) delay else "") ++
(if (default_allow) "Allow: /\n\n" else "Disallow: /\n\n");
for (robots) |each| {
compiled = compiled ++
"User-agent: " ++ each.name ++ "\n" ++
(if (each.allow) "Allow" else "Disallow") ++ ": /\n" ++
(if (each.delay) "Crawl-delay: 4\n\n" else "\n");
}
 
break :brk compiled;
};
 
pub fn endpoint(f: *Frame) Router.Error!void {
f.status = .ok;
f.content_type = .@"text/plain";
f.sendHeaders() catch |err| switch (err) {
error.HeadersFinished => unreachable,
inline else => |e| return e,
};
 
try f.sendRawSlice("\r\n");
try f.sendRawSlice(robots_txt);
}
};
 
return Router.ANY("robots.txt", EP.endpoint);
}
 
test robotsTxt {
// TODO mock a full request frame
_ = robotsTxt(true, 4, &[_]bots.TxtRules{.{ .name = "googlebot", .allow = false }});
}
 
pub const browsers = @import("bot-detection/browsers.zig");
pub const bots = @import("bot-detection/bots.zig");
 
const Router = @import("router.zig");
const Frame = @import("frame.zig");
const UA = @import("user-agent.zig");
const Request = @import("request.zig");
 
const std = @import("std");
const startsWith = std.mem.startsWith;
 
filename was Deleted added: 80, removed: 11, total 69
@@ -0,0 +1,17 @@
pub const Rules = struct {
pub fn rfc9110(ua: UA, r: *const Request, score: *f16) !void {
_ = ua;
_ = r;
_ = score;
if (false) {}
}
};
 
pub const TxtRules = struct {
name: []const u8,
allow: bool,
delay: bool = false,
};
 
const UA = @import("../user-agent.zig");
const Request = @import("../request.zig");
 
src/content-type.zig added: 80, removed: 11, total 69
@@ -1,13 +1,16 @@
base: ContentBase,
parameter: ?CharSet = null,
 
const ContentType = @This();
 
pub const @"text/plain": ContentType = .{ .base = .{ .text = .plain } };
pub const @"text/html": ContentType = .{ .base = .{ .text = .html }, .parameter = .@"utf-8" };
 
pub const default: ContentType = .{
.base = .{ .text = .html },
.parameter = .@"utf-8",
};
 
const ContentType = @This();
 
pub const ContentBase = union(Base) {
application: Application,
audio: Audio,
 
src/user-agent.zig added: 80, removed: 11, total 69
@@ -243,7 +243,7 @@ pub fn init(ua_str: []const u8) UserAgent {
}
 
const Request = @import("request.zig");
const BotDetection = @import("bot-detection.zig");
pub const BotDetection = @import("bot-detection.zig");
 
const BOTDETC_ENABLED: bool = verse_buildopts.botdetection or builtin.is_test;
 
 
src/verse.zig added: 80, removed: 11, total 69
@@ -20,6 +20,9 @@ pub const fileOnDisk = @import("static-file.zig").fileOnDisk;
const endpoint = @import("endpoint.zig");
pub const Endpoints = endpoint.Endpoints;
 
// TODO this needs a better home (namespace)
pub const robotsTxt = Request.UserAgent.BotDetection.robotsTxt;
 
const errors = @import("errors.zig");
pub const RoutingError = Router.RoutingError;
pub const ServerError = errors.ServerError;