Skip to content
Merged
99 changes: 99 additions & 0 deletions .github/workflows/e2e-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,105 @@ jobs:
BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
kill `cat LPD.pid` `cat PROXY.id`

# e2e tests w/ web-bot-auth configuration on.
wba-demo-scripts:
name: wba-demo-scripts
needs: zig-build-release

runs-on: ubuntu-latest
timeout-minutes: 15

steps:
- uses: actions/checkout@v4
with:
repository: 'lightpanda-io/demo'
fetch-depth: 0

- run: npm install

- name: download artifact
uses: actions/download-artifact@v4
with:
name: lightpanda-build-release

- run: chmod a+x ./lightpanda

- run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem

- name: run end to end tests
run: |
./lightpanda serve \
--web_bot_auth_key_file private_key.pem \
--web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
--web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
& echo $! > LPD.pid
go run runner/main.go
kill `cat LPD.pid`

- name: build proxy
run: |
cd proxy
go build

- name: run end to end tests through proxy
run: |
./proxy/proxy & echo $! > PROXY.id
./lightpanda serve \
--web_bot_auth_key_file private_key.pem \
--web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
--web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
--http_proxy 'http://127.0.0.1:3000' \
& echo $! > LPD.pid
go run runner/main.go
kill `cat LPD.pid` `cat PROXY.id`

- name: run request interception through proxy
run: |
export PROXY_USERNAME=username PROXY_PASSWORD=password
./proxy/proxy & echo $! > PROXY.id
./lightpanda serve & echo $! > LPD.pid
URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js
BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
kill `cat LPD.pid` `cat PROXY.id`

wba-test:
name: wba-test
needs: zig-build-release

env:
LIGHTPANDA_DISABLE_TELEMETRY: true

runs-on: ubuntu-latest
timeout-minutes: 5

steps:
- uses: actions/checkout@v4
with:
repository: 'lightpanda-io/demo'
fetch-depth: 0

- run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem

- name: download artifact
uses: actions/download-artifact@v4
with:
name: lightpanda-build-release

- run: chmod a+x ./lightpanda

- name: run wba test
run: |
node webbotauth/validator.js &
VALIDATOR_PID=$!
sleep 1

./lightpanda fetch http://127.0.0.1:8989/ \
--web_bot_auth_key_file private_key.pem \
--web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
--web_bot_auth_domain ${{ vars.WBA_DOMAIN }}

wait $VALIDATOR_PID

cdp-and-hyperfine-bench:
name: cdp-and-hyperfine-bench
needs: zig-build-release
Expand Down
52 changes: 52 additions & 0 deletions src/Config.zig
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ const Allocator = std.mem.Allocator;
const log = @import("log.zig");
const dump = @import("browser/dump.zig");

const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config;

pub const RunMode = enum {
help,
fetch,
Expand Down Expand Up @@ -161,6 +163,17 @@ pub fn cdpTimeout(self: *const Config) usize {
};
}

pub fn webBotAuth(self: *const Config) ?WebBotAuthConfig {
return switch (self.mode) {
inline .serve, .fetch, .mcp => |opts| WebBotAuthConfig{
.key_file = opts.common.web_bot_auth_key_file orelse return null,
.keyid = opts.common.web_bot_auth_keyid orelse return null,
.domain = opts.common.web_bot_auth_domain orelse return null,
},
.help, .version => null,
};
}

pub fn maxConnections(self: *const Config) u16 {
return switch (self.mode) {
.serve => |opts| opts.cdp_max_connections,
Expand Down Expand Up @@ -227,6 +240,10 @@ pub const Common = struct {
log_format: ?log.Format = null,
log_filter_scopes: ?[]log.Scope = null,
user_agent_suffix: ?[]const u8 = null,

web_bot_auth_key_file: ?[]const u8 = null,
web_bot_auth_keyid: ?[]const u8 = null,
web_bot_auth_domain: ?[]const u8 = null,
};

/// Pre-formatted HTTP headers for reuse across Http and Client.
Expand Down Expand Up @@ -334,6 +351,14 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void {
\\--user_agent_suffix
\\ Suffix to append to the Lightpanda/X.Y User-Agent
\\
\\--web_bot_auth_key_file
\\ Path to the Ed25519 private key PEM file.
\\
\\--web_bot_auth_keyid
\\ The JWK thumbprint of your public key.
\\
\\--web_bot_auth_domain
\\ Your domain e.g. yourdomain.com
;

// MAX_HELP_LEN|
Expand Down Expand Up @@ -855,5 +880,32 @@ fn parseCommonArg(
return true;
}

if (std.mem.eql(u8, "--web_bot_auth_key_file", opt)) {
const str = args.next() orelse {
log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_key_file" });
return error.InvalidArgument;
};
common.web_bot_auth_key_file = try allocator.dupe(u8, str);
return true;
}

if (std.mem.eql(u8, "--web_bot_auth_keyid", opt)) {
const str = args.next() orelse {
log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_keyid" });
return error.InvalidArgument;
};
common.web_bot_auth_keyid = try allocator.dupe(u8, str);
return true;
}

if (std.mem.eql(u8, "--web_bot_auth_domain", opt)) {
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe these have well know meaning to users, but should we validate these? They're used very specifically, like this shouldn't include the protocol and shouldn't include the trailing slash. I realize that's what a "domain" is..but..

Also, keyid has to be base64 encoded. WE'll generate invalid JSON if this isn't well formed. Will that be an issue? Or will things just fail gracefully?

const str = args.next() orelse {
log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_domain" });
return error.InvalidArgument;
};
common.web_bot_auth_domain = try allocator.dupe(u8, str);
return true;
}

return false;
}
7 changes: 7 additions & 0 deletions src/browser/HttpClient.zig
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ const Notification = @import("../Notification.zig");
const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar;
const Robots = @import("../network/Robots.zig");
const RobotStore = Robots.RobotStore;
const WebBotAuth = @import("../network/WebBotAuth.zig");

const Allocator = std.mem.Allocator;
const ArenaAllocator = std.heap.ArenaAllocator;
Expand Down Expand Up @@ -702,6 +703,12 @@ fn makeRequest(self: *Client, conn: *Net.Connection, transfer: *Transfer) anyerr
try conn.secretHeaders(&header_list, &self.network.config.http_headers); // Add headers that must be hidden from intercepts
try conn.setHeaders(&header_list);

// If we have WebBotAuth, sign our request.
if (self.network.web_bot_auth) |*wba| {
const authority = URL.getHost(req.url);
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be the origin? The doc's say "This should be equal to the value of the Host header sent by the request." If you access http://localhost:1234/campfire-commerce/, the Host header is localhost:1234 not localhost.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This does return the port. If we used getHostName, it would strip the port.

try wba.signRequest(transfer.arena.allocator(), &header_list, authority);
}

// Add cookies.
if (header_list.cookies) |cookies| {
try conn.setCookies(cookies);
Expand Down
9 changes: 9 additions & 0 deletions src/browser/URL.zig
Original file line number Diff line number Diff line change
Expand Up @@ -1400,3 +1400,12 @@ test "URL: unescape" {
try testing.expectEqual("hello%2", result);
}
}

test "URL: getHost" {
try testing.expectEqualSlices(u8, "example.com:8080", getHost("https://example.com:8080/path"));
try testing.expectEqualSlices(u8, "example.com", getHost("https://example.com/path"));
try testing.expectEqualSlices(u8, "example.com:443", getHost("https://example.com:443/"));
try testing.expectEqualSlices(u8, "example.com", getHost("https://user:pass@example.com/page"));
try testing.expectEqualSlices(u8, "example.com:8080", getHost("https://user:pass@example.com:8080/page"));
try testing.expectEqualSlices(u8, "", getHost("not-a-url"));
}
10 changes: 10 additions & 0 deletions src/crypto.zig
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,8 @@ pub extern fn X25519_keypair(out_public_value: *[32]u8, out_private_key: *[32]u8

pub const NID_X25519 = @as(c_int, 948);
pub const EVP_PKEY_X25519 = NID_X25519;
pub const NID_ED25519 = 949;
pub const EVP_PKEY_ED25519 = NID_ED25519;

pub extern fn EVP_PKEY_new_raw_private_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY;
pub extern fn EVP_PKEY_new_raw_public_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY;
Expand All @@ -236,3 +238,11 @@ pub extern fn EVP_PKEY_CTX_free(ctx: ?*EVP_PKEY_CTX) void;
pub extern fn EVP_PKEY_derive_init(ctx: ?*EVP_PKEY_CTX) c_int;
pub extern fn EVP_PKEY_derive(ctx: ?*EVP_PKEY_CTX, key: [*c]u8, out_key_len: [*c]usize) c_int;
pub extern fn EVP_PKEY_derive_set_peer(ctx: ?*EVP_PKEY_CTX, peer: [*c]EVP_PKEY) c_int;
pub extern fn EVP_PKEY_free(pkey: ?*EVP_PKEY) void;

pub extern fn EVP_DigestSignInit(ctx: ?*EVP_MD_CTX, pctx: ?*?*EVP_PKEY_CTX, typ: ?*const EVP_MD, e: ?*ENGINE, pkey: ?*EVP_PKEY) c_int;
pub extern fn EVP_DigestSign(ctx: ?*EVP_MD_CTX, sig: [*c]u8, sig_len: *usize, data: [*c]const u8, data_len: usize) c_int;
pub extern fn EVP_MD_CTX_new() ?*EVP_MD_CTX;
pub extern fn EVP_MD_CTX_free(ctx: ?*EVP_MD_CTX) void;
pub const struct_evp_md_ctx_st = opaque {};
pub const EVP_MD_CTX = struct_evp_md_ctx_st;
11 changes: 11 additions & 0 deletions src/network/Runtime.zig
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ const libcurl = @import("../sys/libcurl.zig");

const net_http = @import("http.zig");
const RobotStore = @import("Robots.zig").RobotStore;
const WebBotAuth = @import("WebBotAuth.zig");

const Runtime = @This();

Expand All @@ -42,6 +43,7 @@ allocator: Allocator,
config: *const Config,
ca_blob: ?net_http.Blob,
robot_store: RobotStore,
web_bot_auth: ?WebBotAuth,

connections: []net_http.Connection,
available: std.DoublyLinkedList = .{},
Expand Down Expand Up @@ -205,13 +207,19 @@ pub fn init(allocator: Allocator, config: *const Config) !Runtime {
available.append(&connections[i].node);
}

const web_bot_auth = if (config.webBotAuth()) |wba_cfg|
try WebBotAuth.fromConfig(allocator, &wba_cfg)
else
null;

return .{
.allocator = allocator,
.config = config,
.ca_blob = ca_blob,
.robot_store = RobotStore.init(allocator),
.connections = connections,
.available = available,
.web_bot_auth = web_bot_auth,
.pollfds = pollfds,
.wakeup_pipe = pipe,
};
Expand All @@ -238,6 +246,9 @@ pub fn deinit(self: *Runtime) void {
self.allocator.free(self.connections);

self.robot_store.deinit();
if (self.web_bot_auth) |wba| {
wba.deinit(self.allocator);
}

globalDeinit();
}
Expand Down
Loading
Loading