diff --git a/clervers/http2/README.md b/clervers/http2/README.md index ed0ffa1..ae7360c 100644 --- a/clervers/http2/README.md +++ b/clervers/http2/README.md @@ -12,10 +12,12 @@ sudo go build -o /tmp/grpc_server server/main.go && /tmp/grpc_server ### Run client This command makes a request to the server and awaits for its reply, for a total of 2 times. ```bash + sudo go build -o /tmp/grpc_client client/main.go && /tmp/grpc_client --count 2 ``` ## Note The commands in this README file stores the compiled executables in the `/tmp` folder. Ensure you have the necessary priviliges to access and run the workload. -Will add scripts to remove the compiled files to preserve tidiness if deemed necessary. Users can also manually change the output location to the current directory. \ No newline at end of file +Will add scripts to remove the compiled files to preserve tidiness if deemed necessary. Users can also manually change the output location to the current directory. + diff --git a/uprobe_https/LICENSE b/uprobe_https/LICENSE new file mode 100644 index 0000000..f49a4e1 --- /dev/null +++ b/uprobe_https/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/uprobe_https/README.md b/uprobe_https/README.md new file mode 100644 index 0000000..ad62d81 --- /dev/null +++ b/uprobe_https/README.md @@ -0,0 +1,27 @@ +# Uprobe Sniffer for HTTPS Observability + +## Overview + +This directory contains the source code for an HTTPS packet sniffer leveraging eBPF programs attached to uprobes in user-space. This allows for non-intrusive observability for HTTPS protocols under minimal maintainence effort. + +We run a user-space agent that attaches eBPF programs to SSL_read, SSL_write, transport.(*loopyWriter).writeHeader and transport.(*http2Server).operateHeaders. The agent also handles the transfer of data between user-space and kernel-space, and outputs the captured data to the terminal. + +## Components + +- **module_src**: Contains Golang implementations of attaching uprobes, structs for communication between user-space and kernel-space, read/write buffers etc. +- **sniffer**: Contains source code for the user-space agent and eBPF programs. + +## Dependencies +Please refer to the `ebpf_http_dependencies` directory. + +## Usage +FOR CURRENT VERSION: start the http1s server before running the sniffer +```bash +sudo go run ./sniffer/main.go ./sniffer/ebpf_src.c --pid $(pgrep -f "python3 ./server/https_server.py 43421") +``` +Ensure that you have the necessary packages installed and the correct version of bcc (v0.24.0) compiled. Exit the sniffer using Ctrl-C. + +## Note +At the current stage, this sniffer only supports the observation of HTTPS through uprobes. This will be upgraded later to support the attachment of kprobes as well as HTTP/1.x and HTTP/2.0 observability. + +We also plan to use Grafana to provide a better interface for users, replacing the current mode of outputting to the terminal. diff --git a/uprobe_https/client.py b/uprobe_https/client.py new file mode 100644 index 0000000..32e3109 --- /dev/null +++ b/uprobe_https/client.py @@ -0,0 +1,6 @@ +# simple script to send a client request +# modified by Yunxi Shen + +import requests +body = {"Hello World": "I'm testing my HTTPS sniffer", "Results=": "Good"} +resp = requests.post("https://127.0.0.1:43421/example", json=body, verify=False) \ No newline at end of file diff --git a/uprobe_https/go.mod b/uprobe_https/go.mod new file mode 100644 index 0000000..16b4a2c --- /dev/null +++ b/uprobe_https/go.mod @@ -0,0 +1,16 @@ +module my_modules + +go 1.22 + +replace my_modules => ./ + +require ( + github.com/alexflint/go-arg v1.4.3 + github.com/iovisor/gobpf v0.2.0 + golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 +) + +require ( + github.com/alexflint/go-scalar v1.1.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect +) diff --git a/uprobe_https/go.sum b/uprobe_https/go.sum new file mode 100644 index 0000000..6a25acd --- /dev/null +++ b/uprobe_https/go.sum @@ -0,0 +1,21 @@ +github.com/alexflint/go-arg v1.4.3 h1:9rwwEBpMXfKQKceuZfYcwuc/7YY7tWJbFsgG5cAU/uo= +github.com/alexflint/go-arg v1.4.3/go.mod h1:3PZ/wp/8HuqRZMUUgu7I+e1qcpUbvmS258mRXkFH4IA= +github.com/alexflint/go-scalar v1.1.0 h1:aaAouLLzI9TChcPXotr6gUhq+Scr8rl0P9P4PnltbhM= +github.com/alexflint/go-scalar v1.1.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/iovisor/gobpf v0.2.0 h1:34xkQxft+35GagXBk3n23eqhm0v7q0ejeVirb8sqEOQ= +github.com/iovisor/gobpf v0.2.0/go.mod h1:WSY9Jj5RhdgC3ci1QaacvbFdQ8cbrEjrpiZbLHLt2s4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/uprobe_https/module_src/bpfwrapper/perfbufferreaders.go b/uprobe_https/module_src/bpfwrapper/perfbufferreaders.go new file mode 100644 index 0000000..96a01a5 --- /dev/null +++ b/uprobe_https/module_src/bpfwrapper/perfbufferreaders.go @@ -0,0 +1,84 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0. + */ + +package bpfwrapper + +import ( + "fmt" + bpf "github.com/iovisor/gobpf/bcc" +) + +// ProbeEventLoop is the signature for the callback functions to extract the events from the input channel. +type ProbeEventLoop func(inputChan chan []byte) + +// ProbeChannel represents a single handler to a channel of events in the BPF. +type ProbeChannel struct { + // Name of the BPF channel. + name string + // Event loop handler, a method which receive a channel for the input events from the implementation, and parse them. + eventLoop ProbeEventLoop + // A go channel which holds the messages from the BPF module. + eventChannel chan []byte + // A go channel for lost events. + lostEventsChannel chan uint64 + // The bpf perf map that links our user mode channel to the BPF module. + perfMap *bpf.PerfMap +} + +// NewProbeChannel creates a new probe channel with the given handle for the given bpf channel name. +func NewProbeChannel(name string, handler ProbeEventLoop) *ProbeChannel { + return &ProbeChannel{ + name: name, + eventLoop: handler, + } +} + +// Start initiate a goroutine for the event loop handler, for a lost events messages and the perf map. +func (probeChannel *ProbeChannel) Start(module *bpf.Module) error { + probeChannel.eventChannel = make(chan []byte) + probeChannel.lostEventsChannel = make(chan uint64) + + table := bpf.NewTable(module.TableId(probeChannel.name), module) + + var err error + probeChannel.perfMap, err = bpf.InitPerfMapWithPageCnt(table, probeChannel.eventChannel, probeChannel.lostEventsChannel, 8192) + if err != nil { + return fmt.Errorf("failed to init perf mapping for %q due to: %v", probeChannel.name, err) + } + + go probeChannel.eventLoop(probeChannel.eventChannel) + go func() { + for { + <-probeChannel.lostEventsChannel + } + }() + + probeChannel.perfMap.Start() + return nil +} + +// LaunchPerfBufferConsumers launches all probe channels. +func LaunchPerfBufferConsumers(module *bpf.Module, probeList []*ProbeChannel) error { + for _, probeChannel := range probeList { + if err := probeChannel.Start(module); err != nil { + return err + } + } + + return nil +} diff --git a/uprobe_https/module_src/bpfwrapper/uprobes.go b/uprobe_https/module_src/bpfwrapper/uprobes.go new file mode 100644 index 0000000..75eb4a6 --- /dev/null +++ b/uprobe_https/module_src/bpfwrapper/uprobes.go @@ -0,0 +1,56 @@ +package bpfwrapper + +import ( + "fmt" + "github.com/iovisor/gobpf/bcc" + "log" +) + +// ProbeType represents whether the probe is an entry or a return. +type ProbeType int + +const ( + EntryType ProbeType = 0 + ReturnType ProbeType = 1 + +) + +// Uprobe represents a single uprobe hook. +type Uprobe struct { + // The name of the function to hook. + FunctionToHook string + // The name of the hook function. + HookName string + // Whether an uprobe or ret-uprobe. + Type ProbeType + // Whether the function to hook is syscall or not. + BinaryPath string +} + +// AttachUprobes attaches the given uprobe list. +func AttachUprobes(soPath string, pid int, bpfModule *bcc.Module, kprobeList []Uprobe) error { + for _, probe := range kprobeList { + functionToHook := probe.FunctionToHook + + probeFD, err := bpfModule.LoadUprobe(probe.HookName) + if err != nil { + return fmt.Errorf("failed to load %q due to: %v", probe.HookName, err) + } + + switch probe.Type { + case EntryType: + log.Printf("Loading %q for %q as kprobe\n", probe.HookName, probe.FunctionToHook) + if err = bpfModule.AttachUprobe(soPath, functionToHook, probeFD, pid); err != nil { + return fmt.Errorf("failed to attach kprobe %q to %q due to: %v", probe.HookName, functionToHook, err) + } + case ReturnType: + log.Printf("Loading %q for %q as kretprobe\n", probe.HookName, probe.FunctionToHook) + if err = bpfModule.AttachUretprobe(soPath, functionToHook, probeFD, pid); err != nil { + return fmt.Errorf("failed to attach kretprobe %q to %q due to: %v", probe.HookName, functionToHook, err) + } + default: + return fmt.Errorf("unknown uprobe type %d given for %q", probe.Type, probe.HookName) + } + } + return nil +} diff --git a/uprobe_https/module_src/privileges/privileges.go b/uprobe_https/module_src/privileges/privileges.go new file mode 100644 index 0000000..b374b08 --- /dev/null +++ b/uprobe_https/module_src/privileges/privileges.go @@ -0,0 +1,26 @@ +package privileges + +import ( + "log" + "os/user" + "runtime/debug" +) + +// AbortIfNotRoot checks the current user permissions, if the permissions are not elevated, we abort. +func AbortIfNotRoot() { + current, err := user.Current() + if err != nil { + log.Panic(err) + } + + if current.Uid != "0" { + log.Panic("sniffer must run under superuser privileges") + } +} + +// RecoverFromCrashes is a defer function that caches all panics being thrown from the application. +func RecoverFromCrashes() { + if err := recover(); err != nil { + log.Printf("Application crashed: %v\nstack: %s\n", err, string(debug.Stack())) + } +} diff --git a/uprobe_https/module_src/settings/clock.go b/uprobe_https/module_src/settings/clock.go new file mode 100644 index 0000000..591e53b --- /dev/null +++ b/uprobe_https/module_src/settings/clock.go @@ -0,0 +1,48 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0. + */ + +package settings + +import ( + "fmt" + "time" + + "golang.org/x/sys/unix" +) + +var ( + realTimeOffset uint64 = 0 +) + +// InitRealTimeOffset calculates the offset between the real clock and the monotonic clock used in the BPF. +func InitRealTimeOffset() error { + var monotonicTime, realTime unix.Timespec + if err := unix.ClockGettime(unix.CLOCK_MONOTONIC, &monotonicTime); err != nil { + return fmt.Errorf("failed getting monotonic clock due to: %v", err) + } + if err := unix.ClockGettime(unix.CLOCK_REALTIME, &realTime); err != nil { + return fmt.Errorf("failed getting real clock time due to: %v", err) + } + realTimeOffset = uint64(time.Second)*(uint64(realTime.Sec)-uint64(monotonicTime.Sec)) + uint64(realTime.Nsec) - uint64(monotonicTime.Nsec) + return nil +} + +// GetRealTimeOffset is a getter for the real-time-offset. +func GetRealTimeOffset() uint64 { + return realTimeOffset +} diff --git a/uprobe_https/sniffer/ebpf_src.c b/uprobe_https/sniffer/ebpf_src.c new file mode 100644 index 0000000..5d7a07c --- /dev/null +++ b/uprobe_https/sniffer/ebpf_src.c @@ -0,0 +1,222 @@ +// +build ignore + +// modified from pixie and datadog repos +// combines uprobe for http/2.0 and uprobe for tls encrpytion +// modified by Yunxi Shen + +#include + +#define MAX_SIZE 400 + +#define HEADER_FIELD_STR_SIZE 128 +#define MAX_HEADER_COUNT 64 + +struct tls_data_args_t { + const char* buf; +}; + +struct tls_ex_data_args_t { + const char* buf; + size_t *buf_size; +}; + +enum traffic_direction_t { + kEgress, + kIngress, +}; + +struct data_event_t { + uint64_t timestamp_ns; + uint32_t pid; + enum traffic_direction_t direction; + char msg [MAX_SIZE]; +}; + + +struct header_field_t { + int32_t size; + char msg[HEADER_FIELD_STR_SIZE]; +}; + +struct go_grpc_http2_header_event_t { + struct header_field_t name; + struct header_field_t value; +}; + +// This matches the golang string object memory layout. Used to help read golang string objects in BPF code. +struct gostring { + const char* ptr; + int64_t len; +}; + + +BPF_HASH(tls_write_args_map, uint64_t, struct tls_data_args_t); +BPF_HASH(tls_read_args_map, uint64_t, struct tls_data_args_t); + +BPF_HASH(tls_write_ex_args_map, uint64_t, struct tls_ex_data_args_t); +BPF_HASH(tls_read_ex_args_map, uint64_t, struct tls_ex_data_args_t); + +BPF_PERF_OUTPUT(data_events); + +BPF_PERCPU_ARRAY(pid, int, 1); + +BPF_PERF_OUTPUT(go_http2_header_events); + +static inline void process_data(struct pt_regs* ctx, uint64_t id, size_t bytes_count, enum traffic_direction_t direction, const char* buf) { + if (buf == NULL) return; + if (PT_REGS_RC(ctx) <= 0) return; + + int key = 0; + int *pidToAllow = pid.lookup(&key); + if (pidToAllow != NULL && *pidToAllow != -1) { + bpf_trace_printk("%d\n", *pidToAllow); + const int current_pid = id >> 32; + if (current_pid != *pidToAllow) return; + } + + struct data_event_t event = {}; + event.timestamp_ns = bpf_ktime_get_ns(); + event.direction = direction; + event.pid = id >> 32; + int i; + for (i = 0; buf[i] != '\0'; ++i) { + if (i < MAX_SIZE - 1) { + event.msg[i] = buf[i]; + } + else { + // Ensure we don't write past the end of `msg` + break; + } + } + // size_t msg_size = bytes_count < MAX_SIZE ? bytes_count : MAX_SIZE; + // bpf_probe_read(&event.msg, msg_size, buf); + data_events.perf_submit(ctx, &event, sizeof(struct data_event_t)); +} + +int probe_entry_ssl_read(struct pt_regs *ctx) { + uint64_t id = bpf_get_current_pid_tgid(); + struct tls_data_args_t read_args = {}; + read_args.buf = (char*)PT_REGS_PARM2(ctx); + tls_read_args_map.update(&id, &read_args); + return 0; +} + +int probe_ret_ssl_read(struct pt_regs *ctx) { + uint64_t id = bpf_get_current_pid_tgid(); + struct tls_data_args_t* read_args = tls_read_args_map.lookup(&id); + if (read_args != NULL) { + tls_read_args_map.delete(&id); + process_data(ctx, id, PT_REGS_RC(ctx), kIngress, read_args->buf); + } + return 0; +} + +int probe_entry_ssl_write(struct pt_regs *ctx) { + uint64_t id = bpf_get_current_pid_tgid(); + struct tls_data_args_t write_args = {}; + write_args.buf = (const char *) PT_REGS_PARM2(ctx); + tls_write_args_map.update(&id, &write_args); + return 0; +} + +int probe_ret_ssl_write(struct pt_regs *ctx) { + uint64_t id = bpf_get_current_pid_tgid(); + struct tls_data_args_t* write_args = tls_write_args_map.lookup(&id); + if (write_args != NULL) { + tls_write_args_map.delete(&id); + process_data(ctx, id, PT_REGS_RC(ctx), kEgress, write_args->buf); + } + return 0; +} + +int probe_entry_ssl_read_ex(struct pt_regs *ctx) { + uint64_t id = bpf_get_current_pid_tgid(); + struct tls_ex_data_args_t read_args = {}; + read_args.buf = (char*)PT_REGS_PARM2(ctx); + read_args.buf_size = (size_t*)PT_REGS_PARM4(ctx); + tls_read_ex_args_map.update(&id, &read_args); + return 0; +} + +int probe_ret_ssl_read_ex(struct pt_regs *ctx) { + uint64_t id = bpf_get_current_pid_tgid(); + struct tls_ex_data_args_t* read_args = tls_read_ex_args_map.lookup(&id); + if (read_args != NULL) { + tls_read_ex_args_map.delete(&id); + size_t bytes_count = 0; + bpf_probe_read_user(&bytes_count, sizeof(bytes_count), read_args->buf_size); + process_data(ctx, id, bytes_count, kIngress, read_args->buf); + } + return 0; +} + +int probe_entry_ssl_write_ex(struct pt_regs *ctx) { + uint64_t id = bpf_get_current_pid_tgid(); + struct tls_ex_data_args_t write_args = {}; + write_args.buf = (const char *)PT_REGS_PARM2(ctx); + write_args.buf_size = (size_t*)PT_REGS_PARM4(ctx); + tls_write_ex_args_map.update(&id, &write_args); + return 0; +} + +int probe_ret_ssl_write_ex(struct pt_regs *ctx) { + uint64_t id = bpf_get_current_pid_tgid(); + + struct tls_ex_data_args_t* write_args = tls_write_ex_args_map.lookup(&id); + if (write_args != NULL) { + tls_write_ex_args_map.delete(&id); + size_t bytes_count = 0; + bpf_probe_read_user(&bytes_count, sizeof(bytes_count), write_args->buf_size); + process_data(ctx, id, bytes_count, kEgress, write_args->buf); + } + return 0; +} + + + +// Copy the content of a hpack.HeaderField object into header_field_t object. +static void copy_header_field(struct header_field_t* dst, const void* header_field_ptr) { + struct gostring str = {}; + bpf_probe_read(&str, sizeof(str), header_field_ptr); + if (str.len <= 0) { + dst->size = 0; + return; + } + dst->size = str.len < HEADER_FIELD_STR_SIZE ? str.len : HEADER_FIELD_STR_SIZE; + bpf_probe_read(dst->msg, HEADER_FIELD_STR_SIZE, str.ptr); +} + +// Copies and submits content of an array of hpack.HeaderField to perf buffer. +static void submit_headers(struct pt_regs* ctx, void* fields_ptr, int64_t fields_len) { + // Size of the golang hpack.HeaderField struct. + const size_t header_field_size = 40; + struct go_grpc_http2_header_event_t event = {}; + for (size_t i = 0; i < MAX_HEADER_COUNT; ++i) { + if (i >= fields_len) { + continue; + } + const void* header_field_ptr = fields_ptr + i * header_field_size; + copy_header_field(&event.name, header_field_ptr); + copy_header_field(&event.value, header_field_ptr + 16); + go_http2_header_events.perf_submit(ctx, &event, sizeof(event)); + } +} + +// Signature: func (l *loopyWriter) writeHeader(streamID uint32, endStream bool, hf []hpack.HeaderField, onWrite func()) +int probe_loopy_writer_write_header(struct pt_regs* ctx) { + void* fields_ptr = (void*) PT_REGS_PARM1(ctx); + int64_t fields_len = (int64_t)PT_REGS_PARM2(ctx); + submit_headers(ctx, fields_ptr, fields_len); + return 0; +} + +// Signature: func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(*Stream), +// traceCtx func(context.Context, string) context.Context) +int probe_http2_server_operate_headers(struct pt_regs* ctx) { + void* fields_ptr = (void*) ctx->r8; + // void* fields_ptr = (void *)PT_REGS_PARM4(ctx); + int64_t fields_len = (int64_t) ctx->r11; + // int64_t fields_len = (int64_t) PT_REGS_PARM5(ctx); + submit_headers(ctx, fields_ptr, fields_len); + return 0; +} \ No newline at end of file diff --git a/uprobe_https/sniffer/main.go b/uprobe_https/sniffer/main.go new file mode 100644 index 0000000..591b6d3 --- /dev/null +++ b/uprobe_https/sniffer/main.go @@ -0,0 +1,247 @@ +// modified from pixie and datadog repos +// combines uprobe for http/2.0 and uprobe for tls encrpytion +// modified by Yunxi Shen + +package main + +import ( + "bytes" + "encoding/binary" + "fmt" + "io/ioutil" + "log" + "my_modules/module_src/bpfwrapper" + "my_modules/module_src/privileges" + "my_modules/module_src/settings" + "os" + "os/signal" + "runtime" + "syscall" + "time" + "unsafe" + + "github.com/alexflint/go-arg" + "github.com/iovisor/gobpf/bcc" +) + +const headerFieldStrSize = 128 + +type headerField struct { + Size uint32 + Msg [headerFieldStrSize]byte +} + +// http2HeaderEvent's memory layout is identical to the go_grpc_http2_header_event_t in bpf_program.go, such that the +// event data obtained from the perf buffer can be directly copied to http2HeaderEvent. +type http2HeaderEvent struct { + Name headerField + Value headerField +} + +var ( + hooks = []bpfwrapper.Uprobe{ + { + FunctionToHook: "SSL_write", + HookName: "probe_entry_ssl_write", + Type: bpfwrapper.EntryType, + }, + { + FunctionToHook: "SSL_write", + HookName: "probe_ret_ssl_write", + Type: bpfwrapper.ReturnType, + }, + { + FunctionToHook: "SSL_read", + HookName: "probe_entry_ssl_read", + Type: bpfwrapper.EntryType, + }, + { + FunctionToHook: "SSL_read", + HookName: "probe_ret_ssl_read", + Type: bpfwrapper.ReturnType, + }, + { + FunctionToHook: "SSL_write_ex", + HookName: "probe_entry_ssl_write", + Type: bpfwrapper.EntryType, + }, + { + FunctionToHook: "SSL_write_ex", + HookName: "probe_ret_ssl_write", + Type: bpfwrapper.ReturnType, + }, + { + FunctionToHook: "SSL_read_ex", + HookName: "probe_entry_ssl_read", + Type: bpfwrapper.EntryType, + }, + { + FunctionToHook: "SSL_read_ex", + HookName: "probe_ret_ssl_read", + Type: bpfwrapper.ReturnType, + }, + // { + // FunctionToHook: "google.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader", + // HookName: "probe_ret_ssl_read", + // Type: bpfwrapper.EntryType, + // }, + } +) + +// args represents the command line arguments. +var args struct { + BPFFile string `arg:"required,positional"` + PID int `arg:"--pid" default:"-1"` +} + +// DataEvent is a conversion of the following C-Struct into GO. +// +// struct data_event_t { +// uint64_t timestamp_ns; +// uint32_t pid; +// enum traffic_direction_t direction; +// char msg [400]; +// }; +type DataEvent struct { + TimestampNano uint64 + PID uint32 + Direction int32 + Buffer [400]byte +} + +func formatHeaderField(field headerField) string { + return string(field.Msg[0:field.Size]) +} + +func formatHeaderEvent(event http2HeaderEvent) string { + return fmt.Sprintf("{ Header_Name:'%s', Header_Value='%s' }", formatHeaderField(event.Name), formatHeaderField(event.Value)) +} + +func openEventCallback(inputChan chan []byte) { + for data := range inputChan { + if data == nil { + return + } + var event DataEvent + + if err := binary.Read(bytes.NewReader(data), bcc.GetHostByteOrder(), &event); err != nil { + log.Printf("Failed to decode received data: %+v", err) + continue + } + + event.TimestampNano += settings.GetRealTimeOffset() + if event.Direction == 0 { // egress + fmt.Printf("----------------------------------------\nResponse to client {pid: %v, time: %v, buffer: %s}\n", event.PID, time.Unix(0, int64(event.TimestampNano)), string(event.Buffer[:])) + } else { + fmt.Printf("----------------------------------------\nRequest from client {pid: %v, time: %v, buffer: %s}\n", event.PID, time.Unix(0, int64(event.TimestampNano)), string(event.Buffer[:])) + } + } +} + +func openEventCallback_new(inputChan chan []byte) { + for data := range inputChan { + if data == nil { + return + } + var parsed http2HeaderEvent + if err := binary.Read(bytes.NewReader(data), bcc.GetHostByteOrder(), &parsed); err != nil { + panic(err) + } + fmt.Println(formatHeaderEvent(parsed)) + } + fmt.Printf("----------------------------------------\n") +} + +var ( + numberOfCPUs = runtime.NumCPU() +) + +func fillPerCPUArray(bpfModule *bcc.Module, arrayName string, key int, value int) error { + arr := make([]int, numberOfCPUs) + for i := 0; i < numberOfCPUs; i++ { + arr[i] = value + } + + controlValues := bcc.NewTable(bpfModule.TableId(arrayName), bpfModule) + return controlValues.SetP(unsafe.Pointer(&key), unsafe.Pointer(&arr[0])) +} + +func mustAttachUprobe(bccMod *bcc.Module, binaryProg, symbol, probeFn string) { + uprobeFD, err := bccMod.LoadUprobe(probeFn) + if err != nil { + panic(err) + } + err = bccMod.AttachUprobe(binaryProg, symbol, uprobeFD, -1 /*pid*/) + if err != nil { + panic(err) + } +} + +// func mustAttachUretprobe(bccMod *bcc.Module, binaryProg, symbol, probeFn string) { +// uprobeFD, err := bccMod.LoadUprobe(probeFn) +// if err != nil { +// panic(err) +// } +// err = bccMod.AttachUretprobe(binaryProg, symbol, uprobeFD, -1 /*pid*/) +// if err != nil { +// panic(err) +// } +// } + +func main() { + arg.MustParse(&args) + + bpfSourceCodeContent, err := ioutil.ReadFile(args.BPFFile) + if err != nil { + log.Panic(err) + } + + defer privileges.RecoverFromCrashes() + privileges.AbortIfNotRoot() + + if err := settings.InitRealTimeOffset(); err != nil { + log.Printf("Failed fixing BPF clock, timings will be offseted: %v", err) + } + + // Catching all termination signals to perform a cleanup when being stopped. + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM) + + bpfModule := bcc.NewModule(string(bpfSourceCodeContent), nil) + if bpfModule == nil { + log.Panic("bpf is nil") + } + defer bpfModule.Close() + + callbacks := []*bpfwrapper.ProbeChannel{bpfwrapper.NewProbeChannel("data_events", openEventCallback)} + + callbacks_new := []*bpfwrapper.ProbeChannel{bpfwrapper.NewProbeChannel("go_http2_header_events", openEventCallback_new)} + + if err := bpfwrapper.LaunchPerfBufferConsumers(bpfModule, callbacks); err != nil { + log.Panic(err) + } + + if err := bpfwrapper.LaunchPerfBufferConsumers(bpfModule, callbacks_new); err != nil { + log.Panic(err) + } + + if err := fillPerCPUArray(bpfModule, "pid", 0, args.PID); err != nil { + log.Panic(err) + } + + if err := bpfwrapper.AttachUprobes("/usr/lib/x86_64-linux-gnu/libssl.so.3", args.PID, bpfModule, hooks); err != nil { + log.Panic(err) + } + + const loopWriterWriteHeaderSymbol = "google.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader" + const loopWriterWriteHeaderProbeFn = "probe_loopy_writer_write_header" + mustAttachUprobe(bpfModule, "/tmp/grpc_server", loopWriterWriteHeaderSymbol, loopWriterWriteHeaderProbeFn) + + const http2ServerOperateHeadersSymbol = "google.golang.org/grpc/internal/transport.(*http2Server).operateHeaders" + const http2ServerOperateHeadersProbeFn = "probe_http2_server_operate_headers" + mustAttachUprobe(bpfModule, "/tmp/grpc_server", http2ServerOperateHeadersSymbol, http2ServerOperateHeadersProbeFn) + + log.Println("Sniffer is ready\n----------------------------------------") + <-sig + log.Println("Signaled to terminate") +}