feat: send logs to userspace + revamp (#14)

Update bpf_printk statements to be also sent to userspace so they can be
logged alongside events. All log lines are considered errors and should
be checked (unless running a debug eBPF binary).

Revamps the CO-RE reads that were performed in the eBPF code to use
slimmer types and to use BPF helper macros for readability.

Adds more tests as well as ebpf benchmarks.

Author: deansheather

.gitattributes

@@ -1,6 +1,6 @@
-bpf/bpf_core_read.h linguist-vendored
-bpf/bpf_helper_defs.h linguist-vendored
-bpf/bpf_helpers.h linguist-vendored
-bpf/handler-bpfeb.o linguist-generated
-bpf/handler-bpfel.o linguist-generated
-bpf/vmlinux.h linguist-vendored
+bpf/bpf_core_read.h linguist-generated=true
+bpf/bpf_helper_defs.h linguist-generated=true
+bpf/bpf_helpers.h linguist-generated=true
+bpf/handler-bpfeb.o linguist-generated=true
+bpf/handler-bpfel.o linguist-generated=true
+bpf/vmlinux.h linguist-generated=true

.github/workflows/enterprise-release.yaml

@@ -51,7 +51,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: "^1.20.7"
+          go-version: "^1.21.9"
 
       - name: Build binaries
         run: |

.github/workflows/enterprise.yaml

@@ -29,15 +29,15 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
-  test-go:
-    name: "test/go"
+  test-go-enterprise:
+    name: "test/go-enterprise"
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: "^1.20.7"
+          go-version: "^1.21.9"
 
       - name: Echo Go Cache Paths
         id: go-cache-paths
@@ -57,59 +57,5 @@ jobs:
           path: ${{ steps.go-cache-paths.outputs.go-mod }}
           key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
 
-      - name: Install gotestsum
-        uses: jaxxstorm/[email protected]
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          repo: gotestyourself/gotestsum
-          tag: v1.10.1
-
       - name: Test
-        shell: bash
-        run: |
-          set +e
-          gotestsum \
-            --junitfile="gotests.xml" \
-            --jsonfile="gotestsum.json" \
-            --packages="./..." \
-            --debug \
-            -- \
-            -exec sudo \
-            -parallel=8 \
-            -timeout=5m \
-            -short \
-            -failfast \
-            -covermode=atomic \
-            -coverprofile="gotests.coverage" \
-            -coverpkg=./...
-          ret=$?
-          set -e
-          if ((ret)); then
-            # Eternalize test timeout logs because "re-run failed" erases
-            # artifacts and gotestsum doesn't always capture it:
-            # https://github.com/gotestyourself/gotestsum/issues/292
-            # Multiple test packages could've failed, each one may or may
-            # not run into the edge case. PS. Don't summon ShellCheck here.
-            for testWithStack in $(grep 'panic: test timed out' gotestsum.json | grep -E -o '("Test":[^,}]*)'); do
-              if [ -n "$testWithStack" ] && grep -q "${testWithStack}.*PASS" gotestsum.json; then
-                echo "Conditions met for gotestsum stack trace missing bug, outputting panic trace:"
-                grep -A 999999 "${testWithStack}.*panic: test timed out" gotestsum.json
-              fi
-            done
-          fi
-          exit $ret
-
-      - uses: actions/upload-artifact@v3
-        if: success() || failure()
-        with:
-          name: gotestsum-debug.json
-          path: ./gotestsum.json
-          retention-days: 7
-
-      - uses: actions/upload-artifact@v3
-        if: success() || failure()
-        with:
-          name: gotests.xml
-          path: ./gotests.xml
-          retention-days: 30
+        run: make test/go-enterprise

.github/workflows/quality.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v2
         with:
-          go-version: "^1.20.7"
+          go-version: "^1.21.9"
 
       - name: Run make fmt/go
         run: make fmt/go
@@ -75,12 +75,12 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v2
         with:
-          go-version: "^1.20.7"
+          go-version: "^1.21.9"
 
       - name: Install golangci-lint
         run: |
           curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \
-            | sh -s -- -b $(go env GOPATH)/bin v1.53.2
+            | sh -s -- -b $(go env GOPATH)/bin v1.57.2
 
       # Linting needs to be done on each build variation of GOOS.
       - name: Run make lint/go/linux
@@ -90,6 +90,37 @@ jobs:
       - name: Run make lint/go/other
         run: make lint/go/other
 
+  test-go:
+    name: "test/go"
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "^1.21.9"
+
+      - name: Echo Go Cache Paths
+        id: go-cache-paths
+        run: |
+          echo "::set-output name=go-build::$(go env GOCACHE)"
+          echo "::set-output name=go-mod::$(go env GOMODCACHE)"
+
+      - name: Go Build Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-build }}
+          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.**', '**.go') }}
+
+      - name: Go Mod Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-mod }}
+          key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
+
+      - name: Test
+        run: make test/go
+
   lint-c:
     name: lint/c
     runs-on: ubuntu-20.04
@@ -100,6 +131,14 @@ jobs:
       - name: Run make lint/c
         run: make lint/c
 
+      - name: Ensure DEBUG is disabled
+        run: |
+          # look for uncommented "#define DEBUG" in bpf/handler.c
+          if grep -q "^#define DEBUG" bpf/handler.c; then
+            echo "DEBUG is enabled in bpf/handler.c"
+            exit 1
+          fi
+
   lint-shellcheck:
     name: lint/shellcheck
     runs-on: ubuntu-20.04

.gitignore

@@ -33,3 +33,5 @@ build
 *.tfplan
 *.lock.hcl
 .terraform/
+
+/exectrace

.golangci.yaml

@@ -20,30 +20,30 @@ linters-settings:
     enabled-checks:
       # - appendAssign
       # - appendCombine
-      - argOrder
+      #- argOrder
       # - assignOp
       # - badCall
-      - badCond
+      #- badCond
       - badLock
       - badRegexp
       - boolExprSimplify
       # - builtinShadow
       - builtinShadowDecl
-      - captLocal
-      - caseOrder
-      - codegenComment
+      #- captLocal
+      #- caseOrder
+      #- codegenComment
       # - commentedOutCode
       - commentedOutImport
-      - commentFormatting
-      - defaultCaseOrder
+      #- commentFormatting
+      #- defaultCaseOrder
       - deferUnlambda
       # - deprecatedComment
       # - docStub
-      - dupArg
-      - dupBranchBody
-      - dupCase
+      #- dupArg
+      #- dupBranchBody
+      #- dupCase
       - dupImport
-      - dupSubExpr
+      #- dupSubExpr
       # - elseif
       - emptyFallthrough
       # - emptyStringTest
@@ -52,56 +52,56 @@ linters-settings:
       # - exitAfterDefer
       # - exposedSyncMutex
       # - filepathJoin
-      - flagDeref
-      - flagName
+      #- flagDeref
+      #- flagName
       - hexLiteral
       # - httpNoBody
       # - hugeParam
       # - ifElseChain
       # - importShadow
       - indexAlloc
       - initClause
-      - mapKey
+      #- mapKey
       - methodExprCall
       # - nestingReduce
-      - newDeref
+      #- newDeref
       - nilValReturn
       # - octalLiteral
-      - offBy1
+      #- offBy1
       # - paramTypeCombine
       # - preferStringWriter
       # - preferWriteByte
       # - ptrToRefParam
       # - rangeExprCopy
       # - rangeValCopy
-      - regexpMust
+      #- regexpMust
       - regexpPattern
       # - regexpSimplify
       #- ruleguard
-      - singleCaseSwitch
-      - sloppyLen
+      #- singleCaseSwitch
+      #- sloppyLen
       # - sloppyReassign
-      - sloppyTypeAssert
+      #- sloppyTypeAssert
       - sortSlice
       - sprintfQuotedString
       - sqlQuery
       # - stringConcatSimplify
       # - stringXbytes
       # - suspiciousSorting
-      - switchTrue
+      #- switchTrue
       - truncateCmp
       - typeAssertChain
       # - typeDefFirst
-      - typeSwitchVar
+      #- typeSwitchVar
       # - typeUnparen
-      - underef
+      #- underef
       # - unlabelStmt
       # - unlambda
       # - unnamedResult
       # - unnecessaryBlock
       # - unnecessaryDefer
       # - unslice
-      - valSwap
+      #- valSwap
       - weakCond
       # - whyNoLint
       # - wrapperFunc
@@ -207,15 +207,17 @@ issues:
       linters:
         - exhaustruct
 
+  exclude-files:
+    - scripts/rules.go
+
+  exclude-dirs:
+    - node_modules
+
   fix: true
   max-issues-per-linter: 0
   max-same-issues: 0
 
 run:
-  skip-dirs:
-    - node_modules
-  skip-files:
-    - scripts/rules.go
   timeout: 10m
 
 # Over time, add more and more linters from

Makefile

@@ -24,12 +24,12 @@ handlers: bpf/handler-bpfeb.o bpf/handler-bpfel.o
 clean: clean-enterprise
 	rm -rf bpf/handler-bpfeb.o bpf/handler-bpfel.o
 
-ci/.clang-image: ci/images/clang-13/Dockerfile
+ci/.clang-image: ci/images/clang-13/Dockerfile ci/scripts/clang_image.sh
 	./ci/scripts/clang_image.sh
 	touch ci/.clang-image
 
 # bpfeb is big endian, bpfel is little endian.
-bpf/handler-bpfeb.o bpf/handler-bpfel.o: bpf/*.h bpf/*.c ci/.clang-image
+bpf/handler-bpfeb.o bpf/handler-bpfel.o: bpf/*.h bpf/*.c ci/.clang-image ci/scripts/build_handler.sh
 	./ci/scripts/build_handler.sh "$(@F)"
 
 .PHONY: fmt
@@ -76,22 +76,20 @@ lint/shellcheck:
 	./ci/scripts/shellcheck.sh
 
 .PHONY: test
-test: test/go
+test: test/go test/go-enterprise
 
 .PHONY: test/go
 test/go:
-	go clean -testcache
-	gotestsum --debug -- -v -short ./...
+	go test -exec sudo -v -count 1 ./...
+
+.PHONY: test/go-enterprise
+test/go-enterprise:
 	cd enterprise
-	gotestsum --debug -- -v -short ./...
+	go test -exec sudo -v -count 1 ./...
 
-# test/go-sudo is equivalent to test/go but runs the test binary using sudo.
-# Some tests are skipped if not running as root.
-.PHONY: test/go-sudo
-test/go-sudo:
+.PHONY: bench
+bench:
 	go clean -testcache
-	gotestsum --debug -- -exec sudo -v -short ./...
-	cd enterprise
-	gotestsum --debug -- -exec sudo -v -short ./...
+	COUNT=10000 ./bench/bench.sh
 
 include Makefile.enterprise