chore(make): re-create test targets (aquasecurity#1559)

Instead of grouping tests by tracee-ebpf and tracee-rules
we will group them by scope, i.e. unit, integration, and
end-to-end tests.

               .
              / \\
             /   \ \
            / E2E \' \
           / TESTS \'' \
          /---------\ ' '\
         /           \''   \
        / INTEGRATION \ ' '  \
       /     TESTS     \   ''  \
      /-----------------\' ' ' ' \
     /                   \' ' '  ' \
    /                     \' '   ' ' \
   /      UNIT TESTS       \  '   ' /
  /                         \   ' /
 /                           \'  /
/_____________________________\/

NB There are still some tests that do not fall into category
of unit, integration, nor end-to-end tests, but fixing them
is out of scope of this patch.

Resolves: aquasecurity#1522

Signed-off-by: Daniel Pacak <[email protected]>

Author: danielpacak

.github/workflows/pr.yaml

@@ -32,10 +32,9 @@ jobs:
             gofmt -s -d .
             exit 1
           fi
-  verify-tracee-ebpf:
-    name: Verify tracee-ebpf
-    needs:
-      - verify-code
+
+  run-unit-tests:
+    name: "Run Unit Tests"
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout Code
@@ -47,16 +46,15 @@ jobs:
         with:
           go-version: ${{ env.GO_VERSION }}
           opa-version: ${{ env.OPA_VERSION }}
-      - name: Build tracee-ebpf
-        run: |
-          make tracee-ebpf
-      - name: Test tracee-ebpf
+      - name: Run Unit Tests
         run: |
-          make test-tracee-ebpf
-  verify-tracee-rules:
-    name: Verify tracee-rules
+          make test-unit
+
+  run-integration-tests:
+    name: "Run Integration Tests"
     needs:
       - verify-code
+      - run-unit-tests
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout Code
@@ -68,16 +66,15 @@ jobs:
         with:
           go-version: ${{ env.GO_VERSION }}
           opa-version: ${{ env.OPA_VERSION }}
-      - name: Build tracee-rules
-        run: |
-          make tracee-rules
-      - name: Test tracee-rules
+      - name: Run Integration Tests
         run: |
-          make test-tracee-rules
+          make test-integration
+
   verify-signatures:
     name: Verify Signatures
     needs:
       - verify-code
+      - run-unit-tests
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout Code
@@ -98,8 +95,7 @@ jobs:
   release-snapshot:
     name: Release Snapshot
     needs:
-      - verify-tracee-ebpf
-      - verify-tracee-rules
+      - run-integration-tests
       - verify-signatures
     runs-on: ubuntu-20.04
     steps:
@@ -137,8 +133,7 @@ jobs:
   smoke-test-noncore:
     name: "[Smoke] Test non CO-RE"
     needs:
-      - verify-tracee-ebpf
-      - verify-tracee-rules
+      - run-integration-tests
       - verify-signatures
     runs-on: ubuntu-20.04
     steps:
@@ -160,7 +155,7 @@ jobs:
       - name: Run tests
         run: |
           docker image pull aquasec/tracee-tester:latest
-          go test -v -run "TestTraceeSignatures" ./tests/tracee_test.go \
+          go test -v -run "TestTraceeSignatures" ./tests/e2e/e2e_test.go \
             -tracee-image-ref "tracee:full" \
             -tracee-tester-image-ref "aquasec/tracee-tester:latest" \
             -tracee-signatures "TRC-2"
@@ -171,8 +166,7 @@ jobs:
   smoke-test-core:
     name: "[Smoke] Test CO-RE"
     needs:
-      - verify-tracee-ebpf
-      - verify-tracee-rules
+      - run-integration-tests
       - verify-signatures
     runs-on: ubuntu-20.04
     steps:
@@ -194,7 +188,7 @@ jobs:
       - name: Run tests
         run: |
           docker image pull aquasec/tracee-tester:latest
-          go test -v -run "TestTraceeSignatures" ./tests/tracee_test.go \
+          go test -v -run "TestTraceeSignatures" ./tests/e2e/e2e_test.go \
             -tracee-image-ref "tracee:latest" \
             -tracee-tester-image-ref "aquasec/tracee-tester:latest" \
             -tracee-signatures "TRC-2"

.github/workflows/scheduled-signatures-test.yaml

@@ -33,6 +33,6 @@ jobs:
       - name: Run tests
         run: |
           docker image pull aquasec/tracee-tester:latest
-          go test -v -run "TestTraceeSignatures" ./tests/tracee_test.go \
+          go test -v -run "TestTraceeSignatures" ./tests/e2e/e2e_test.go \
             -tracee-image-ref "tracee:latest" \
             -tracee-tester-image-ref "aquasec/tracee-tester:latest"

Makefile

@@ -234,9 +234,9 @@ help:
 	@echo ""
 	@echo "# test"
 	@echo ""
-	@echo "    $$ make test                 	# run all go & opa tests"
-	@echo "    $$ make test-tracee-ebpf     	# go test tracee-ebpf"
-	@echo "    $$ make test-tracee-rules    	# go test tracee-rules"
+	@echo "    $$ make test-types           	# run unit tests for types module"
+	@echo "    $$ make test-unit            	# run unit tests"
+	@echo "    $$ make test-integration     	# run integration tests"
 	@echo "    $$ make test-rules           	# opa test (tracee-rules)"
 	@echo ""
 	@echo "# flags"
@@ -488,18 +488,6 @@ clean-tracee-ebpf:
 	$(CMD_RM) -rf $(OUTPUT_DIR)/tracee-ebpf
 	$(CMD_RM) -rf .*.md5
 
-.PHONY: test-tracee-ebpf
-test-tracee-ebpf: \
-	.checkver_$(CMD_GO) \
-	$(OUTPUT_DIR)/tracee.bpf.core.o
-#
-	$(MAKE) $(OUTPUT_DIR)/btfhub
-	$(GO_ENV_EBPF) $(CMD_GO) test \
-		-tags $(GO_TAGS_EBPF) \
-		-v \
-		./cmd/tracee-ebpf/... \
-		./pkg/...
-
 #
 # btfhub (expensive: only run if core obj changed)
 #
@@ -567,17 +555,44 @@ clean-tracee-rules:
 #
 	$(CMD_RM) -rf $(OUTPUT_DIR)/tracee-rules
 
-.PHONY: test-tracee-rules
-test-tracee-rules: \
+.PHONY: test-unit
+test-unit: \
+	.checkver_$(CMD_GO) \
+	tracee-ebpf \
+	test-types
+#
+	$(GO_ENV_EBPF) \
+	$(CMD_GO) test \
+		-tags ebpf \
+		-short \
+		-v \
+		-coverprofile=coverage.txt \
+		./...
+
+.PHONY: test-types
+test-types: \
 	.checkver_$(CMD_GO)
 #
-	$(GO_ENV_RULES) $(CMD_GO) test \
-		-tags $(GO_TAGS_RULES) \
+	# Note that we must changed the directory here because types is a standalone Go module.
+	cd ./types && $(CMD_GO) test \
+		-short \
+		-race \
+		-v \
+		-coverprofile=coverage.txt \
+		./...
+
+.PHONY: test-integration
+test-integration: \
+	.checkver_$(CMD_GO) \
+	tracee-ebpf
+#
+	TRC_BIN=$(abspath $(OUTPUT_DIR)/tracee-ebpf) \
+	$(GO_ENV_EBPF) \
+	$(CMD_GO) test \
+		-tags ebpf,integration \
 		-v \
-		./cmd/tracee-rules/... \
-		./pkg/rules/...
-	cd ./types; $(CMD_GO) test -v ./...
-	cd ..
+		-run "Test_Events" ./tests/integration/...
+
 #
 # rules
 #
@@ -626,13 +641,6 @@ test-rules: \
 #
 	$(CMD_OPA) test $(REGO_SIGNATURES_DIR) --verbose
 
-#
-# test
-#
-
-.PHONY: test
-test: test-tracee-ebpf test-tracee-rules test-rules
-
 #
 # clean
 #

Vagrantfile

@@ -2,9 +2,9 @@
 # vi: set ft=ruby :
 
 Vagrant.configure("2") do |config|
-  # config.vm.box = "ubuntu/focal64"     # Ubuntu 20.04 Focal Fossa (no CORE)
-  # config.vm.box = "ubuntu/hirsute64"   # Ubuntu 21.04 Hirsute Hippo (CORE)
-  config.vm.box = "ubuntu/impish64"      # Ubuntu 21.10 Impish Indri (CORE)
+  # config.vm.box = "ubuntu/focal64"     # Ubuntu 20.04 Focal Fossa (non CO-RE)
+  # config.vm.box = "ubuntu/hirsute64"   # Ubuntu 21.04 Hirsute Hippo (CO-RE)
+  config.vm.box = "ubuntu/impish64"      # Ubuntu 21.10 Impish Indri (CO-RE)
 
   config.vm.provider "virtualbox" do |vb|
     vb.gui = false

cmd/tracee-rules/output_test.go

@@ -251,14 +251,14 @@ HostName: foobar.local
 	}
 }
 
-func TestJsonTemplateV1(t *testing.T) {
+func TestOutputTemplates(t *testing.T) {
 	testCases := []struct {
 		testName     string
 		finding      detect.Finding
 		expectedJson string
 	}{
 		{
-			testName: "convert finding to v1",
+			testName: "Should output finding as raw JSON",
 			finding: detect.Finding{
 				Data: map[string]interface{}{
 					"a": 123,
@@ -303,15 +303,13 @@ func TestJsonTemplateV1(t *testing.T) {
 	}
 
 	jsonTemplate, err := setupTemplate("templates/rawjson.tmpl")
-
 	require.NoError(t, err)
 
 	for _, tc := range testCases {
 		t.Run(tc.testName, func(t *testing.T) {
-			//pass finding through template and compare to a v1 json
 			var buf bytes.Buffer
-			jsonTemplate.Execute(&buf, tc.finding)
-
+			err := jsonTemplate.Execute(&buf, tc.finding)
+			require.NoError(t, err)
 			assert.JSONEq(t, tc.expectedJson, buf.String())
 		})
 	}

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/aquasecurity/tracee/types v0.0.0-20220228102148-dffb469aed94
 	github.com/google/gopacket v1.1.19
 	github.com/hashicorp/golang-lru v0.5.4
-	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/mitchellh/go-ps v1.0.0
 	github.com/onsi/gomega v1.17.0
 	github.com/open-policy-agent/opa v0.35.0

go.sum

@@ -481,8 +481,6 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
-github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=