ci: add code coverage publishing and check to pipeline (#3544)

* generate code coverage

* ut regression check

* move coverage report generation to separate step

* move link where it's needed

* add windows ut coverage

* publish multiple coverage reports since unable to merge diff oses

* remove debug always succeed

Author: QxBytes

Diff for: .pipelines/templates/run-unit-tests.yaml

@@ -12,18 +12,28 @@ stages:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
         steps:
           - script: |
+              set -e
               make tools
-              # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
+
+              # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml),
+              # stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code.
+              # Read all output from fd 4 (output from tee) and write to top stdout
               { { { {
                     sudo -E env "PATH=$PATH" make test-all;
                     echo $? >&3;
                     } | tee >(build/tools/bin/go-junit-report > report.xml) >&4;
                   } 3>&1;
                 } | { read xs; exit $xs; }
               } 4>&1
+
+              mv coverage-all.out linux-coverage.out
             retryCountOnTaskFailure: 3
             name: "Test"
             displayName: "Run Tests"
+          - task: PublishPipelineArtifact@1
+            inputs:
+              targetPath: 'linux-coverage.out'
+              artifactName: 'linux-coverage'
 
   - stage: test_windows
     displayName: Test ACN Windows
@@ -40,7 +50,66 @@ stages:
           # Only run one go test per script
           - script: |
               cd azure-container-networking/
-              go test -timeout 30m ./npm/... ./cni/... ./platform/...
+              go test -timeout 30m -covermode atomic -coverprofile=windows-coverage.out ./npm/... ./cni/... ./platform/...
+              go tool cover -func=windows-coverage.out
             retryCountOnTaskFailure: 3
             name: "TestWindows"
             displayName: "Run Windows Tests"
+          - task: PublishPipelineArtifact@1
+            inputs:
+              targetPath: 'windows-coverage.out'
+              artifactName: 'windows-coverage'
+
+  - stage: code_coverage
+    displayName: Code Coverage Check
+    dependsOn:
+      - test
+      - test_windows
+    jobs:
+      - job: coverage
+        displayName: Check Coverage
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+        steps:
+          - task: DownloadPipelineArtifact@2
+            inputs:
+              artifact: 'linux-coverage'
+              path: './'
+          - task: DownloadPipelineArtifact@2
+            inputs:
+              artifact: 'windows-coverage'
+              path: './'
+          - bash: |
+              make tools
+              sudo ln -s $(pwd)/build/tools/bin/gocov /usr/local/bin/gocov
+              sudo ln -s $(pwd)/build/tools/bin/gocov-xml /usr/local/bin/gocov-xml
+
+              GOOS=linux gocov convert linux-coverage.out > linux-coverage.json
+              GOOS=linux gocov-xml < linux-coverage.json > linux-coverage.xml
+
+              GOOS=windows gocov convert windows-coverage.out > windows-coverage.json
+              GOOS=windows gocov-xml < windows-coverage.json > windows-coverage.xml
+
+              mkdir coverage
+
+              mv linux-coverage.xml coverage/
+              mv windows-coverage.xml coverage/
+            name: "Coverage"
+            displayName: "Generate Coverage Report"
+            condition: always()
+          - task: PublishCodeCoverageResults@2
+            displayName: "Publish Code Coverage Report"
+            condition: always()
+            inputs:
+              summaryFileLocation: coverage/*
+          - task: BuildQualityChecks@8
+            displayName: "Check Code Coverage Regression"
+            condition: always()
+            inputs:
+              checkCoverage: true
+              coverageFailOption: "build"
+              coverageType: "lines"
+              fallbackOnPRTargetBranch: false
+              baseBranchRef: "master"
+              allowCoverageVariance: true
+              coverageVariance: 0.25

Diff for: Makefile

@@ -758,9 +758,8 @@ RESTART_CASE ?= false
 CNI_TYPE ?= cilium
 
 test-all: ## run all unit tests.
-	@$(eval COVER_FILTER=`go list $(COVER_PKG)/... | tr '\n' ','`)
-	@echo Test coverpkg: $(COVER_FILTER)
-	go test -mod=readonly -buildvcs=false -tags "unit" --skip 'TestE2E*' -coverpkg=$(COVER_FILTER) -race -covermode atomic -coverprofile=coverage.out $(COVER_PKG)/...
+	go test -mod=readonly -buildvcs=false -tags "unit" --skip 'TestE2E*' -race -covermode atomic -coverprofile=coverage-all.out $(COVER_PKG)/...
+	go tool cover -func=coverage-all.out
 
 test-integration: ## run all integration tests.
 	AZURE_IPAM_VERSION=$(AZURE_IPAM_VERSION) \