add revised opensearch query builder package (#249)

## What

Add revised openSearch query builder package `osquery`. Keep the old
package for backwards compatibility.

Notable changes:

- mapping entirely handled by passed filter field mapping (no setting
for appending `.keyword` to the filter field name anymore)
- improved error handling: fail on invalid input rather than executing
an ambiguous query
- removed several specific filters
- removed all nested query functionality

Also added tests which test against a real openSearch instance.

## Why

The old version was inconvenient to use and error handling was not
ideal.

## References

VTI-255

Author: mgoetzegb

.github/actions/go-setup/action.yaml

@@ -4,7 +4,6 @@ description: 'Setup go and cache and go download mods'
 runs:
   using: "composite"
   steps:
-    - uses: actions/checkout@v4
     - uses: actions/setup-go@v4
       with:
         cache: true

.github/workflows/codecov.yml

@@ -13,5 +13,29 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - uses: ./.github/actions/go-setup
-      - run: |
-          go test -v ./...
+      - run: make test-codecov
+      - name: Upload unit test coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          files: cov-unit-tests.out
+          flags: unit-tests
+          name: Unit Tests
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true
+
+  opensearch-test:
+    name: Execute Opensearch tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/go-setup
+      - name: Execute OpenSearch tests
+        run: make test-opensearch
+      - name: Upload OpenSearch test coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          files: cov-os-tests.out
+          flags: opensearch-tests
+          name: OpenSearch Tests
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true

.github/workflows/tests.yml

@@ -1 +1,6 @@
+# common IDEs
 .idea
+.vscode
+
+# coverage reports
+*.out

.gitignore

@@ -3,6 +3,7 @@ REGISTRY := docker-gps.greenbone.net
 # Define submodules
 PKG_DIR := pkg
 
+.PHONY: all
 all: build test
 
 .PHONY: help
@@ -27,6 +28,7 @@ OS="$(shell go env var GOOS | xargs)"
 ALL_GO_DIRS := $(shell find $(PKG_DIR) -name '*.go' -exec dirname {} \; | sort -u)
 
 # Clean up
+.PHONY: clean
 clean:
 	go clean -i ./...
 
@@ -64,19 +66,40 @@ build: build-common ## build go library packages
 	go build -trimpath ./...
 
 .PHONY: test
-test: ## run all tests
-	go test -test.short ./...
+test: ## run short tests
+	go test -short ./...
 
 .PHONY: test-codecov
 test-codecov:
-	go test -cover -coverprofile=coverage.txt ./...
-
-.PHONY: all build test clean
+	go test -cover -coverprofile=cov-unit-tests.out ./...
+
+.PHONY: start-opensearch-test-service
+start-opensearch-test-service:
+	docker compose -f ./pkg/openSearch/ostesting/compose.yml -p opensearch-test up -d --wait --wait-timeout=120
+
+.PHONY: stop-opensearch-test-service
+stop-opensearch-test-service:
+	docker compose -f ./pkg/openSearch/ostesting/compose.yml -p opensearch-test down -v
+
+.PHONY: run-opensearch-tests
+run-opensearch-tests:
+	TEST_OPENSEARCH=1 go test ./pkg/openSearch/osquery -coverprofile=cov-os-tests.out
+
+# test-opensearch runs whole opensearch execution with docker setup and cleanup
+# we want to stop-opensearch-test-service regardless of the result of run-opensearch-tests
+# and return the result of run-opensearch-tests
+.PHONY: test-opensearch
+test-opensearch:
+	$(MAKE) start-opensearch-test-service
+	$(MAKE) run-opensearch-tests; \
+	status=$$?; \
+	$(MAKE) stop-opensearch-test-service; \
+	exit $$status
 
 .PHONY: generate_docs
 generate_docs: check_tools
 	gomarkdoc -e --output '{{.Dir}}/README.md' \
-		--exclude-dirs .,./pkg/configReader/helper,./pkg/dbcrypt/config,./pkg/openSearch/openSearchClient/config,./pkg/swagger/ginSwagger.go \
+		--exclude-dirs .,./pkg/configReader/helper,./pkg/dbcrypt/config,./pkg/openSearch/openSearchClient/config \
 		./...
 
 check_tools:

Makefile

@@ -8,11 +8,11 @@ subpackages:
 * [esextensions](esextension/README.md) - extensions for the esquery library
 * [openSearchClient](openSearchClient/README.md) - a client for OpenSearch designed to allow easy mocking/
 * [openSearchQuery](openSearchQuery/README.md) - query builders for OpenSearch
+* [osquery](osquery/README.md) - query builders for OpenSearch (simplified version)
+* [ostesting](ostesting/README.md) - conveniently test against a real openSearch instance
 
 # License
 
 Copyright (C) 2022-2023 [Greenbone AG][Greenbone AG]
 
 Licensed under the [GNU General Public License v3.0 or later](../../LICENSE).
-
-

pkg/openSearch/README.md

@@ -567,7 +567,7 @@ func NewIndexFunction(openSearchProjectClient *opensearchapi.Client) *IndexFunct
 
 
 <a name="IndexFunction.AliasExists"></a>
-### func \(\*IndexFunction\) [AliasExists](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L219>)
+### func \(\*IndexFunction\) [AliasExists](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L238>)
 
 ```go
 func (i *IndexFunction) AliasExists(aliasName string) (bool, error)
@@ -594,7 +594,7 @@ func (i *IndexFunction) CreateOrPutAlias(aliasName string, indexNames ...string)
 
 
 <a name="IndexFunction.DeleteAliasFromIndex"></a>
-### func \(\*IndexFunction\) [DeleteAliasFromIndex](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L168>)
+### func \(\*IndexFunction\) [DeleteAliasFromIndex](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L187>)
 
 ```go
 func (i *IndexFunction) DeleteAliasFromIndex(indexName string, aliasName string) error
@@ -612,7 +612,7 @@ func (i *IndexFunction) DeleteIndex(indexName string) error
 
 
 <a name="IndexFunction.ForceMerge"></a>
-### func \(\*IndexFunction\) [ForceMerge](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L395>)
+### func \(\*IndexFunction\) [ForceMerge](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L414>)
 
 ```go
 func (i *IndexFunction) ForceMerge(index string, maximumNumberOfSegments int) error
@@ -621,7 +621,7 @@ func (i *IndexFunction) ForceMerge(index string, maximumNumberOfSegments int) er
 
 
 <a name="IndexFunction.GetIndexSettings"></a>
-### func \(\*IndexFunction\) [GetIndexSettings](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L342>)
+### func \(\*IndexFunction\) [GetIndexSettings](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L361>)
 
 ```go
 func (i *IndexFunction) GetIndexSettings(index string) (map[string]interface{}, error)
@@ -639,7 +639,7 @@ func (i *IndexFunction) GetIndexes(pattern string) ([]string, error)
 
 
 <a name="IndexFunction.GetIndexesForAlias"></a>
-### func \(\*IndexFunction\) [GetIndexesForAlias](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L248>)
+### func \(\*IndexFunction\) [GetIndexesForAlias](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L267>)
 
 ```go
 func (i *IndexFunction) GetIndexesForAlias(aliasName string) ([]string, error)
@@ -657,7 +657,7 @@ func (i *IndexFunction) IndexExists(indexName string) (bool, error)
 
 
 <a name="IndexFunction.IndexHasAlias"></a>
-### func \(\*IndexFunction\) [IndexHasAlias](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L194>)
+### func \(\*IndexFunction\) [IndexHasAlias](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L213>)
 
 ```go
 func (i *IndexFunction) IndexHasAlias(indexNames []string, aliasNames []string) (bool, error)
@@ -666,7 +666,7 @@ func (i *IndexFunction) IndexHasAlias(indexNames []string, aliasNames []string)
 
 
 <a name="IndexFunction.RefreshIndex"></a>
-### func \(\*IndexFunction\) [RefreshIndex](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L318>)
+### func \(\*IndexFunction\) [RefreshIndex](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L337>)
 
 ```go
 func (i *IndexFunction) RefreshIndex(index string) error
@@ -675,7 +675,7 @@ func (i *IndexFunction) RefreshIndex(index string) error
 
 
 <a name="IndexFunction.RemoveIndexesFromAlias"></a>
-### func \(\*IndexFunction\) [RemoveIndexesFromAlias](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L267>)
+### func \(\*IndexFunction\) [RemoveIndexesFromAlias](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L286>)
 
 ```go
 func (i *IndexFunction) RemoveIndexesFromAlias(indexesToRemove []string, aliasName string) error
@@ -684,7 +684,7 @@ func (i *IndexFunction) RemoveIndexesFromAlias(indexesToRemove []string, aliasNa
 
 
 <a name="IndexFunction.SetIndexSettings"></a>
-### func \(\*IndexFunction\) [SetIndexSettings](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L370>)
+### func \(\*IndexFunction\) [SetIndexSettings](<https://github.com/greenbone/opensight-golang-libraries/blob/main/pkg/openSearch/openSearchClient/indexFunctions.go#L389>)
 
 ```go
 func (i *IndexFunction) SetIndexSettings(index string, settingsBody io.Reader) error

pkg/openSearch/openSearchClient/README.md

@@ -36,7 +36,7 @@ func NewOpenSearchProjectClient(ctx context.Context, config config.OpensearchCli
 			Client: opensearch.Config{
 				Transport: &http.Transport{
 					TLSClientConfig: &tls.Config{
-						InsecureSkipVerify: true, // nolint:gosec
+						InsecureSkipVerify: true, //nolint:gosec
 					},
 				},
 				Addresses: []string{

pkg/openSearch/openSearchClient/opensearchProjectClient.go

@@ -224,7 +224,7 @@ func createMappedField(dtoField filter.RequestField, fieldMapping map[string]str
 	entityName, ok := fieldMapping[dtoField.Name]
 	if !ok {
 		return filter.RequestField{}, filter.NewInvalidFilterFieldError(
-			"Mapping for filter field '%s' is currently not implemented.", dtoField.Name)
+			"mapping for filter field '%s' is currently not implemented", dtoField.Name)
 	}
 
 	return filter.RequestField{

pkg/openSearch/openSearchQuery/boolQueryBuilder.go

@@ -137,7 +137,7 @@ func HandleCompareOperatorNotBeginsWith(fieldName string, fieldKeys []string, fi
 				})...,
 			)
 	} else { // for single values
-		return esquery.Prefix(fieldName+".keyword", fieldValue.(string))
+		return esquery.Bool().MustNot(esquery.Prefix(fieldName+".keyword", fieldValue.(string)))
 	}
 }