diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md index 31c7aa3b8a..50dd8bca32 100644 --- a/docs/howto/system_testing.md +++ b/docs/howto/system_testing.md @@ -534,12 +534,14 @@ for system tests. | policy_template | string | | Name of policy template associated with the data stream and input. Required when multiple policy templates include the input being tested. | | service | string | | Name of a specific Docker service to setup for the test. | | service_notify_signal | string | | Signal name to send to 'service' when the test policy has been applied to the Agent. This can be used to trigger the service after the Agent is ready to receive data. | +| signal_types | []string | | For otel packages with dynamic_signal_types, this specifies which signal data streams to assert on. Otherwise all detected data streams for the dataset will be asserted against. | skip.link | URL | | URL linking to an issue about why the test is skipped. | | skip.reason | string | | Reason to skip the test. If specified the test will not execute. | | skip_ignored_fields | array string | | List of fields to be skipped when performing validation of fields ignored during ingestion. | | skip_transform_validation | boolean | | Disable or enable the transforms validation performed in system tests. | | vars | dictionary | | Package level variables to set (i.e. declared in `$package_root/manifest.yml`). If not specified the defaults from the manifest are used. | | wait_for_data_timeout | duration | | Amount of time to wait for data to be present in Elasticsearch. Defaults to 10m. | +| wait_for_dynamic_streams_stable | duration | | For packages with dynamic data streams (e.g. `dynamic_signal_types`), minimum time the discovered data stream count must stay unchanged after the first stream appears before discovery completes. Defaults to 10s. | For example, the `apache/access` data stream's `test-access-log-config.yml` is shown below. diff --git a/internal/docs/readme_test.go b/internal/docs/readme_test.go index 1d120b9d1f..43559ea321 100644 --- a/internal/docs/readme_test.go +++ b/internal/docs/readme_test.go @@ -5,6 +5,7 @@ package docs import ( + "fmt" "os" "path/filepath" "testing" @@ -16,6 +17,21 @@ import ( "github.com/elastic/elastic-package/internal/packages" ) +const ( + sampleEventReadmeTemplateRel = "_dev/build/docs/README.md" + sampleEventReadmeDocPrefix = `{{- generatedHeader }} +# README +Introduction to the package +` +) + +func readmeEventTemplateLine(eventTarget string) string { + if eventTarget == "" { + return "{{ event }}" + } + return fmt.Sprintf(`{{ event %q }}`, eventTarget) +} + func TestGenerateReadme(t *testing.T) { cases := []struct { title string @@ -134,36 +150,76 @@ http://www.example.com/bar func TestRenderReadmeWithSampleEvent(t *testing.T) { cases := []struct { - title string - packageRoot string - templatePath string - dataStreamName string - readmeTemplateContents string - sampleEventJsonContents string - expected string + title string + eventTarget string // data stream folder and {{ event "…" }} argument; empty → package root and {{ event }} + plainValue string // if set, writes sample_event.json with JSON {"v": plainValue} + namedSamples []string + expected string }{ { - title: "README with sample event", - packageRoot: t.TempDir(), - templatePath: "_dev/build/docs/README.md", - readmeTemplateContents: ` -{{- generatedHeader }} + title: "README with sample event", + eventTarget: "example", + plainValue: "event1", + expected: ` + # README Introduction to the package -{{ event "example" }}`, +An example event for ` + "`example`" + ` looks as following: + +` + "```json" + ` +{ + "v": "event1" +} +` + "```" + "\n", + }, + { + title: "named sample_event_*.json only uses first lexicographic basename", + eventTarget: "otel_ds", + namedSamples: []string{"metrics", "logs"}, expected: ` # README Introduction to the package -An example event for ` + "`example`" + ` looks as following: +An example event for ` + "`otel_ds`" + ` looks as following: + +` + "```json" + ` +{ + "v": "logs" +} +` + "```" + "\n", + }, + { + title: "sample_event.json is preferred over sample_event_*.json", + eventTarget: "otel_ds", + plainValue: "plain", + namedSamples: []string{"logs"}, + expected: ` + +# README +Introduction to the package +An example event for ` + "`otel_ds`" + ` looks as following: + +` + "```json" + ` +{ + "v": "plain" +} +` + "```" + "\n", + }, + { + title: "package root: named sample_event_*.json files only uses first lexicographic basename", + eventTarget: "", + namedSamples: []string{"metrics", "logs"}, + expected: ` + +# README +Introduction to the package +An example event looks as following: ` + "```json" + ` { - "id": "event1" + "v": "logs" } -` + "```", - dataStreamName: "example", - sampleEventJsonContents: `{"id": "event1"}`, +` + "```" + "\n", }, } @@ -171,22 +227,30 @@ An example event for ` + "`example`" + ` looks as following: urls := fields.SchemaURLs{} for _, c := range cases { t.Run(c.title, func(t *testing.T) { - filename := filepath.Base(c.templatePath) - templatePath := filepath.Join(c.packageRoot, c.templatePath) + packageRoot := t.TempDir() + filename := filepath.Base(sampleEventReadmeTemplateRel) + templatePath := filepath.Join(packageRoot, sampleEventReadmeTemplateRel) - createReadmeTemplateFile(t, c.packageRoot, c.readmeTemplateContents) - createSampleEventFile(t, c.packageRoot, c.dataStreamName, c.sampleEventJsonContents) - createManifestFile(t, c.packageRoot) + readmeContents := sampleEventReadmeDocPrefix + readmeEventTemplateLine(c.eventTarget) + "\n" + createReadmeTemplateFile(t, packageRoot, readmeContents) - root, err := os.OpenRoot(c.packageRoot) + ds := c.eventTarget + if c.plainValue != "" { + createSampleEventFile(t, packageRoot, ds, fmt.Sprintf(`{"v": %q}`, c.plainValue)) + } + for _, name := range c.namedSamples { + createNamedSampleEventFile(t, packageRoot, ds, name, fmt.Sprintf(`{"v": %q}`, name)) + } + createManifestFile(t, packageRoot) + + root, err := os.OpenRoot(packageRoot) require.NoError(t, err) t.Cleanup(func() { root.Close() }) - rendered, err := renderReadme(root, filename, c.packageRoot, templatePath, linksMap, urls) + rendered, err := renderReadme(root, filename, packageRoot, templatePath, linksMap, urls) require.NoError(t, err) - renderedString := string(rendered) - assert.Equal(t, c.expected, renderedString) + assert.Equal(t, c.expected, string(rendered)) }) } } @@ -500,10 +564,26 @@ func createDocsFolder(t *testing.T, packageRoot string) string { func createSampleEventFile(t *testing.T, packageRoot, dataStreamName, contents string) { t.Helper() - dataStreamFolder := createDataStreamFolder(t, packageRoot, dataStreamName) + writeSampleEventAt(t, packageRoot, dataStreamName, sampleEventFile, contents) +} + +// createNamedSampleEventFile writes sample_event_.json. +func createNamedSampleEventFile(t *testing.T, packageRoot, dataStreamName, name, contents string) { + t.Helper() + require.NotEmpty(t, name, "sample event name is required") + writeSampleEventAt(t, packageRoot, dataStreamName, fmt.Sprintf("sample_event_%s.json", name), contents) +} - sampleEventFile := filepath.Join(dataStreamFolder, sampleEventFile) - err := os.WriteFile(sampleEventFile, []byte(contents), 0644) +func writeSampleEventAt(t *testing.T, packageRoot, dataStreamName, fileName, contents string) { + t.Helper() + var dir string + if dataStreamName == "" { + dir = packageRoot + } else { + dir = createDataStreamFolder(t, packageRoot, dataStreamName) + } + path := filepath.Join(dir, fileName) + err := os.WriteFile(path, []byte(contents), 0644) require.NoError(t, err) } diff --git a/internal/docs/sample_event.go b/internal/docs/sample_event.go index b0ef3f8e54..70eb5241ac 100644 --- a/internal/docs/sample_event.go +++ b/internal/docs/sample_event.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" "github.com/Masterminds/semver/v3" @@ -20,11 +21,15 @@ import ( const sampleEventFile = "sample_event.json" func renderSampleEvent(packageRoot, dataStreamName string) (string, error) { - var eventPath string + var dir string if dataStreamName == "" { - eventPath = filepath.Join(packageRoot, sampleEventFile) + dir = packageRoot } else { - eventPath = filepath.Join(packageRoot, "data_stream", dataStreamName, sampleEventFile) + dir = filepath.Join(packageRoot, "data_stream", dataStreamName) + } + eventPath, err := resolveSampleEventPath(dir) + if err != nil { + return "", err } body, err := os.ReadFile(eventPath) @@ -60,6 +65,27 @@ func renderSampleEvent(packageRoot, dataStreamName string) (string, error) { return builder.String(), nil } +func resolveSampleEventPath(dir string) (string, error) { + plain := filepath.Join(dir, sampleEventFile) + if info, err := os.Stat(plain); err == nil && !info.IsDir() { + return plain, nil + } + + matches, err := filepath.Glob(filepath.Join(dir, "sample_event_*.json")) + if err != nil { + return "", fmt.Errorf("glob sample_event_*.json failed (dir: %s): %w", dir, err) + } + if len(matches) == 0 { + return "", fmt.Errorf("sample event file not found (looked for %s and sample_event_*.json under %s): %w", + sampleEventFile, dir, os.ErrNotExist) + } + + sort.Slice(matches, func(i, j int) bool { + return filepath.Base(matches[i]) < filepath.Base(matches[j]) + }) + return matches[0], nil +} + func stripDataStreamFolderSuffix(dataStreamName string) string { dataStreamName = strings.ReplaceAll(dataStreamName, "_metrics", "") dataStreamName = strings.ReplaceAll(dataStreamName, "_logs", "") diff --git a/internal/packages/packages.go b/internal/packages/packages.go index f86c57bed5..48d20b8803 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -221,10 +221,11 @@ type PolicyTemplate struct { Inputs []Input `config:"inputs,omitempty" json:"inputs,omitempty" yaml:"inputs,omitempty"` // For purposes of "input packages" - Input string `config:"input,omitempty" json:"input,omitempty" yaml:"input,omitempty"` - Type string `config:"type,omitempty" json:"type,omitempty" yaml:"type,omitempty"` - TemplatePath string `config:"template_path,omitempty" json:"template_path,omitempty" yaml:"template_path,omitempty"` - Vars []Variable `config:"vars,omitempty" json:"vars,omitempty" yaml:"vars,omitempty"` + Input string `config:"input,omitempty" json:"input,omitempty" yaml:"input,omitempty"` + Type string `config:"type,omitempty" json:"type,omitempty" yaml:"type,omitempty"` + DynamicSignalTypes bool `config:"dynamic_signal_types,omitempty" json:"dynamic_signal_types,omitempty" yaml:"dynamic_signal_types,omitempty"` + TemplatePath string `config:"template_path,omitempty" json:"template_path,omitempty" yaml:"template_path,omitempty"` + Vars []Variable `config:"vars,omitempty" json:"vars,omitempty" yaml:"vars,omitempty"` } // Owner defines package owners, either a single person or a team. diff --git a/internal/testrunner/runners/static/runner.go b/internal/testrunner/runners/static/runner.go index 102be38c47..566ec4be40 100644 --- a/internal/testrunner/runners/static/runner.go +++ b/internal/testrunner/runners/static/runner.go @@ -20,6 +20,7 @@ const ( TestType testrunner.TestType = "static" sampleEventJSON = "sample_event.json" + sampleEventGlob = "sample_event*.json" ) type runner struct { diff --git a/internal/testrunner/runners/static/tester.go b/internal/testrunner/runners/static/tester.go index 0bc3b6037d..80a9c9118e 100644 --- a/internal/testrunner/runners/static/tester.go +++ b/internal/testrunner/runners/static/tester.go @@ -6,7 +6,6 @@ package static import ( "context" - "errors" "fmt" "os" "path/filepath" @@ -101,8 +100,8 @@ func (r tester) run(ctx context.Context) ([]testrunner.TestResult, error) { return result.WithError(fmt.Errorf("failed to read manifest: %w", err)) } - // join together results from verifyStreamConfig and verifySampleEvent - return append(r.verifyStreamConfig(ctx, r.packageRoot), r.verifySampleEvent(pkgManifest)...), nil + // join together results from verifyStreamConfig and verifySampleEvents + return append(r.verifyStreamConfig(ctx, r.packageRoot), r.verifySampleEvents(pkgManifest)...), nil } func (r tester) verifyStreamConfig(ctx context.Context, packageRoot string) []testrunner.TestResult { @@ -134,44 +133,61 @@ func (r tester) verifyStreamConfig(ctx context.Context, packageRoot string) []te return results } -func (r tester) verifySampleEvent(pkgManifest *packages.PackageManifest) []testrunner.TestResult { - resultComposer := testrunner.NewResultComposer(testrunner.TestResult{ +func (r tester) verifySampleEvents(pkgManifest *packages.PackageManifest) []testrunner.TestResult { + defaultResultComposer := testrunner.NewResultComposer(testrunner.TestResult{ Name: "Verify " + sampleEventJSON, TestType: TestType, Package: r.testFolder.Package, DataStream: r.testFolder.DataStream, }) - sampleEventPath, found, err := r.getSampleEventPath() + sampleEventPaths, err := r.getSampleEventPaths() if err != nil { - results, _ := resultComposer.WithError(err) + results, _ := defaultResultComposer.WithError(err) return results } - if !found { + if len(sampleEventPaths) == 0 { // Nothing to do. return []testrunner.TestResult{} } - if r.withCoverage { - coverage, err := testrunner.GenerateBaseFileCoverageReport(resultComposer.CoveragePackageName(), sampleEventPath, r.coverageType, true) - if err != nil { - results, _ := resultComposer.WithErrorf("coverage report generation failed: %w", err) - return results - } - resultComposer = resultComposer.WithCoverage(coverage) - } - expectedDatasets, err := r.getExpectedDatasets(pkgManifest) if err != nil { - results, _ := resultComposer.WithError(err) + results, _ := defaultResultComposer.WithError(err) return results } repositoryRoot, err := files.FindRepositoryRootFrom(r.packageRoot) if err != nil { - results, _ := resultComposer.WithErrorf("cannot find repository root from %s: %w", r.packageRoot, err) + results, _ := defaultResultComposer.WithErrorf("cannot find repository root from %s: %w", r.packageRoot, err) return results } defer repositoryRoot.Close() + + var allResults []testrunner.TestResult + for _, sampleEventPath := range sampleEventPaths { + results := r.verifySampleEvent(sampleEventPath, pkgManifest, expectedDatasets, repositoryRoot) + allResults = append(allResults, results...) + } + return allResults +} + +func (r tester) verifySampleEvent(sampleEventPath string, pkgManifest *packages.PackageManifest, expectedDatasets []string, repositoryRoot *os.Root) []testrunner.TestResult { + resultComposer := testrunner.NewResultComposer(testrunner.TestResult{ + Name: "Verify " + filepath.Base(sampleEventPath), + TestType: TestType, + Package: r.testFolder.Package, + DataStream: r.testFolder.DataStream, + }) + + if r.withCoverage { + coverage, err := testrunner.GenerateBaseFileCoverageReport(resultComposer.CoveragePackageName(), sampleEventPath, r.coverageType, true) + if err != nil { + results, _ := resultComposer.WithErrorf("coverage report generation failed: %w", err) + return results + } + resultComposer = resultComposer.WithCoverage(coverage) + } + fieldsDir := filepath.Join(filepath.Dir(sampleEventPath), "fields") fieldsValidator, err := fields.CreateValidator(repositoryRoot, r.packageRoot, fieldsDir, fields.WithSpecVersion(pkgManifest.SpecVersion), @@ -205,25 +221,18 @@ func (r tester) verifySampleEvent(pkgManifest *packages.PackageManifest) []testr return results } -func (r tester) getSampleEventPath() (string, bool, error) { - var sampleEventPath string +func (r tester) getSampleEventPaths() ([]string, error) { + var dir string if r.testFolder.DataStream != "" { - sampleEventPath = filepath.Join( - r.packageRoot, - "data_stream", - r.testFolder.DataStream, - sampleEventJSON) + dir = filepath.Join(r.packageRoot, "data_stream", r.testFolder.DataStream) } else { - sampleEventPath = filepath.Join(r.packageRoot, sampleEventJSON) - } - _, err := os.Stat(sampleEventPath) - if errors.Is(err, os.ErrNotExist) { - return "", false, nil + dir = r.packageRoot } + matches, err := filepath.Glob(filepath.Join(dir, sampleEventGlob)) if err != nil { - return "", false, fmt.Errorf("stat file failed: %w", err) + return nil, fmt.Errorf("globbing for sample event files failed: %w", err) } - return sampleEventPath, true, nil + return matches, nil } func (r tester) getExpectedDatasets(pkgManifest *packages.PackageManifest) ([]string, error) { diff --git a/internal/testrunner/runners/static/tester_test.go b/internal/testrunner/runners/static/tester_test.go new file mode 100644 index 0000000000..96ef990f9c --- /dev/null +++ b/internal/testrunner/runners/static/tester_test.go @@ -0,0 +1,66 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package static + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-package/internal/files" + "github.com/elastic/elastic-package/internal/testrunner" +) + +func TestGetSampleEventPaths(t *testing.T) { + repositoryRoot, err := files.FindRepositoryRoot() + require.NoError(t, err) + t.Cleanup(func() { _ = repositoryRoot.Close() }) + + otelPackageRoot := filepath.Join(repositoryRoot.Name(), "test", "packages", "parallel", "sql_server_input_otel") + + cases := []struct { + title string + packageRoot string + dataStream string + expectedNames []string + }{ + { + title: "OTel input package with type-qualified sample events", + packageRoot: otelPackageRoot, + expectedNames: []string{ + "sample_event.json", + "sample_event_logs.json", + "sample_event_metrics.json", + }, + }, + { + title: "Package root with no sample event files", + packageRoot: t.TempDir(), + expectedNames: nil, + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + r := tester{ + packageRoot: c.packageRoot, + testFolder: testrunner.TestFolder{ + DataStream: c.dataStream, + }, + } + + paths, err := r.getSampleEventPaths() + require.NoError(t, err) + + var names []string + for _, p := range paths { + names = append(names, filepath.Base(p)) + } + assert.Equal(t, c.expectedNames, names) + }) + } +} diff --git a/internal/testrunner/runners/system/test_config.go b/internal/testrunner/runners/system/test_config.go index 3b919acc06..2d07f1643a 100644 --- a/internal/testrunner/runners/system/test_config.go +++ b/internal/testrunner/runners/system/test_config.go @@ -33,13 +33,14 @@ var ( type testConfig struct { testrunner.SkippableConfig `config:",inline"` - Input string `config:"input"` - PolicyTemplate string `config:"policy_template"` // Policy template associated with input. Required when multiple policy templates include the input being tested. - Service string `config:"service"` - ServiceNotifySignal string `config:"service_notify_signal"` // Signal to send when the agent policy is applied. - IgnoreServiceError bool `config:"ignore_service_error"` - WaitForDataTimeout time.Duration `config:"wait_for_data_timeout"` - SkipIgnoredFields []string `config:"skip_ignored_fields"` + Input string `config:"input"` + PolicyTemplate string `config:"policy_template"` // Policy template associated with input. Required when multiple policy templates include the input being tested. + Service string `config:"service"` + ServiceNotifySignal string `config:"service_notify_signal"` // Signal to send when the agent policy is applied. + IgnoreServiceError bool `config:"ignore_service_error"` + WaitForDataTimeout time.Duration `config:"wait_for_data_timeout"` + WaitForDynamicStreamsStable time.Duration `config:"wait_for_dynamic_streams_stable"` // The minimum time the count of discovered data streams must stay unchanged before discovery finishes. Default is 10s. + SkipIgnoredFields []string `config:"skip_ignored_fields"` Deployer string `config:"deployer"` // Name of the service deployer to use for this test. @@ -79,6 +80,10 @@ type testConfig struct { Path string `config:",ignore"` // Path of config file. ServiceVariantName string `config:",ignore"` // Name of test variant when using variants.yml. + // SignalTypes restricts dynamic_signal_types discovery to the listed signal type prefixes + // (e.g. ["logs", "metrics"]). When empty, all discovered streams are used. + SignalTypes []string `config:"signal_types"` + // Agent related properties Agent struct { agentdeployer.AgentSettings `config:",inline"` diff --git a/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-build-datastream-scenarios-explicit-signal-types.yaml b/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-build-datastream-scenarios-explicit-signal-types.yaml new file mode 100644 index 0000000000..a2fd48baa2 --- /dev/null +++ b/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-build-datastream-scenarios-explicit-signal-types.yaml @@ -0,0 +1,274 @@ +--- +version: 2 +interactions: + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/ + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 548 + uncompressed: false + body: | + { + "name" : "7b2ebb59e3c9", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "w312eKWCTb6Hb01ROdimBw", + "version" : { + "number" : "8.15.0-SNAPSHOT", + "build_flavor" : "default", + "build_type" : "docker", + "build_hash" : "822b187af48f9a5560ad365743998315038dad85", + "build_date" : "2024-07-03T13:25:55.204194663Z", + "build_snapshot" : true, + "lucene_version" : "9.11.1", + "minimum_wire_compatibility_version" : "7.17.0", + "minimum_index_compatibility_version" : "7.0.0" + }, + "tagline" : "You Know, for Search" + } + headers: + Content-Length: + - "548" + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 1ms + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/_data_stream/logs-*-default,metrics-*-default + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "data_streams": [ + { + "name": "logs-myreceiver.otel-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-logs-myreceiver.otel-default-000001", "index_uuid": "abc123"}], + "generation": 1, + "status": "GREEN", + "template": "logs-myreceiver", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "logs", + "lifecycle": {} + }, + { + "name": "metrics-myreceiver.otel-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-metrics-myreceiver.otel-default-000001", "index_uuid": "def456"}], + "generation": 1, + "status": "GREEN", + "template": "metrics-myreceiver", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "metrics", + "lifecycle": {} + } + ] + } + headers: + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 1ms + - id: 2 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/_data_stream/logs-*-default,metrics-*-default + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "data_streams": [ + { + "name": "logs-myreceiver.otel-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-logs-myreceiver.otel-default-000001", "index_uuid": "abc123"}], + "generation": 1, + "status": "GREEN", + "template": "logs-myreceiver", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "logs", + "lifecycle": {} + }, + { + "name": "metrics-myreceiver.otel-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-metrics-myreceiver.otel-default-000001", "index_uuid": "def456"}], + "generation": 1, + "status": "GREEN", + "template": "metrics-myreceiver", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "metrics", + "lifecycle": {} + } + ] + } + headers: + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 1ms + - id: 3 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/_data_stream/logs-*-default,metrics-*-default + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "data_streams": [ + { + "name": "logs-myreceiver.otel-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-logs-myreceiver.otel-default-000001", "index_uuid": "abc123"}], + "generation": 1, + "status": "GREEN", + "template": "logs-myreceiver", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "logs", + "lifecycle": {} + }, + { + "name": "metrics-myreceiver.otel-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-metrics-myreceiver.otel-default-000001", "index_uuid": "def456"}], + "generation": 1, + "status": "GREEN", + "template": "metrics-myreceiver", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "metrics", + "lifecycle": {} + } + ] + } + headers: + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 1ms diff --git a/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-discover-datastreams-found.yaml b/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-discover-datastreams-found.yaml new file mode 100644 index 0000000000..84d6e41efa --- /dev/null +++ b/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-discover-datastreams-found.yaml @@ -0,0 +1,132 @@ +--- +version: 2 +interactions: + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/ + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 548 + uncompressed: false + body: | + { + "name" : "7b2ebb59e3c9", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "w312eKWCTb6Hb01ROdimBw", + "version" : { + "number" : "8.15.0-SNAPSHOT", + "build_flavor" : "default", + "build_type" : "docker", + "build_hash" : "822b187af48f9a5560ad365743998315038dad85", + "build_date" : "2024-07-03T13:25:55.204194663Z", + "build_snapshot" : true, + "lucene_version" : "9.11.1", + "minimum_wire_compatibility_version" : "7.17.0", + "minimum_index_compatibility_version" : "7.0.0" + }, + "tagline" : "You Know, for Search" + } + headers: + Content-Length: + - "548" + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 12.812117ms + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/_data_stream/*-foo.bar-default + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "data_streams": [ + { + "name": "logs-foo.bar-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-logs-foo.bar-default-000001", "index_uuid": "abc123"}], + "generation": 1, + "status": "GREEN", + "template": "logs-foo.bar", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "logs", + "lifecycle": {} + }, + { + "name": "metrics-foo.bar-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-metrics-foo.bar-default-000001", "index_uuid": "def456"}], + "generation": 1, + "status": "GREEN", + "template": "metrics-foo.bar", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "metrics", + "lifecycle": {} + } + ] + } + headers: + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 5.123ms diff --git a/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-discover-datastreams-notfound.yaml b/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-discover-datastreams-notfound.yaml new file mode 100644 index 0000000000..8e76c13304 --- /dev/null +++ b/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-discover-datastreams-notfound.yaml @@ -0,0 +1,121 @@ +--- +version: 2 +interactions: + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/ + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 548 + uncompressed: false + body: | + { + "name" : "7b2ebb59e3c9", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "w312eKWCTb6Hb01ROdimBw", + "version" : { + "number" : "8.15.0-SNAPSHOT", + "build_flavor" : "default", + "build_type" : "docker", + "build_hash" : "822b187af48f9a5560ad365743998315038dad85", + "build_date" : "2024-07-03T13:25:55.204194663Z", + "build_snapshot" : true, + "lucene_version" : "9.11.1", + "minimum_wire_compatibility_version" : "7.17.0", + "minimum_index_compatibility_version" : "7.0.0" + }, + "tagline" : "You Know, for Search" + } + headers: + Content-Length: + - "548" + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 12.812117ms + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/_data_stream/*-foo.bar-default + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "error": { + "root_cause": [ + { + "type": "index_not_found_exception", + "reason": "no such index [*-foo.bar-default]", + "resource.type": "index_or_alias", + "resource.id": "*-foo.bar-default", + "index_uuid": "_na_", + "index": "*-foo.bar-default" + } + ], + "type": "index_not_found_exception", + "reason": "no such index [*-foo.bar-default]", + "resource.type": "index_or_alias", + "resource.id": "*-foo.bar-default", + "index_uuid": "_na_", + "index": "*-foo.bar-default" + }, + "status": 404 + } + headers: + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 404 Not Found + code: 404 + duration: 3.456ms diff --git a/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-wait-for-all-datastreams-timeout.yaml b/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-wait-for-all-datastreams-timeout.yaml new file mode 100644 index 0000000000..71a33e8279 --- /dev/null +++ b/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-wait-for-all-datastreams-timeout.yaml @@ -0,0 +1,101 @@ +--- +version: 2 +interactions: + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/ + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 548 + uncompressed: false + body: | + { + "name" : "7b2ebb59e3c9", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "w312eKWCTb6Hb01ROdimBw", + "version" : { + "number" : "8.15.0-SNAPSHOT", + "build_flavor" : "default", + "build_type" : "docker", + "build_hash" : "822b187af48f9a5560ad365743998315038dad85", + "build_date" : "2024-07-03T13:25:55.204194663Z", + "build_snapshot" : true, + "lucene_version" : "9.11.1", + "minimum_wire_compatibility_version" : "7.17.0", + "minimum_index_compatibility_version" : "7.0.0" + }, + "tagline" : "You Know, for Search" + } + headers: + Content-Length: + - "548" + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 1ms + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/_data_stream/*-myreceiver.otel-default + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 27 + uncompressed: false + body: | + {"error": "index_not_found_exception", "status": 404} + headers: + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 404 Not Found + code: 404 + duration: 1ms diff --git a/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-wait-for-all-datastreams.yaml b/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-wait-for-all-datastreams.yaml new file mode 100644 index 0000000000..1a038f982e --- /dev/null +++ b/internal/testrunner/runners/system/testdata/elasticsearch-8-mock-wait-for-all-datastreams.yaml @@ -0,0 +1,371 @@ +--- +version: 2 +interactions: + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/ + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 548 + uncompressed: false + body: | + { + "name" : "7b2ebb59e3c9", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "w312eKWCTb6Hb01ROdimBw", + "version" : { + "number" : "8.15.0-SNAPSHOT", + "build_flavor" : "default", + "build_type" : "docker", + "build_hash" : "822b187af48f9a5560ad365743998315038dad85", + "build_date" : "2024-07-03T13:25:55.204194663Z", + "build_snapshot" : true, + "lucene_version" : "9.11.1", + "minimum_wire_compatibility_version" : "7.17.0", + "minimum_index_compatibility_version" : "7.0.0" + }, + "tagline" : "You Know, for Search" + } + headers: + Content-Length: + - "548" + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 1ms + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/_data_stream/*-myreceiver.otel-default + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + {"data_streams": []} + headers: + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 1ms + - id: 2 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/_data_stream/*-myreceiver.otel-default + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "data_streams": [ + { + "name": "logs-myreceiver.otel-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-logs-myreceiver.otel-default-000001", "index_uuid": "abc123"}], + "generation": 1, + "status": "GREEN", + "template": "logs-myreceiver.otel", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "logs", + "lifecycle": {} + } + ] + } + headers: + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 1ms + - id: 3 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/_data_stream/*-myreceiver.otel-default + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "data_streams": [ + { + "name": "logs-myreceiver.otel-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-logs-myreceiver.otel-default-000001", "index_uuid": "abc123"}], + "generation": 1, + "status": "GREEN", + "template": "logs-myreceiver.otel", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "logs", + "lifecycle": {} + }, + { + "name": "metrics-myreceiver.otel-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-metrics-myreceiver.otel-default-000001", "index_uuid": "def456"}], + "generation": 1, + "status": "GREEN", + "template": "metrics-myreceiver.otel", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "metrics", + "lifecycle": {} + } + ] + } + headers: + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 1ms + - id: 4 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/_data_stream/*-myreceiver.otel-default + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "data_streams": [ + { + "name": "logs-myreceiver.otel-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-logs-myreceiver.otel-default-000001", "index_uuid": "abc123"}], + "generation": 1, + "status": "GREEN", + "template": "logs-myreceiver.otel", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "logs", + "lifecycle": {} + }, + { + "name": "metrics-myreceiver.otel-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-metrics-myreceiver.otel-default-000001", "index_uuid": "def456"}], + "generation": 1, + "status": "GREEN", + "template": "metrics-myreceiver.otel", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "metrics", + "lifecycle": {} + } + ] + } + headers: + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 1ms + - id: 5 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1 + url: https://127.0.0.1:9200/_data_stream/*-myreceiver.otel-default + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "data_streams": [ + { + "name": "logs-myreceiver.otel-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-logs-myreceiver.otel-default-000001", "index_uuid": "abc123"}], + "generation": 1, + "status": "GREEN", + "template": "logs-myreceiver.otel", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "logs", + "lifecycle": {} + }, + { + "name": "metrics-myreceiver.otel-default", + "timestamp_field": {"name": "@timestamp"}, + "indices": [{"index_name": ".ds-metrics-myreceiver.otel-default-000001", "index_uuid": "def456"}], + "generation": 1, + "status": "GREEN", + "template": "metrics-myreceiver.otel", + "hidden": false, + "system": false, + "prefer_ilm": true, + "next_generation_managed_by": "Index Lifecycle Management", + "ilm_policy": "metrics", + "lifecycle": {} + } + ] + } + headers: + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 1ms diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index 2916cb9fe6..17ccf7db87 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -127,7 +127,9 @@ const ( // are stored on the Agent container's filesystem. ServiceLogsAgentDir = "/tmp/service_logs" - waitForDataDefaultTimeout = 10 * time.Minute + waitForDataDefaultTimeout = 10 * time.Minute + waitForDynamicStreamsStableDuration = 10 * time.Second + dataStreamDiscoveryPollInterval = 1 * time.Second otelCollectorInputName = "otelcol" otelSuffixDataset = "otel" @@ -997,20 +999,26 @@ func ignoredDeprecationWarning(stackVersion *semver.Version, warning deprecation return false } -type scenarioTest struct { - // dataStream is the name of the target data stream where documents are indexed +// scenarioDataStream holds per-stream data for one data stream within a test scenario. +// A scenario always has at least one entry; dynamic_signal_types scenarios may have more. +type scenarioDataStream struct { dataStream string indexTemplateName string - policyTemplate packages.PolicyTemplate - kibanaPolicy kibana.PackagePolicy - dataStreamDataset string - syntheticEnabled bool docs []common.MapStr - deprecationWarnings []deprecationWarning ignoredFields []string degradedDocs []common.MapStr - agent agentdeployer.DeployedAgent - startTestTime time.Time + syntheticEnabled bool + deprecationWarnings []deprecationWarning +} + +type scenarioTest struct { + policyTemplate packages.PolicyTemplate + kibanaPolicy kibana.PackagePolicy + dataStreamDataset string + agent agentdeployer.DeployedAgent + startTestTime time.Time + // dataStreams holds one entry for standard scenarios and one-or-more for dynamic_signal_types. + dataStreams []scenarioDataStream } func (r *tester) deleteDataStream(ctx context.Context, dataStream string) error { @@ -1031,6 +1039,164 @@ func (r *tester) deleteDataStream(ctx context.Context, dataStream string) error return nil } +// discoveredDataStream contains the information retrieved from ES for a single +// data stream during dynamic_signal_types discovery. +type discoveredDataStream struct { + name string + indexTemplate string +} + +// dataStreamDataType returns the data type prefix of a data stream name — +// the segment before the first "-" (e.g. "logs" from "logs-sqlserverreceiver.otel-default"). +func dataStreamDataType(name string) string { + dataType, _, _ := strings.Cut(name, "-") + return dataType +} + +// searchDataStreams queries ES for all data streams matching the given wildcard patterns. +// Multiple patterns are joined with "," to use ES native multi-pattern syntax. +// The caller is responsible for constructing the patterns, e.g. "*-{dataset}-{namespace}". +func (r *tester) searchDataStreams(ctx context.Context, patterns []string) ([]discoveredDataStream, error) { + pattern := strings.Join(patterns, ",") + resp, err := r.esAPI.Indices.GetDataStream( + r.esAPI.Indices.GetDataStream.WithContext(ctx), + r.esAPI.Indices.GetDataStream.WithName(pattern), + ) + if err != nil { + return nil, fmt.Errorf("data stream discovery request failed (pattern: %s): %w", pattern, err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + if resp.IsError() { + return nil, fmt.Errorf("data stream discovery request failed (pattern: %s): %s", pattern, resp.String()) + } + + var body struct { + DataStreams []struct { + Name string `json:"name"` + Template string `json:"template"` + } `json:"data_streams"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + return nil, fmt.Errorf("failed to decode data stream discovery response: %w", err) + } + + result := make([]discoveredDataStream, 0, len(body.DataStreams)) + for _, ds := range body.DataStreams { + result = append(result, discoveredDataStream{ + name: ds.Name, + indexTemplate: ds.Template, + }) + } + return result, nil +} + +// discoverDataStreams polls Elasticsearch for data streams matching patterns until there is at least +// one match and the stream count has stayed the same for waitForDynamicStreamsStable. +func (r *tester) discoverDataStreams(ctx context.Context, config *testConfig, patterns []string) ([]discoveredDataStream, error) { + waitForDataTimeout := waitForDataDefaultTimeout + if config.WaitForDataTimeout > 0 { + waitForDataTimeout = config.WaitForDataTimeout + } + + waitForStableDuration := waitForDynamicStreamsStableDuration + if config.WaitForDynamicStreamsStable > 0 { + waitForStableDuration = config.WaitForDynamicStreamsStable + } + period := dataStreamDiscoveryPollInterval + initialPollCount := wait.PollBudget(waitForStableDuration, period) + + var currentStreams []discoveredDataStream + pollCount := initialPollCount + + streamNames := func() []string { + names := make([]string, len(currentStreams)) + for i, s := range currentStreams { + names[i] = s.name + } + return names + } + + patternDisplay := strings.Join(patterns, ",") + + logger.Debugf("Waiting for data streams matching %s (timeout: %s, or results are stable for: %s)...", + patternDisplay, waitForDataTimeout, waitForStableDuration) + + passed, err := wait.UntilTrue(ctx, func(ctx context.Context) (bool, error) { + streams, err := r.searchDataStreams(ctx, patterns) + if err != nil { + return false, err + } + if len(streams) == 0 { + return false, nil + } + if len(streams) != len(currentStreams) { + pollCount = initialPollCount + } else { + pollCount-- + } + currentStreams = streams + return pollCount == 0, nil + }, period, waitForDataTimeout) + + if err != nil { + return nil, err + } + if !passed && len(currentStreams) == 0 { + return nil, testrunner.ErrTestCaseFailed{Reason: fmt.Sprintf("no data streams matching %s appeared within %s", patternDisplay, waitForDataTimeout)} + } + if passed { + logger.Debugf("Discovery finished; found %d data stream(s) matching %s: %s", len(currentStreams), patternDisplay, strings.Join(streamNames(), ", ")) + } + return currentStreams, nil +} + +// verifyDataStream waits for documents to arrive in a single data stream, checks the service +// exit code, fetches deprecation warnings, and checks synthetic source mode. All results are +// written directly into the sds struct. +func (r *tester) verifyDataStream(ctx context.Context, config *testConfig, service servicedeployer.DeployedService, sds *scenarioDataStream) error { + hits, waitErr := r.waitForDocs(ctx, config, sds.dataStream) + + // before checking "waitErr" error, it is necessary to check if the service has finished + // with an error, to report it as a test case failed + if service != nil && config.Service != "" && !config.IgnoreServiceError { + exited, code, err := service.ExitCode(ctx, config.Service) + if err != nil && !errors.Is(err, servicedeployer.ErrNotSupported) { + return err + } + if exited && code > 0 { + return testrunner.ErrTestCaseFailed{Reason: fmt.Sprintf("the test service %s unexpectedly exited with code %d", config.Service, code)} + } + } + + if waitErr != nil { + return waitErr + } + + warnings, err := r.getDeprecationWarnings(ctx, sds.dataStream) + if err != nil { + return fmt.Errorf("failed to get deprecation warnings for data stream %s: %w", sds.dataStream, err) + } + logger.Debugf("Found %d deprecation warnings for data stream %s", len(warnings), sds.dataStream) + sds.deprecationWarnings = append(sds.deprecationWarnings, warnings...) + + logger.Debugf("Check whether or not synthetic source mode is enabled (data stream %s)...", sds.dataStream) + syntheticEnabled, err := isSyntheticSourceModeEnabled(ctx, r.esAPI, sds.dataStream) + if err != nil { + return fmt.Errorf("failed to check if synthetic source mode is enabled for data stream %s: %w", sds.dataStream, err) + } + logger.Debugf("Data stream %s has synthetic source mode enabled: %t", sds.dataStream, syntheticEnabled) + + sds.syntheticEnabled = syntheticEnabled + sds.docs = hits.getDocs(syntheticEnabled) + sds.ignoredFields = hits.IgnoredFields + sds.degradedDocs = hits.DegradedDocs + return nil +} + func (r *tester) prepareScenario(ctx context.Context, config *testConfig, stackConfig stack.Config, svcInfo servicedeployer.ServiceInfo) (*scenarioTest, error) { serviceOptions := r.createServiceOptions(config.ServiceVariantName, config.Deployer) @@ -1122,17 +1288,15 @@ func (r *tester) prepareScenario(ctx context.Context, config *testConfig, stackC return nil, fmt.Errorf("could not add data stream config to policy: %w", err) } } + scenario.kibanaPolicy = policy scenario.dataStreamDataset = dsDataset - scenario.indexTemplateName = buildIndexTemplateName(dsType, dsDataset) - scenario.dataStream = BuildDataStreamName(dsType, dsDataset, policy.Namespace, policyTemplate, r.pkgManifest.Type) - r.cleanTestScenarioHandler = func(ctx context.Context) error { - logger.Debugf("Deleting data stream for testing %s", scenario.dataStream) - err := r.deleteDataStream(ctx, scenario.dataStream) - if err != nil { - return fmt.Errorf("failed to delete data stream %s: %w", scenario.dataStream, err) + logger.Debugf("Deleting data streams for test %s in namespace %s", r.configFileName, policy.Namespace) + pattern := fmt.Sprintf("*-*-%s", policy.Namespace) + if err := r.deleteDataStream(ctx, pattern); err != nil { + return fmt.Errorf("failed to delete data streams for test %s in namespace %s: %w", r.configFileName, policy.Namespace, err) } return nil } @@ -1237,42 +1401,22 @@ func (r *tester) prepareScenario(ctx context.Context, config *testConfig, stackC return &scenario, nil } - hits, waitErr := r.waitForDocs(ctx, config, scenario.dataStream) - - // before checking "waitErr" error , it is necessary to check if the service has finished with error - // to report it as a test case failed - if service != nil && config.Service != "" && !config.IgnoreServiceError { - exited, code, err := service.ExitCode(ctx, config.Service) - if err != nil && !errors.Is(err, servicedeployer.ErrNotSupported) { - return nil, err - } - if exited && code > 0 { - return nil, testrunner.ErrTestCaseFailed{Reason: fmt.Sprintf("the test service %s unexpectedly exited with code %d", config.Service, code)} - } - } - - if waitErr != nil { - return nil, waitErr - } - - // Get deprecation warnings after ensuring that there are ingested docs and thus the - // data stream exists. - scenario.deprecationWarnings, err = r.getDeprecationWarnings(ctx, scenario.dataStream) + scenario.dataStreams, err = r.buildDataStreamScenarios(ctx, dsType, dsDataset, policy.Namespace, policyTemplate, config) if err != nil { - return nil, fmt.Errorf("failed to get deprecation warnings for data stream %s: %w", scenario.dataStream, err) + return nil, err } - logger.Debugf("Found %d deprecation warnings for data stream %s", len(scenario.deprecationWarnings), scenario.dataStream) - logger.Debugf("Check whether or not synthetic source mode is enabled (data stream %s)...", scenario.dataStream) - scenario.syntheticEnabled, err = isSyntheticSourceModeEnabled(ctx, r.esAPI, scenario.dataStream) - if err != nil { - return nil, fmt.Errorf("failed to check if synthetic source mode is enabled for data stream %s: %w", scenario.dataStream, err) + dataStreamNames := make([]string, len(scenario.dataStreams)) + for i, sds := range scenario.dataStreams { + dataStreamNames[i] = sds.dataStream } - logger.Debugf("Data stream %s has synthetic source mode enabled: %t", scenario.dataStream, scenario.syntheticEnabled) + logger.Debugf("Testing %d data stream(s): %s", len(scenario.dataStreams), strings.Join(dataStreamNames, ", ")) - scenario.docs = hits.getDocs(scenario.syntheticEnabled) - scenario.ignoredFields = hits.IgnoredFields - scenario.degradedDocs = hits.DegradedDocs + for i := range scenario.dataStreams { + if err := r.verifyDataStream(ctx, config, service, &scenario.dataStreams[i]); err != nil { + return nil, err + } + } if r.runSetup { opts := scenarioStateOpts{ @@ -1293,14 +1437,50 @@ func (r *tester) prepareScenario(ctx context.Context, config *testConfig, stackC return &scenario, nil } +// buildDataStreamScenarios determines the set of data streams to test for a given scenario. +// When policyTemplate.DynamicSignalTypes is true or is an otelcol input with type "traces", data streams are discovered dynamically +// via ES polling. If SignalTypes is empty, a full wildcard pattern is used; otherwise one +// pattern per signal type is built and all are sent to ES in a single request. +// When DynamicSignalTypes is false, a single stream is built from dsType, dsDataset, and namespace. +func (r *tester) buildDataStreamScenarios(ctx context.Context, dsType, dsDataset, namespace string, policyTemplate packages.PolicyTemplate, config *testConfig) ([]scenarioDataStream, error) { + canHaveMultipleDataStreams := policyTemplate.DynamicSignalTypes || (policyTemplate.Input == otelCollectorInputName && policyTemplate.Type == "traces") + + if canHaveMultipleDataStreams { + var patterns []string + if len(config.SignalTypes) == 0 { + patterns = []string{fmt.Sprintf("*-*-%s", namespace)} + } else { + for _, st := range config.SignalTypes { + patterns = append(patterns, fmt.Sprintf("%s-*-%s", st, namespace)) + } + } + discovered, err := r.discoverDataStreams(ctx, config, patterns) + if err != nil { + return nil, err + } + scenarios := make([]scenarioDataStream, len(discovered)) + for i, dsd := range discovered { + scenarios[i] = scenarioDataStream{ + dataStream: dsd.name, + indexTemplateName: dsd.indexTemplate, + } + } + return scenarios, nil + } + return []scenarioDataStream{{ + dataStream: BuildDataStreamName(dsType, dsDataset, namespace, policyTemplate, r.pkgManifest.Type), + indexTemplateName: buildIndexTemplateName(dsType, dsDataset), + }}, nil +} + // buildIndexTemplateName builds the expected index template name that is installed in Elasticsearch -// when the package data stream is added to the policy. func buildIndexTemplateName(dsType, dsDataset string) string { return fmt.Sprintf("%s-%s", dsType, dsDataset) } // BuildDataStreamName builds the expected data stream name that is installed in Elasticsearch // when the package data stream is added to the policy. + func BuildDataStreamName(dsType, dsDataset, namespace string, policyTemplate packages.PolicyTemplate, packageType string) string { dataset := dsDataset @@ -1651,88 +1831,91 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re if r.dataStream != "" { fieldsDir = filepath.Join(r.dataStream, "fields") } - fieldsValidator, err := fields.CreateValidator(repositoryRoot, r.packageRoot, fieldsDir, - fields.WithSchemaURLs(r.schemaURLs), - fields.WithSpecVersion(r.pkgManifest.SpecVersion), - fields.WithNumericKeywordFields(config.NumericKeywordFields), - fields.WithStringNumberFields(config.StringNumberFields), - fields.WithExpectedDatasets(expectedDatasets), - fields.WithEnabledImportAllECSSChema(true), - fields.WithDisableNormalization(scenario.syntheticEnabled), - // When using the OTel collector input, just a subset of validations are performed (e.g. check expected datasets) - fields.WithOTelValidation(r.isTestUsingOTelCollectorInput(scenario.policyTemplate.Input)), - ) + + stackVersion, err := semver.NewVersion(r.stackVersion.Number) if err != nil { - return result.WithErrorf("creating fields validator for data stream failed (path: %s): %w", fieldsDir, err) + return result.WithErrorf("failed to parse stack version: %w", err) } - if errs := validateFields(scenario.docs, fieldsValidator); len(errs) > 0 { - return result.WithError(testrunner.ErrTestCaseFailed{ - Reason: fmt.Sprintf("one or more errors found in documents stored in %s data stream", scenario.dataStream), - Details: errs.Error(), - }) + specVersion, err := semver.NewVersion(r.pkgManifest.SpecVersion) + if err != nil { + return result.WithErrorf("failed to parse format version %q: %w", r.pkgManifest.SpecVersion, err) } - if !r.isTestUsingOTelCollectorInput(scenario.policyTemplate.Input) && r.fieldValidationMethod == mappingsMethod { - logger.Debug("Performing validation based on mappings") - exceptionFields := listExceptionFields(scenario.docs, fieldsValidator) - - mappingsValidator, err := fields.CreateValidatorForMappings(r.esClient, - fields.WithMappingValidatorFallbackSchema(fieldsValidator.Schema), - fields.WithMappingValidatorIndexTemplate(scenario.indexTemplateName), - fields.WithMappingValidatorDataStream(scenario.dataStream), - fields.WithMappingValidatorExceptionFields(exceptionFields), + for _, sds := range scenario.dataStreams { + logger.Debugf("Validating data stream %s (index template %s)", sds.dataStream, sds.indexTemplateName) + fieldsValidator, err := fields.CreateValidator(repositoryRoot, r.packageRoot, fieldsDir, + fields.WithSchemaURLs(r.schemaURLs), + fields.WithSpecVersion(r.pkgManifest.SpecVersion), + fields.WithNumericKeywordFields(config.NumericKeywordFields), + fields.WithStringNumberFields(config.StringNumberFields), + fields.WithExpectedDatasets(expectedDatasets), + fields.WithEnabledImportAllECSSChema(true), + fields.WithDisableNormalization(sds.syntheticEnabled), + // When using the OTel collector input, just a subset of validations are performed (e.g. check expected datasets) + fields.WithOTelValidation(r.isTestUsingOTelCollectorInput(scenario.policyTemplate.Input)), ) if err != nil { - return result.WithErrorf("creating mappings validator for data stream failed (data stream: %s): %w", scenario.dataStream, err) + return result.WithErrorf("creating fields validator for data stream failed (path: %s): %w", fieldsDir, err) } - if errs := validateMappings(ctx, mappingsValidator); len(errs) > 0 { + if errs := validateFields(sds.docs, fieldsValidator); len(errs) > 0 { return result.WithError(testrunner.ErrTestCaseFailed{ - Reason: fmt.Sprintf("one or more errors found in mappings in %s index template", scenario.indexTemplateName), + Reason: fmt.Sprintf("one or more errors found in documents stored in %s data stream", sds.dataStream), Details: errs.Error(), }) } - } - stackVersion, err := semver.NewVersion(r.stackVersion.Number) - if err != nil { - return result.WithErrorf("failed to parse stack version: %w", err) - } + if !r.isTestUsingOTelCollectorInput(scenario.policyTemplate.Input) && r.fieldValidationMethod == mappingsMethod { + logger.Debug("Performing validation based on mappings") + exceptionFields := listExceptionFields(sds.docs, fieldsValidator) - err = validateIgnoredFields(stackVersion, scenario, config) - if err != nil { - return result.WithError(err) - } + mappingsValidator, err := fields.CreateValidatorForMappings(r.esClient, + fields.WithMappingValidatorFallbackSchema(fieldsValidator.Schema), + fields.WithMappingValidatorIndexTemplate(sds.indexTemplateName), + fields.WithMappingValidatorDataStream(sds.dataStream), + fields.WithMappingValidatorExceptionFields(exceptionFields), + ) + if err != nil { + return result.WithErrorf("creating mappings validator for data stream failed (data stream: %s): %w", sds.dataStream, err) + } - docs := scenario.docs - if scenario.syntheticEnabled { - docs, err = fieldsValidator.SanitizeSyntheticSourceDocs(scenario.docs) - if err != nil { - results, _ := result.WithErrorf("failed to sanitize synthetic source docs: %w", err) - return results, nil + if errs := validateMappings(ctx, mappingsValidator); len(errs) > 0 { + return result.WithError(testrunner.ErrTestCaseFailed{ + Reason: fmt.Sprintf("one or more errors found in mappings in %s index template", sds.indexTemplateName), + Details: errs.Error(), + }) + } } - } - specVersion, err := semver.NewVersion(r.pkgManifest.SpecVersion) - if err != nil { - return result.WithErrorf("failed to parse format version %q: %w", r.pkgManifest.SpecVersion, err) - } + if err := validateIgnoredFields(stackVersion, sds, config); err != nil { + return result.WithError(err) + } - // Write sample events file from first doc, if requested - if err := r.generateTestResultFile(docs, *specVersion); err != nil { - return result.WithError(err) - } + docs := sds.docs + if sds.syntheticEnabled { + docs, err = fieldsValidator.SanitizeSyntheticSourceDocs(sds.docs) + if err != nil { + results, _ := result.WithErrorf("failed to sanitize synthetic source docs: %w", err) + return results, nil + } + } - // Check Hit Count within docs, if 0 then it has not been specified - if assertionPass, message := assertHitCount(config.Assert.HitCount, docs); !assertionPass { - result.FailureMsg = message - } + // Write sample events file from first doc, if requested + if err := r.generateTestResultFile(docs, *specVersion, sds, len(scenario.dataStreams) > 1); err != nil { + return result.WithError(err) + } - // Check transforms if present - if err := r.checkTransforms(ctx, config, r.pkgManifest, scenario.dataStream, scenario.policyTemplate.Input, scenario.syntheticEnabled); err != nil { - results, _ := result.WithError(err) - return results, nil + // Check Hit Count within docs, if 0 then it has not been specified + if assertionPass, message := assertHitCount(config.Assert.HitCount, docs); !assertionPass { + result.FailureMsg = message + } + + // Check transforms if present + if err := r.checkTransforms(ctx, config, r.pkgManifest, sds.dataStream, scenario.policyTemplate.Input, sds.syntheticEnabled); err != nil { + results, _ := result.WithError(err) + return results, nil + } } if scenario.agent != nil { @@ -1745,7 +1928,11 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re } } - if results := r.checkDeprecationWarnings(stackVersion, scenario.deprecationWarnings, config.Name()); len(results) > 0 { + var allDeprecationWarnings []deprecationWarning + for _, sds := range scenario.dataStreams { + allDeprecationWarnings = append(allDeprecationWarnings, sds.deprecationWarnings...) + } + if results := r.checkDeprecationWarnings(stackVersion, allDeprecationWarnings, config.Name()); len(results) > 0 { return results, nil } @@ -1790,8 +1977,11 @@ func (r *tester) expectedDatasets(scenario *scenarioTest, config *testConfig) ([ // Input packages whose input is `otelcol` must add the `.otel` suffix // Example: httpcheck.metrics.otel expectedDataset += "." + otelSuffixDataset + // Traces can also emit to a shared logs data stream (e.g. logs-generic.otel-*). + expectedDatasets = []string{expectedDataset, "generic." + otelSuffixDataset} + } else { + expectedDatasets = []string{expectedDataset} } - expectedDatasets = []string{expectedDataset} } return expectedDatasets, nil @@ -1827,7 +2017,7 @@ func (r *tester) runTest(ctx context.Context, config *testConfig, stackConfig st } if dump, ok := os.LookupEnv(dumpScenarioDocsEnv); ok && dump != "" { - err := dumpScenarioDocs(scenario.docs) + err := dumpScenarioDocs(scenario.dataStreams) if err != nil { return nil, fmt.Errorf("failed to dump scenario docs: %w", err) } @@ -1849,7 +2039,12 @@ func (r *tester) isTestUsingOTelCollectorInput(policyTemplateInput string) bool return true } -func dumpScenarioDocs(docs any) error { +func dumpScenarioDocs(dataStreams []scenarioDataStream) error { + allDocs := make(map[string][]common.MapStr, len(dataStreams)) + for _, sds := range dataStreams { + allDocs[sds.dataStream] = sds.docs + } + timestamp := time.Now().Format("20060102150405") path := filepath.Join(os.TempDir(), fmt.Sprintf("elastic-package-test-docs-dump-%s.json", timestamp)) f, err := os.Create(path) @@ -1863,7 +2058,7 @@ func dumpScenarioDocs(docs any) error { enc := json.NewEncoder(f) enc.SetIndent("", " ") enc.SetEscapeHTML(false) - if err := enc.Encode(docs); err != nil { + if err := enc.Encode(allDocs); err != nil { return fmt.Errorf("failed to encode docs: %w", err) } return nil @@ -2169,14 +2364,14 @@ func filterIndependentAgents(allAgents []kibana.Agent, agentInfo agentdeployer.A return filtered } -func writeSampleEvent(path string, doc common.MapStr, specVersion semver.Version) error { +func writeSampleEvent(path string, doc common.MapStr, specVersion semver.Version, filename string) error { jsonFormatter := formatter.JSONFormatterBuilder(specVersion) body, err := jsonFormatter.Encode(doc) if err != nil { return fmt.Errorf("marshalling sample event failed: %w", err) } - err = os.WriteFile(filepath.Join(path, "sample_event.json"), append(body, '\n'), 0644) + err = os.WriteFile(filepath.Join(path, filename), append(body, '\n'), 0644) if err != nil { return fmt.Errorf("writing sample event failed: %w", err) } @@ -2269,16 +2464,16 @@ func listExceptionFields(docs []common.MapStr, fieldsValidator *fields.Validator return allFields } -func validateIgnoredFields(stackVersion *semver.Version, scenario *scenarioTest, config *testConfig) error { +func validateIgnoredFields(stackVersion *semver.Version, ds scenarioDataStream, config *testConfig) error { skipIgnoredFields := append([]string(nil), config.SkipIgnoredFields...) if stackVersion.LessThan(semver.MustParse("8.14.0")) { // Pre 8.14 Elasticsearch commonly has event.original not mapped correctly, exclude from check: https://github.com/elastic/elasticsearch/pull/106714 skipIgnoredFields = append(skipIgnoredFields, "event.original") } - ignoredFields := make([]string, 0, len(scenario.ignoredFields)) + ignoredFields := make([]string, 0, len(ds.ignoredFields)) - for _, field := range scenario.ignoredFields { + for _, field := range ds.ignoredFields { if !slices.Contains(skipIgnoredFields, field) { ignoredFields = append(ignoredFields, field) } @@ -2289,8 +2484,8 @@ func validateIgnoredFields(stackVersion *semver.Version, scenario *scenarioTest, ID any `json:"_id"` Timestamp any `json:"@timestamp,omitempty"` IgnoredFields any `json:"ignored_field_values"` - }, len(scenario.degradedDocs)) - for i, d := range scenario.degradedDocs { + }, len(ds.degradedDocs)) + for i, d := range ds.degradedDocs { issues[i].ID = d["_id"] if source, ok := d["_source"].(map[string]any); ok { if ts, ok := source["@timestamp"]; ok { @@ -2306,7 +2501,7 @@ func validateIgnoredFields(stackVersion *semver.Version, scenario *scenarioTest, return testrunner.ErrTestCaseFailed{ Reason: "found ignored fields in data stream", - Details: fmt.Sprintf("found ignored fields in data stream %s: %v. Affected documents: %s", scenario.dataStream, ignoredFields, degradedDocsJSON), + Details: fmt.Sprintf("found ignored fields in data stream %s: %v. Affected documents: %s", ds.dataStream, ignoredFields, degradedDocsJSON), } } @@ -2332,7 +2527,7 @@ func assertHitCount(expected int, docs []common.MapStr) (pass bool, message stri return true, "" } -func (r *tester) generateTestResultFile(docs []common.MapStr, specVersion semver.Version) error { +func (r *tester) generateTestResultFile(docs []common.MapStr, specVersion semver.Version, sds scenarioDataStream, qualifyByType bool) error { if !r.generateTestResult { return nil } @@ -2342,7 +2537,14 @@ func (r *tester) generateTestResultFile(docs []common.MapStr, specVersion semver rootPath = filepath.Join(rootPath, "data_stream", ds) } - if err := writeSampleEvent(rootPath, docs[0], specVersion); err != nil { + filename := "sample_event.json" + if qualifyByType { + // For dynamic_signal_types packages, qualify the filename by signal type + // (e.g. "logs-sqlserverreceiver.otel-default" → "sample_event_logs.json"). + filename = fmt.Sprintf("sample_event_%s.json", dataStreamDataType(sds.dataStream)) + } + + if err := writeSampleEvent(rootPath, docs[0], specVersion, filename); err != nil { return fmt.Errorf("failed to write sample event file: %w", err) } diff --git a/internal/testrunner/runners/system/tester_test.go b/internal/testrunner/runners/system/tester_test.go index d4d4f11889..cfd1720721 100644 --- a/internal/testrunner/runners/system/tester_test.go +++ b/internal/testrunner/runners/system/tester_test.go @@ -439,6 +439,101 @@ func TestIsSyntheticSourceModeEnabled(t *testing.T) { }) } } +func TestSearchDataStreams(t *testing.T) { + const pattern = "*-foo.bar-default" + + t.Run("happy path returns two streams", func(t *testing.T) { + client := estest.NewClient(t, "testdata/elasticsearch-8-mock-discover-datastreams-found", nil) + r := &tester{esAPI: client.API} + + streams, err := r.searchDataStreams(t.Context(), []string{pattern}) + require.NoError(t, err) + require.Len(t, streams, 2) + + assert.Equal(t, "logs-foo.bar-default", streams[0].name) + assert.Equal(t, "logs-foo.bar", streams[0].indexTemplate) + + assert.Equal(t, "metrics-foo.bar-default", streams[1].name) + assert.Equal(t, "metrics-foo.bar", streams[1].indexTemplate) + }) + + t.Run("404 returns empty slice with no error", func(t *testing.T) { + client := estest.NewClient(t, "testdata/elasticsearch-8-mock-discover-datastreams-notfound", nil) + r := &tester{esAPI: client.API} + + streams, err := r.searchDataStreams(t.Context(), []string{pattern}) + require.NoError(t, err) + assert.Empty(t, streams) + }) +} + +func TestDiscoverDataStreams(t *testing.T) { + const pattern = "*-myreceiver.otel-default" + + t.Run("returns error when no streams appear within timeout", func(t *testing.T) { + client := estest.NewClient(t, "testdata/elasticsearch-8-mock-wait-for-all-datastreams-timeout", nil) + r := &tester{esAPI: client.API} + cfg := &testConfig{ + WaitForDataTimeout: 100 * time.Millisecond, + WaitForDynamicStreamsStable: 2 * time.Second, + } + + _, err := r.discoverDataStreams(t.Context(), cfg, []string{pattern}) + require.Error(t, err) + assert.Contains(t, err.Error(), "no data streams matching") + }) + + t.Run("phase 2 picks up late-arriving stream", func(t *testing.T) { + client := estest.NewClient(t, "testdata/elasticsearch-8-mock-wait-for-all-datastreams", nil) + r := &tester{esAPI: client.API} + cfg := &testConfig{ + WaitForDynamicStreamsStable: 2 * time.Second, + } + + streams, err := r.discoverDataStreams(t.Context(), cfg, []string{pattern}) + require.NoError(t, err) + require.Len(t, streams, 2) + + names := make(map[string]string) + for _, s := range streams { + names[s.name] = s.indexTemplate + } + assert.Equal(t, "logs-myreceiver.otel", names["logs-myreceiver.otel-default"]) + assert.Equal(t, "metrics-myreceiver.otel", names["metrics-myreceiver.otel-default"]) + }) +} + +func TestBuildDataStreamScenarios(t *testing.T) { + t.Run("standard single stream", func(t *testing.T) { + r := &tester{pkgManifest: &packages.PackageManifest{Type: "integration"}} + pt := packages.PolicyTemplate{Name: "bar"} + cfg := &testConfig{} + + got, err := r.buildDataStreamScenarios(t.Context(), "logs", "foo.bar", "default", pt, cfg) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "logs-foo.bar-default", got[0].dataStream) + assert.Equal(t, "logs-foo.bar", got[0].indexTemplateName) + }) + + t.Run("explicit signal_types produce one entry per type", func(t *testing.T) { + client := estest.NewClient(t, "testdata/elasticsearch-8-mock-build-datastream-scenarios-explicit-signal-types", nil) + r := &tester{pkgManifest: &packages.PackageManifest{Type: "input"}, esAPI: client.API} + pt := packages.PolicyTemplate{Name: "myreceiver", Input: "otelcol", DynamicSignalTypes: true} + cfg := &testConfig{ + SignalTypes: []string{"logs", "metrics"}, + WaitForDynamicStreamsStable: 2 * time.Second, + } + + got, err := r.buildDataStreamScenarios(t.Context(), "logs", "myreceiver", "default", pt, cfg) + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, "logs-myreceiver.otel-default", got[0].dataStream) + assert.Equal(t, "logs-myreceiver", got[0].indexTemplateName) + assert.Equal(t, "metrics-myreceiver.otel-default", got[1].dataStream) + assert.Equal(t, "metrics-myreceiver", got[1].indexTemplateName) + }) +} func TestPipelineErrorMessage(t *testing.T) { testCases := []struct { diff --git a/internal/wait/wait.go b/internal/wait/wait.go index b826573911..8e9a6685e7 100644 --- a/internal/wait/wait.go +++ b/internal/wait/wait.go @@ -9,6 +9,21 @@ import ( "time" ) +// PollBudget returns how many polls spaced by period are needed to cover window, rounded up +// (ceil(window/period)). +// If window or period is zero or negative, PollBudget returns 1 so callers always get a positive +// budget without dividing by zero. Otherwise the result is at least 1. +func PollBudget(window, period time.Duration) int { + if period <= 0 || window <= 0 { + return 1 + } + n := int((window + period - 1) / period) + if n < 1 { + return 1 + } + return n +} + // UntilTrue waits till the context is cancelled or the given function returns an error or true. func UntilTrue(ctx context.Context, fn func(ctx context.Context) (bool, error), period, timeout time.Duration) (bool, error) { timeoutTimer := time.NewTimer(timeout) diff --git a/internal/wait/wait_test.go b/internal/wait/wait_test.go new file mode 100644 index 0000000000..5beb4e6de5 --- /dev/null +++ b/internal/wait/wait_test.go @@ -0,0 +1,35 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package wait + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestPollBudget(t *testing.T) { + cases := []struct { + name string + window time.Duration + period time.Duration + want int + }{ + {"zero window", 0, time.Second, 1}, + {"negative window", -time.Second, time.Second, 1}, + {"zero period", time.Second, 0, 1}, + {"negative period", time.Second, -time.Millisecond, 1}, + {"exact multiple", 2 * time.Second, time.Second, 2}, + {"ceil partial", 1500 * time.Millisecond, time.Second, 2}, + {"sub-period window still one poll", time.Millisecond, time.Second, 1}, + {"milliseconds", 500 * time.Millisecond, 100 * time.Millisecond, 5}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + assert.Equal(t, c.want, PollBudget(c.window, c.period)) + }) + } +} diff --git a/test/packages/parallel/sql_server_input_otel.stack_version b/test/packages/parallel/sql_server_input_otel.stack_version new file mode 100644 index 0000000000..ce6f507def --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel.stack_version @@ -0,0 +1 @@ +9.4.0-SNAPSHOT \ No newline at end of file diff --git a/test/packages/parallel/sql_server_input_otel/LICENSE.txt b/test/packages/parallel/sql_server_input_otel/LICENSE.txt new file mode 100644 index 0000000000..d317b57b29 --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/LICENSE.txt @@ -0,0 +1,93 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor's trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/test/packages/parallel/sql_server_input_otel/_dev/build/docs/README.md b/test/packages/parallel/sql_server_input_otel/_dev/build/docs/README.md new file mode 100644 index 0000000000..0eb70eddb4 --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/_dev/build/docs/README.md @@ -0,0 +1,56 @@ +# SQL Server OpenTelemetry Input Package + +## Overview + +The SQL Server OpenTelemetry Input Package for Elastic enables collection of telemetry data from Microsoft SQL Server instances through OpenTelemetry protocols using the [sqlserverreceiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/sqlserverreceiver). + +### How it works + +This package receives telemetry data from SQL Server instances by configuring the SQL Server receiver in the Input Package, which then gets applied to the sqlserverreceiver present in the EDOT collector, which then forwards the data to Elastic Agent. The Elastic Agent processes and enriches the data before sending it to Elasticsearch for indexing and analysis. + +## Requirements + +### Windows Performance Counters + +Make sure to run the collector as administrator to collect all performance counters for metrics. + +### Direct Connection + +When configured to directly connect to the SQL Server instance, the user must have the following permissions: + +1. At least one of the following permissions: + - `CREATE DATABASE` + - `ALTER ANY DATABASE` + - `VIEW ANY DATABASE` + +2. Permission to view server state: + - SQL Server pre-2022: `VIEW SERVER STATE` + - SQL Server 2022 and later: `VIEW SERVER PERFORMANCE STATE` + +## Configuration + +For the full list of settings exposed for the receiver and examples, refer to the [configuration](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/sqlserverreceiver#configuration) section. + +## Metrics reference + +For a complete list of all available metrics and their detailed descriptions, refer to the [SQL Server Receiver documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/sqlserverreceiver/documentation.md) in the upstream OpenTelemetry Collector repository. + +## Logs reference + +The SQL Server receiver can collect log events when a direct database connection is configured. Two event types are available, both turned off by default: + +- **Query Sample Events** (`db.server.query_sample`): Captures currently executing queries at scrape time, including session details, wait information, and resource consumption. Enable by setting **Enable Query Sample Events** to `true`. + +- **Top Query Events** (`db.server.top_query`): Captures the most expensive queries by execution time within a configurable lookback window, including execution counts, CPU time, and logical reads. Enable by setting **Enable Top Query Events** to `true`. + +Both event types require a direct database connection. Configure either the individual connection settings (server, port, username, and password) or the datasource connection string. The `query_sample_collection` and `top_query_collection` settings control the behavior of each event type. + +For a complete list of log attributes, refer to the [SQL Server Receiver logs documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/sqlserverreceiver/logs-documentation.md) in the upstream OpenTelemetry Collector repository. + +## Known limitations + +### Feature gate: `receiver.sqlserver.RemoveServerResourceAttribute` + +Starting with EDOT Collector versions based on OpenTelemetry Collector Contrib v0.129.0+, the upstream receiver includes a feature gate `receiver.sqlserver.RemoveServerResourceAttribute` that removes `server.address` and `server.port` from resource attributes, as they are not identified as resource attributes in the semantic conventions. This feature gate is currently opt-in. Refer to the [upstream documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/sqlserverreceiver#feature-gate) for details. + +{{ event }} diff --git a/test/packages/parallel/sql_server_input_otel/_dev/deploy/docker/Dockerfile b/test/packages/parallel/sql_server_input_otel/_dev/deploy/docker/Dockerfile new file mode 100644 index 0000000000..d27b87275b --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/_dev/deploy/docker/Dockerfile @@ -0,0 +1,12 @@ +ARG MSSQL_VERSION=${MSSQL_VERSION:-2019-latest} +FROM mcr.microsoft.com/mssql/server:${MSSQL_VERSION} + +ENV ACCEPT_EULA='Y' +ENV SA_PASSWORD='1234_asdf' + +COPY init.sh /init/init.sh +COPY workload.sh /init/workload.sh + +HEALTHCHECK --interval=10s --timeout=3s --start-period=30s --retries=5 CMD test -f /tmp/init_done && (/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P ${SA_PASSWORD} -No -Q "SELECT 1" 2>/dev/null || /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P ${SA_PASSWORD} -Q "SELECT 1" 2>/dev/null) || exit 1 + +CMD /bin/bash -c "/opt/mssql/bin/sqlservr & /init/init.sh && sleep infinity" diff --git a/test/packages/parallel/sql_server_input_otel/_dev/deploy/docker/docker-compose.yml b/test/packages/parallel/sql_server_input_otel/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 0000000000..ccdef8273d --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,27 @@ +version: '2.3' +services: + sql_server_input_otel: + image: mcr.microsoft.com/mssql/server:${MSSQL_VERSION:-2019-latest} + user: root + build: + context: . + args: + MSSQL_VERSION: ${MSSQL_VERSION:-2019-latest} + ports: + - 1433 + sql_server_input_otel_workload: + image: mcr.microsoft.com/mssql/server:${MSSQL_VERSION:-2019-latest} + build: + context: . + args: + MSSQL_VERSION: ${MSSQL_VERSION:-2019-latest} + depends_on: + sql_server_input_otel: + condition: service_healthy + healthcheck: + test: ["CMD", "test", "-f", "/tmp/workload_ready"] + interval: 10s + timeout: 3s + start_period: 60s + retries: 5 + entrypoint: ["/bin/bash", "/init/workload.sh"] \ No newline at end of file diff --git a/test/packages/parallel/sql_server_input_otel/_dev/deploy/docker/init.sh b/test/packages/parallel/sql_server_input_otel/_dev/deploy/docker/init.sh new file mode 100755 index 0000000000..461d39c4db --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/_dev/deploy/docker/init.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +TIMEOUT=60 + +if [ -x /opt/mssql-tools18/bin/sqlcmd ]; then + SQLCMD="/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U SA -P $SA_PASSWORD -No" +else + SQLCMD="/opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P $SA_PASSWORD" +fi + +for ((i=0; i/dev/null 2>&1 + if [ $? -eq 0 ]; then + break + fi + sleep 1 +done + +set -euo pipefail + +$SQLCMD -Q "IF DB_ID('testdb') IS NULL CREATE DATABASE testdb;" + +$SQLCMD -d testdb -Q " +IF OBJECT_ID('dbo.test_table', 'U') IS NULL +BEGIN + CREATE TABLE dbo.test_table ( + id INT PRIMARY KEY IDENTITY(1,1), + name NVARCHAR(100) + ); +END; +INSERT INTO dbo.test_table (name) VALUES (N'Hello World'), (N'Test Entry'); +" + +touch /tmp/init_done +echo "Initialization complete." diff --git a/test/packages/parallel/sql_server_input_otel/_dev/deploy/docker/workload.sh b/test/packages/parallel/sql_server_input_otel/_dev/deploy/docker/workload.sh new file mode 100755 index 0000000000..7e54fb7535 --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/_dev/deploy/docker/workload.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +SERVER="sql_server_input_otel" +PASSWORD="1234_asdf" +DATABASE="testdb" + +if [ -x /opt/mssql-tools18/bin/sqlcmd ]; then + SQLCMD="/opt/mssql-tools18/bin/sqlcmd -C -S $SERVER -U SA -P $PASSWORD -No -d $DATABASE" +else + SQLCMD="/opt/mssql-tools/bin/sqlcmd -S $SERVER -U SA -P $PASSWORD -d $DATABASE" +fi + +echo "Waiting for $SERVER to be ready..." +for i in $(seq 1 60); do + $SQLCMD -Q "SELECT 1" >/dev/null 2>&1 && break + sleep 1 +done + +echo "Starting workload generation..." +touch /tmp/workload_ready + +# Background: long-running query for query_sample capture +while true; do + $SQLCMD -Q "WAITFOR DELAY '00:00:30'; SELECT * FROM dbo.test_table" >/dev/null 2>&1 +done & + +# Foreground: repeated short queries for top_query capture +while true; do + $SQLCMD -Q "SELECT * FROM dbo.test_table" >/dev/null 2>&1 + sleep 1 +done diff --git a/test/packages/parallel/sql_server_input_otel/_dev/deploy/variants.yml b/test/packages/parallel/sql_server_input_otel/_dev/deploy/variants.yml new file mode 100644 index 0000000000..21a7a1d9e1 --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/_dev/deploy/variants.yml @@ -0,0 +1,4 @@ +variants: + MSSQL_2019: + MSSQL_VERSION: 2019-latest +default: MSSQL_2019 \ No newline at end of file diff --git a/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-all-signals-config.yml b/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-all-signals-config.yml new file mode 100644 index 0000000000..29b58b6235 --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-all-signals-config.yml @@ -0,0 +1,17 @@ +service: sql_server_input_otel +signal_types: + - logs + - metrics +vars: + server: "{{Hostname}}" + port: "{{Port}}" + username: "SA" + password: "1234_asdf" + collection_interval: 10s + initial_delay: 1s + enable_query_sample_events: true + enable_top_query_events: true + top_query_lookback_time: 60s + top_query_collection_interval: 10s +assert: + min_count: 10 diff --git a/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-default-config.yml b/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..05a0e22b2a --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-default-config.yml @@ -0,0 +1,16 @@ +service: sql_server_input_otel +vars: + server: "{{Hostname}}" + port: "{{Port}}" + username: "SA" + password: "1234_asdf" + collection_interval: 10s + initial_delay: 1s +assert: + min_count: 10 + fields_present: + - metrics.sqlserver.batch.request.rate + - metrics.sqlserver.batch.sql_compilation.rate + - metrics.sqlserver.batch.sql_recompilation.rate + - metrics.sqlserver.page.buffer_cache.hit_ratio + - metrics.sqlserver.user.connection.count \ No newline at end of file diff --git a/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-logs-config.yml b/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-logs-config.yml new file mode 100644 index 0000000000..8e4287833b --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/_dev/test/system/test-logs-config.yml @@ -0,0 +1,59 @@ +service: sql_server_input_otel +signal_types: + - logs +vars: + server: "{{Hostname}}" + port: "{{Port}}" + username: "SA" + password: "1234_asdf" + collection_interval: 10s + initial_delay: 1s + enable_query_sample_events: true + enable_top_query_events: true + top_query_lookback_time: 60s + top_query_collection_interval: 10s +assert: + min_count: 10 + fields_present: + # Common to both event types + - db.query.text + - sqlserver.query_hash + - sqlserver.query_plan_hash + - sqlserver.total_elapsed_time + # Top-Query Collection (db.server.top_query) + - sqlserver.total_rows + - sqlserver.total_worker_time + - sqlserver.total_logical_reads + - sqlserver.total_logical_writes + - sqlserver.total_physical_reads + - sqlserver.execution_count + - sqlserver.total_grant_kb + # Query-Sample Collection (db.server.query_sample) + - db.system.name + - db.namespace + - network.peer.address + - network.peer.port + - client.port + - client.address + - sqlserver.query_start + - sqlserver.session_id + - sqlserver.session_status + - sqlserver.request_status + - sqlserver.command + - sqlserver.blocking_session_id + - sqlserver.wait_type + - sqlserver.wait_time + - sqlserver.wait_resource + - sqlserver.open_transaction_count + - sqlserver.transaction_id + - sqlserver.percent_complete + - sqlserver.estimated_completion_time + - sqlserver.cpu_time + - sqlserver.reads + - sqlserver.writes + - sqlserver.logical_reads + - sqlserver.transaction_isolation_level + - sqlserver.lock_timeout + - sqlserver.deadlock_priority + - sqlserver.row_count + - sqlserver.context_info diff --git a/test/packages/parallel/sql_server_input_otel/agent/input/input.yml.hbs b/test/packages/parallel/sql_server_input_otel/agent/input/input.yml.hbs new file mode 100644 index 0000000000..da5ba40655 --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/agent/input/input.yml.hbs @@ -0,0 +1,58 @@ +receivers: + sqlserver: +{{#if server}} + server: {{server}} +{{/if}} +{{#if port}} + port: {{port}} +{{/if}} +{{#if username}} + username: {{username}} +{{/if}} +{{#if password}} + password: {{password}} +{{/if}} +{{#if datasource}} + datasource: {{datasource}} +{{/if}} +{{#if instance_name}} + instance_name: {{instance_name}} +{{/if}} +{{#if computer_name}} + computer_name: {{computer_name}} +{{/if}} + collection_interval: {{collection_interval}} + initial_delay: {{initial_delay}} + events: + db.server.query_sample: + enabled: {{enable_query_sample_events}} + db.server.top_query: + enabled: {{enable_top_query_events}} +{{#if enable_query_sample_events}} + query_sample_collection: + max_rows_per_query: {{query_sample_max_rows_per_query}} +{{/if}} +{{#if enable_top_query_events}} + top_query_collection: + lookback_time: {{top_query_lookback_time}} + max_query_sample_count: {{top_query_max_query_sample_count}} + top_query_count: {{top_query_count}} + collection_interval: {{top_query_collection_interval}} +{{/if}} +{{#if resource_attributes}} + resource_attributes: {{resource_attributes}} +{{/if}} +{{#if metrics}} + metrics: {{metrics}} +{{/if}} +processors: + resourcedetection/system: + detectors: ["system"] +service: + pipelines: + logs: + receivers: [sqlserver] + processors: [resourcedetection/system] + metrics: + receivers: [sqlserver] + processors: [resourcedetection/system] \ No newline at end of file diff --git a/test/packages/parallel/sql_server_input_otel/changelog.yml b/test/packages/parallel/sql_server_input_otel/changelog.yml new file mode 100644 index 0000000000..cf98bfd007 --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.1.0" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/17429 \ No newline at end of file diff --git a/test/packages/parallel/sql_server_input_otel/docs/README.md b/test/packages/parallel/sql_server_input_otel/docs/README.md new file mode 100644 index 0000000000..2b255b7674 --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/docs/README.md @@ -0,0 +1,138 @@ +# SQL Server OpenTelemetry Input Package + +## Overview + +The SQL Server OpenTelemetry Input Package for Elastic enables collection of telemetry data from Microsoft SQL Server instances through OpenTelemetry protocols using the [sqlserverreceiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/sqlserverreceiver). + +### How it works + +This package receives telemetry data from SQL Server instances by configuring the SQL Server receiver in the Input Package, which then gets applied to the sqlserverreceiver present in the EDOT collector, which then forwards the data to Elastic Agent. The Elastic Agent processes and enriches the data before sending it to Elasticsearch for indexing and analysis. + +## Requirements + +### Windows Performance Counters + +Make sure to run the collector as administrator to collect all performance counters for metrics. + +### Direct Connection + +When configured to directly connect to the SQL Server instance, the user must have the following permissions: + +1. At least one of the following permissions: + - `CREATE DATABASE` + - `ALTER ANY DATABASE` + - `VIEW ANY DATABASE` + +2. Permission to view server state: + - SQL Server pre-2022: `VIEW SERVER STATE` + - SQL Server 2022 and later: `VIEW SERVER PERFORMANCE STATE` + +## Configuration + +For the full list of settings exposed for the receiver and examples, refer to the [configuration](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/sqlserverreceiver#configuration) section. + +## Metrics reference + +For a complete list of all available metrics and their detailed descriptions, refer to the [SQL Server Receiver documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/sqlserverreceiver/documentation.md) in the upstream OpenTelemetry Collector repository. + +## Logs reference + +The SQL Server receiver can collect log events when a direct database connection is configured. Two event types are available, both turned off by default: + +- **Query Sample Events** (`db.server.query_sample`): Captures currently executing queries at scrape time, including session details, wait information, and resource consumption. Enable by setting **Enable Query Sample Events** to `true`. + +- **Top Query Events** (`db.server.top_query`): Captures the most expensive queries by execution time within a configurable lookback window, including execution counts, CPU time, and logical reads. Enable by setting **Enable Top Query Events** to `true`. + +Both event types require a direct database connection. Configure either the individual connection settings (server, port, username, and password) or the datasource connection string. The `query_sample_collection` and `top_query_collection` settings control the behavior of each event type. + +For a complete list of log attributes, refer to the [SQL Server Receiver logs documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/sqlserverreceiver/logs-documentation.md) in the upstream OpenTelemetry Collector repository. + +## Known limitations + +### Feature gate: `receiver.sqlserver.RemoveServerResourceAttribute` + +Starting with EDOT Collector versions based on OpenTelemetry Collector Contrib v0.129.0+, the upstream receiver includes a feature gate `receiver.sqlserver.RemoveServerResourceAttribute` that removes `server.address` and `server.port` from resource attributes, as they are not identified as resource attributes in the semantic conventions. This feature gate is currently opt-in. Refer to the [upstream documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/sqlserverreceiver#feature-gate) for details. + +An example event looks as following: + +```json +{ + "@timestamp": "2026-03-23T18:57:30.184Z", + "_metric_names_hash": "8986448896debcf6", + "data_stream": { + "dataset": "sqlserverreceiver.otel", + "namespace": "25898", + "type": "metrics" + }, + "event": { + "agent_id_status": "missing", + "dataset": "sqlserverreceiver.otel", + "ingested": "2026-03-23T18:57:40Z" + }, + "host": { + "name": "elastic-agent-58948", + "os": { + "platform": "linux" + } + }, + "metrics": { + "sqlserver": { + "batch": { + "request": { + "rate": 64 + } + }, + "lock": { + "wait": { + "rate": 3 + } + } + } + }, + "os": { + "type": "linux" + }, + "resource": { + "attributes": { + "host": { + "name": "elastic-agent-58948" + }, + "os": { + "type": "linux" + }, + "service": { + "instance": { + "id": "svc-sql_server_input_otel:1433" + } + } + }, + "schema_url": "https://opentelemetry.io/schemas/1.38.0" + }, + "scope": { + "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sqlserverreceiver", + "version": "9.4.0" + }, + "service": { + "instance": { + "id": "svc-sql_server_input_otel:1433" + }, + "node": { + "name": "svc-sql_server_input_otel:1433" + } + }, + "sqlserver": { + "batch": { + "request": { + "rate": 64 + } + }, + "lock": { + "wait": { + "rate": 3 + } + } + }, + "start_timestamp": "2026-03-23T18:57:29.073Z", + "unit": "{requests}/s" +} +``` diff --git a/test/packages/parallel/sql_server_input_otel/img/sql_server_otellogo.svg b/test/packages/parallel/sql_server_input_otel/img/sql_server_otellogo.svg new file mode 100644 index 0000000000..abac852841 --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/img/sql_server_otellogo.svg @@ -0,0 +1,21 @@ + + + SQL Server with OpenTelemetry + + + + + + + + + + + + + + + + + + diff --git a/test/packages/parallel/sql_server_input_otel/manifest.yml b/test/packages/parallel/sql_server_input_otel/manifest.yml new file mode 100644 index 0000000000..0e02a4b03b --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/manifest.yml @@ -0,0 +1,163 @@ +format_version: 3.6.0 +name: sql_server_input_otel +title: "SQL Server OpenTelemetry Input Package" +version: "0.1.0" +source: + license: "Elastic-2.0" +description: "Collect SQL Server metrics and logs using OpenTelemetry Collector" +type: input +categories: + - datastore + - observability + - opentelemetry +conditions: + kibana: + version: "^9.4.0" + elastic: + subscription: "basic" +icons: + - src: /img/sql_server_otellogo.svg + title: SQL Server logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: sqlserverreceiver + title: SQL Server Metrics and Logs (OpenTelemetry) + description: Collect SQL Server metrics and logs using OpenTelemetry Collector + input: otelcol + template_path: input.yml.hbs + dynamic_signal_types: true + vars: + - name: server + type: text + required: false + title: Server + description: >- + IP Address or hostname of SQL Server instance to connect to. + All connection settings (server, port, username, password) must be provided together for a direct connection. + Cannot be used together with datasource. + show_user: true + - name: port + type: integer + required: false + title: Port + description: >- + Port of the SQL Server instance to connect to. + All connection settings (server, port, username, password) must be provided together for a direct connection. + Cannot be used together with datasource. + show_user: true + - name: username + type: text + required: false + title: Username + description: The username used to connect to the SQL Server instance. + show_user: true + - name: password + type: password + required: false + title: Password + description: The password used to connect to the SQL Server instance. + secret: false + show_user: true + - name: datasource + type: text + required: false + title: Data Source (Connection String) + description: >- + ADO connection string for direct connection to SQL Server. + Use as an alternative to the individual server, port, username, and password settings. + Cannot be used together with server, port, username, or password. + secret: false + show_user: false + - name: instance_name + type: text + required: false + title: Instance Name + description: The instance name identifies the specific SQL Server instance being monitored. If unspecified, metrics will be scraped from all instances. + show_user: false + - name: computer_name + type: text + required: false + title: Computer Name + description: The computer name identifies the SQL Server name or IP address of the computer being monitored. Required when instance_name is specified on Windows. + show_user: false + - name: collection_interval + type: duration + required: false + title: Collection Interval + description: Time between each collection (e.g., 10s, 1m). + default: 10s + show_user: false + - name: initial_delay + type: duration + required: false + title: Initial Delay + description: Defines how long this receiver waits before starting. + default: 1s + show_user: false + - name: enable_query_sample_events + type: bool + required: false + title: Enable Query Sample Events + description: Capture currently executing queries as log events (db.server.query_sample). + default: false + show_user: false + - name: enable_top_query_events + type: bool + required: false + title: Enable Top Query Events + description: Capture the most expensive queries as log events (db.server.top_query). + default: false + show_user: false + - name: query_sample_max_rows_per_query + type: integer + required: false + title: Query Sample Max Rows Per Query + description: Maximum number of rows to collect per scrape for query samples. + default: 100 + show_user: false + - name: top_query_lookback_time + type: duration + required: false + title: Top Query Lookback Time + description: The time window to query for top queries (e.g., 60s, 2m). + default: 60s + show_user: false + - name: top_query_max_query_sample_count + type: integer + required: false + title: Top Query Max Sample Count + description: Maximum number of records to fetch in a single run. + default: 1000 + show_user: false + - name: top_query_count + type: integer + required: false + title: Top Query Count + description: Maximum number of active queries to report in a single run. + default: 250 + show_user: false + - name: top_query_collection_interval + type: duration + required: false + title: Top Query Collection Interval + description: The interval at which top queries should be emitted. + default: 60s + show_user: false + - name: resource_attributes + type: yaml + title: Resource Attributes + description: >- + Enable or disable resource attributes on collected data. + Available attributes: sqlserver.computer.name, sqlserver.instance.name, server.address, server.port, service.instance.id, host.name, sqlserver.database.name. + required: false + show_user: false + - name: metrics + type: yaml + title: Metrics + description: Enable or disable specific metrics + required: false + show_user: false +owner: + github: elastic/ecosystem + type: elastic diff --git a/test/packages/parallel/sql_server_input_otel/sample_event.json b/test/packages/parallel/sql_server_input_otel/sample_event.json new file mode 100644 index 0000000000..b0a6b84033 --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/sample_event.json @@ -0,0 +1,79 @@ +{ + "@timestamp": "2026-03-23T18:57:30.184Z", + "_metric_names_hash": "8986448896debcf6", + "data_stream": { + "dataset": "sqlserverreceiver.otel", + "namespace": "25898", + "type": "metrics" + }, + "event": { + "agent_id_status": "missing", + "dataset": "sqlserverreceiver.otel", + "ingested": "2026-03-23T18:57:40Z" + }, + "host": { + "name": "elastic-agent-58948", + "os": { + "platform": "linux" + } + }, + "metrics": { + "sqlserver": { + "batch": { + "request": { + "rate": 64 + } + }, + "lock": { + "wait": { + "rate": 3 + } + } + } + }, + "os": { + "type": "linux" + }, + "resource": { + "attributes": { + "host": { + "name": "elastic-agent-58948" + }, + "os": { + "type": "linux" + }, + "service": { + "instance": { + "id": "svc-sql_server_input_otel:1433" + } + } + }, + "schema_url": "https://opentelemetry.io/schemas/1.38.0" + }, + "scope": { + "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sqlserverreceiver", + "version": "9.4.0" + }, + "service": { + "instance": { + "id": "svc-sql_server_input_otel:1433" + }, + "node": { + "name": "svc-sql_server_input_otel:1433" + } + }, + "sqlserver": { + "batch": { + "request": { + "rate": 64 + } + }, + "lock": { + "wait": { + "rate": 3 + } + } + }, + "start_timestamp": "2026-03-23T18:57:29.073Z", + "unit": "{requests}/s" +} diff --git a/test/packages/parallel/sql_server_input_otel/sample_event_logs.json b/test/packages/parallel/sql_server_input_otel/sample_event_logs.json new file mode 100644 index 0000000000..0b97fb0991 --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/sample_event_logs.json @@ -0,0 +1,166 @@ +{ + "@timestamp": "2026-03-23T18:56:05.895Z", + "attributes": { + "client": { + "address": "elastic-agent-75417", + "port": 36936 + }, + "db": { + "namespace": "master", + "query": { + "text": "WITH PerfCounters SELECT DISTINCT RTRIM ( spi. object_name ) object_name, RTRIM ( spi. counter_name ) counter_name, RTRIM ( spi. instance_name ), CAST ( spi. cntr_value ), spi. cntr_type FROM sys.dm_os_performance_counters WHERE counter_name IN ( ? ) OR ( spi. object_name LIKE ? OR spi. object_name LIKE ? OR spi. object_name LIKE ? ) OR ( spi. instance_name IN ( ? ) AND spi. counter_name IN ( ? ) ) ) INSERT INTO @PCounters SELECT * FROM PerfCounters" + }, + "system": { + "name": "microsoft.sql_server" + } + }, + "network": { + "peer": { + "address": "172.20.0.2", + "port": 36936 + } + }, + "sqlserver": { + "blocking_session_id": 0, + "command": "INSERT", + "context_info": "", + "cpu_time": 0.043, + "deadlock_priority": -10, + "estimated_completion_time": 0, + "lock_timeout": -0.001, + "logical_reads": 607, + "open_transaction_count": 0, + "percent_complete": 0, + "query_hash": "6df033ef55623a4a", + "query_plan_hash": "49c16c014d7e4505", + "query_start": "2026-03-23T18:56:05.837+00:00", + "reads": 40, + "request_status": "running", + "row_count": 1, + "session_id": 51, + "session_status": "running", + "total_elapsed_time": 0.043, + "transaction_id": 0, + "transaction_isolation_level": 2, + "wait_resource": "", + "wait_time": 0, + "wait_type": "", + "writes": 19 + }, + "user": { + "name": "sa" + } + }, + "client": { + "address": "elastic-agent-75417", + "port": 36936 + }, + "data_stream": { + "dataset": "sqlserverreceiver.otel", + "namespace": "73861", + "type": "logs" + }, + "db": { + "namespace": "master", + "query": { + "text": "WITH PerfCounters SELECT DISTINCT RTRIM ( spi. object_name ) object_name, RTRIM ( spi. counter_name ) counter_name, RTRIM ( spi. instance_name ), CAST ( spi. cntr_value ), spi. cntr_type FROM sys.dm_os_performance_counters WHERE counter_name IN ( ? ) OR ( spi. object_name LIKE ? OR spi. object_name LIKE ? OR spi. object_name LIKE ? ) OR ( spi. instance_name IN ( ? ) AND spi. counter_name IN ( ? ) ) ) INSERT INTO @PCounters SELECT * FROM PerfCounters" + }, + "system": { + "name": "microsoft.sql_server" + } + }, + "event": { + "agent_id_status": "missing", + "dataset": "sqlserverreceiver.otel", + "ingested": "2026-03-23T18:56:15Z" + }, + "event_name": "db.server.query_sample", + "host": { + "name": "elastic-agent-75417", + "os": { + "platform": "linux" + } + }, + "network": { + "peer": { + "address": "172.20.0.2", + "port": 36936 + } + }, + "observed_timestamp": "1970-01-01T00:00:00.000Z", + "os": { + "type": "linux" + }, + "resource": { + "attributes": { + "host": { + "name": "elastic-agent-75417" + }, + "os": { + "type": "linux" + }, + "service": { + "instance": { + "id": "svc-sql_server_input_otel:1433" + } + }, + "sqlserver": { + "computer": { + "name": "elastic-agent-75417" + }, + "instance": { + "name": "3c03efd12c01" + } + } + }, + "schema_url": "https://opentelemetry.io/schemas/1.38.0" + }, + "scope": { + "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sqlserverreceiver", + "version": "9.4.0" + }, + "service": { + "instance": { + "id": "svc-sql_server_input_otel:1433" + }, + "node": { + "name": "svc-sql_server_input_otel:1433" + } + }, + "sqlserver": { + "blocking_session_id": 0, + "command": "INSERT", + "computer": { + "name": "elastic-agent-75417" + }, + "context_info": "", + "cpu_time": 0.043, + "deadlock_priority": -10, + "estimated_completion_time": 0, + "instance": { + "name": "3c03efd12c01" + }, + "lock_timeout": -0.001, + "logical_reads": 607, + "open_transaction_count": 0, + "percent_complete": 0, + "query_hash": "6df033ef55623a4a", + "query_plan_hash": "49c16c014d7e4505", + "query_start": "2026-03-23T18:56:05.837+00:00", + "reads": 40, + "request_status": "running", + "row_count": 1, + "session_id": 51, + "session_status": "running", + "total_elapsed_time": 0.043, + "transaction_id": 0, + "transaction_isolation_level": 2, + "wait_resource": "", + "wait_time": 0, + "wait_type": "", + "writes": 19 + }, + "user": { + "name": "sa" + } +} diff --git a/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json b/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json new file mode 100644 index 0000000000..bb204294f6 --- /dev/null +++ b/test/packages/parallel/sql_server_input_otel/sample_event_metrics.json @@ -0,0 +1,79 @@ +{ + "@timestamp": "2026-03-23T18:56:05.948Z", + "_metric_names_hash": "8986448896debcf6", + "data_stream": { + "dataset": "sqlserverreceiver.otel", + "namespace": "73861", + "type": "metrics" + }, + "event": { + "agent_id_status": "missing", + "dataset": "sqlserverreceiver.otel", + "ingested": "2026-03-23T18:56:15Z" + }, + "host": { + "name": "elastic-agent-75417", + "os": { + "platform": "linux" + } + }, + "metrics": { + "sqlserver": { + "batch": { + "request": { + "rate": 65 + } + }, + "lock": { + "wait": { + "rate": 5 + } + } + } + }, + "os": { + "type": "linux" + }, + "resource": { + "attributes": { + "host": { + "name": "elastic-agent-75417" + }, + "os": { + "type": "linux" + }, + "service": { + "instance": { + "id": "svc-sql_server_input_otel:1433" + } + } + }, + "schema_url": "https://opentelemetry.io/schemas/1.38.0" + }, + "scope": { + "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sqlserverreceiver", + "version": "9.4.0" + }, + "service": { + "instance": { + "id": "svc-sql_server_input_otel:1433" + }, + "node": { + "name": "svc-sql_server_input_otel:1433" + } + }, + "sqlserver": { + "batch": { + "request": { + "rate": 65 + } + }, + "lock": { + "wait": { + "rate": 5 + } + } + }, + "start_timestamp": "2026-03-23T18:56:04.825Z", + "unit": "{requests}/s" +}