add otel lb collector yaml

Author: Anusha Narapureddy

05-sampling.md

@@ -101,4 +101,13 @@ Requires two deployments of the Collector, the first layer routing all the spans
 
 [excalidraw](https://excalidraw.com/#room=6a15d65ba4615c535a40,xcZD6DG977owHRoxpYY4Ag)
 
+Apply the YAML below to deploy a layer of Collectors containing the load-balancing exporter in front of collectors performing tail-sampling:
+
+```shell
+kubectl apply -f https://raw.githubusercontent.com/pavolloffay/kubecon-eu-2024-opentelemetry-kubernetes-tracing-tutorial/backend/05-scale-otel-collectors.yaml
+kubectl get pods -n observability-backend -w
+```
+
+<TODO: Add screenshot>
+
 [Next steps](./06-RED-metrics.md)

backend/05-gateway-collector.yaml

@@ -70,3 +70,68 @@ spec:
         logs:
           receivers: [otlp]
           exporters: [debug]
+---
+apiVersion: opentelemetry.io/v1alpha1
+kind: OpenTelemetryCollector
+metadata:
+  name: otel-gateway
+  namespace: observability-backend
+spec:
+  image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.94.0
+  mode: deployment
+  replicas: 2
+  ports:
+    - port: 8888
+      protocol: TCP
+      name: metrics
+  config: |
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+            endpoint: 0.0.0.0:4318
+    
+    processors:   
+      # Sample 100% of traces with ERROR-ing spans (omit traces with all OK spans)
+      # and traces which have a duration longer than 500ms
+      tail_sampling:
+        decision_wait: 10s # time to wait before making a sampling decision is made
+        num_traces: 100 # number of traces to be kept in memory
+        expected_new_traces_per_sec: 10 # expected rate of new traces per second
+        policies:
+          - name: keep-errors
+            type: status_code
+            status_code: {status_codes: [ERROR]}
+          - name: keep-slow-traces
+            type: latency
+            latency: {threshold_ms: 500}
+
+    exporters:
+      otlp/traces:
+        endpoint: jaeger-collector:4317
+        tls:
+          insecure: true
+      
+      otlphttp/metrics:
+        endpoint: http://prometheus.observability-backend.svc.cluster.local:80/api/v1/otlp/
+        tls:
+          insecure: true
+      
+      debug:
+        verbosity: detailed
+
+    service:
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: [tail_sampling]
+          exporters: [otlp/traces]
+        metrics:
+          receivers: [otlp]
+          exporters: [otlphttp/metrics]
+        logs:
+          receivers: [otlp]
+          exporters: [debug]
+---