OpenTelemetry

traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01
             version-trace id (32 hex)-parent span id (16 hex)-flags (01 = sampled)

OTEL_SERVICE_NAME=my-app
OTEL_RESOURCE_ATTRIBUTES=service.version=1.4.2,deployment.environment.name=prod
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector.observability:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_TRACES_SAMPLER=parentbased_traceidratio
OTEL_TRACES_SAMPLER_ARG=0.1
OTEL_PROPAGATORS=tracecontext,baggage

tr := otel.Tracer("checkout")
ctx, span := tr.Start(ctx, "reserve_inventory")
defer span.End()
span.SetAttributes(attribute.String("sku", sku), attribute.Int("qty", qty))
if err != nil {
    span.RecordError(err)
    span.SetStatus(codes.Error, "reservation failed")
}

import (
    "context"

    "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" // otlptracegrpc for 4317
    "go.opentelemetry.io/otel/propagation"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
)

func setupTracing(ctx context.Context) (func(context.Context) error, error) {
    exp, err := otlptracehttp.New(ctx)          // endpoint, headers and TLS from OTEL_EXPORTER_OTLP_* variables
    if err != nil {
        return nil, err
    }
    res, err := resource.Merge(resource.Default(), // env detector: OTEL_SERVICE_NAME, OTEL_RESOURCE_ATTRIBUTES
        resource.NewWithAttributes(semconv.SchemaURL, semconv.ServiceVersion("1.4.2")))
    if err != nil {
        return nil, err
    }
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exp),               // BatchSpanProcessor; exports every 5s or 512 spans by default
        sdktrace.WithResource(res),
    )
    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{}, propagation.Baggage{}))
    return tp.Shutdown, nil                      // call with a timeout on exit or the last batch is lost
}

// Server: one span per request, named after the route pattern when the mux provides one
mux := http.NewServeMux()
srv := &http.Server{Addr: ":8080", Handler: otelhttp.NewHandler(mux, "server")}

// Client: injects traceparent into every outbound request
client := &http.Client{Transport: otelhttp.NewTransport(http.DefaultTransport)}

uv pip install opentelemetry-distro opentelemetry-exporter-otlp   # SDK, launcher and both OTLP exporters
opentelemetry-bootstrap -a install       # detects installed libraries (Flask, requests, psycopg, ...) and installs their instrumentations
OTEL_SERVICE_NAME=my-app OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 \
  opentelemetry-instrument gunicorn my_app:app   # wraps the interpreter; no code change
opentelemetry-instrument --traces_exporter console --metrics_exporter none python main.py   # spans to stdout for a local check

from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor

provider = TracerProvider(resource=Resource.create({"service.name": "my-app", "service.version": "1.4.2"}))
provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))   # endpoint from OTEL_EXPORTER_OTLP_ENDPOINT
trace.set_tracer_provider(provider)

tracer = trace.get_tracer("checkout")
with tracer.start_as_current_span("reserve_inventory") as span:      # context manager ends the span and restores the parent
    span.set_attribute("sku", sku)
    try:
        reserve(sku)
    except ReservationError as exc:
        span.record_exception(exc)
        span.set_status(trace.StatusCode.ERROR, "reservation failed")
        raise

# Jaeger all-in-one for local work: UI on 16686, OTLP on 4317 and 4318
podman run --rm -p 16686:16686 -p 4317:4317 -p 4318:4318 cr.jaegertracing.io/jaegertracing/jaeger:2.21.0
# Tempo's OTLP receiver listens on the same ports; grpc endpoints take host:port, HTTP takes a URL

exporters:
  otlp_grpc/jaeger:
    endpoint: jaeger.observability:4317
    tls: { insecure: true }              # plaintext inside the cluster; omit for TLS
  otlp_http/tempo:
    endpoint: https://tempo.example.com   # /v1/traces is appended
    headers: { X-Scope-OrgID: my-tenant }
  prometheus:                            # pull: exposes /metrics for Prometheus to scrape
    endpoint: 0.0.0.0:9464
    metric_expiration: 5m                # series without updates disappear after this
    enable_open_metrics: true            # required for exemplars
  prometheus_remote_write:               # push: Prometheus with remote-write receiver, Mimir, Thanos
    endpoint: https://mimir.example.com/api/v1/push
    resource_to_telemetry_conversion: { enabled: true }   # copy resource attributes to labels

processors:
  filter:
    error_mode: ignore
    trace_conditions:
      - span.attributes["url.path"] == "/healthz"          # drop probe spans
      - resource.attributes["service.name"] == "load-test"
    log_conditions:
      - log.severity_number < SEVERITY_NUMBER_WARN            # keep warnings and above
    metric_conditions:
      - metric.name == "go_gc_duration_seconds"
  probabilistic_sampler:      # head-style sampling in the Collector, consistent by trace ID
    sampling_percentage: 15
    mode: proportional        # honours a previous sampler's decision recorded in tracestate

receivers:
  otlp:
    protocols:
      grpc: { endpoint: 0.0.0.0:4317 }     # default is localhost; 0.0.0.0 is needed in a container
      http: { endpoint: 0.0.0.0:4318 }

processors:
  memory_limiter:                 # first: refuses data before the process is OOM-killed
    check_interval: 1s
    limit_percentage: 80
    spike_limit_percentage: 25
  k8s_attributes:                 # adds pod, namespace, node and workload attributes
    auth_type: serviceAccount
    extract:
      metadata: [k8s.namespace.name, k8s.pod.name, k8s.deployment.name, k8s.node.name]
  resource_detection:
    detectors: [env, system]
  transform:
    error_mode: ignore
    trace_statements:
      - context: span
        statements:
          - delete_key(attributes, "http.request.header.authorization")
  batch:                          # after memory_limiter and any sampling
    timeout: 5s
    send_batch_size: 8192

exporters:
  otlp_http/tempo:
    endpoint: https://tempo.example.com
    retry_on_failure: { enabled: true, max_elapsed_time: 300s }
    sending_queue: { enabled: true, queue_size: 5000 }
  prometheus_remote_write:
    endpoint: https://mimir.example.com/api/v1/push
  debug:
    verbosity: detailed           # troubleshooting only: logs every record

extensions:
  health_check: { endpoint: 0.0.0.0:13133 }
  zpages: {}

service:
  extensions: [health_check, zpages]
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, k8s_attributes, transform, batch]
      exporters: [otlp_http/tempo]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, k8s_attributes, batch]
      exporters: [prometheus_remote_write]
  telemetry:
    metrics:
      level: detailed
      readers:
        - pull:
            exporter:
              prometheus: { host: 0.0.0.0, port: 8888 }

apiVersion: opentelemetry.io/v1alpha1
kind: Instrumentation
metadata:
  name: default
spec:
  exporter:
    endpoint: http://otel-collector:4318
  propagators: [tracecontext, baggage]
  sampler:
    type: parentbased_traceidratio
    argument: "0.1"

# Pod template annotation; also -python, -nodejs, -dotnet, -go
metadata:
  annotations:
    instrumentation.opentelemetry.io/inject-java: "true"

processors:
  tail_sampling:
    decision_wait: 10s        # default 30s
    num_traces: 100000        # traces held in memory; default 50000
    policies:
      - name: errors
        type: status_code
        status_code: { status_codes: [ERROR] }
      - name: slow
        type: latency
        latency: { threshold_ms: 1000 }
      - name: baseline
        type: probabilistic
        probabilistic: { sampling_percentage: 5 }

{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"curl-test"}}]},"scopeSpans":[{"scope":{"name":"manual"},"spans":[{"traceId":"5b8efff798038103d269b633813fc60c","spanId":"eee19b7ec3c1b174","name":"test-span","kind":1,"startTimeUnixNano":"1700000000000000000","endTimeUnixNano":"1700000001000000000"}]}]}]}

# Save the JSON above as span.json, then post it to the OTLP/HTTP receiver
curl -sS -w '\n%{http_code}\n' -X POST http://localhost:4318/v1/traces \
  -H 'Content-Type: application/json' -d @span.json

{"partialSuccess":{}}
200

# Validate before rolling out
otelcol validate --config=config.yaml

# Data refused, failed to send, or failed to enqueue
curl -s localhost:8888/metrics | grep -E 'otelcol_(receiver_refused|exporter_send_failed|exporter_enqueue_failed)'

# Queue fill level against capacity
curl -s localhost:8888/metrics | grep -E 'otelcol_exporter_queue_(size|capacity)'

# Spans accepted per receiver
curl -s localhost:8888/metrics | grep otelcol_receiver_accepted_spans

# Collector memory
curl -s localhost:8888/metrics | grep -E 'otelcol_process_(memory_rss|runtime_heap_alloc_bytes)'

# Confirm a service forwards the trace header (look for traceparent on outbound calls in its logs or proxy)
curl -sv -H 'traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01' https://api.example.com/healthz 2>&1 | grep -i traceparent

# Watch records flowing through the debug exporter
kubectl logs -n observability deploy/otel-collector -f | grep -m5 'InstrumentationScope'

# What an SDK process will read from its environment
env | grep ^OTEL_

# Which receivers, processors, exporters and extensions this binary contains
otelcol components | grep -A40 '^processors:'

# Validate with an inline override layered on the file (the yaml: provider)
otelcol validate --config=config.yaml --config='yaml:service::telemetry::logs::level: debug'

# Merge a local override on top of the deployed config (later files win)
otelcol --config=config.yaml --config=overrides.yaml

# Is the health check extension happy (200) or not (503)
curl -s -o /dev/null -w '%{http_code}\n' localhost:13133

# Spans dropped by tail sampling, per policy decision
curl -s localhost:8888/metrics | grep -E 'otelcol_processor_tail_sampling_(count_traces_sampled|sampling_decision_latency)'

# Data the memory_limiter refused
curl -s localhost:8888/metrics | grep otelcol_processor_refused

# Batch sizes actually sent, to tune send_batch_size
curl -s localhost:8888/metrics | grep otelcol_processor_batch_batch_send_size

# Export throughput per exporter and signal
curl -s localhost:8888/metrics | grep -E 'otelcol_exporter_sent_(spans|metric_points|log_records)'

# Late spans that arrived after a tail-sampling decision
curl -s localhost:8888/metrics | grep otelcol_processor_tail_sampling_late_span_age

# Follow sampled trace fragments through zpages for one span name
curl -s 'localhost:55679/debug/tracez?zspanname=GET%20/api&ztype=1&zsubtype=0' | sed 's/<[^>]*>//g' | grep -v '^\s*$' | head -40

# Post a test metric to the OTLP/HTTP receiver
curl -sS -X POST localhost:4318/v1/metrics -H 'Content-Type: application/json' -d '{"resourceMetrics":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"curl-test"}}]},"scopeMetrics":[{"metrics":[{"name":"test.counter","sum":{"aggregationTemporality":2,"isMonotonic":true,"dataPoints":[{"asInt":"1","timeUnixNano":"'$(date +%s%N)'"}]}}]}]}]}'

# Post a test log record with a trace ID so the backend can link it
curl -sS -X POST localhost:4318/v1/logs -H 'Content-Type: application/json' -d '{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"curl-test"}}]},"scopeLogs":[{"logRecords":[{"timeUnixNano":"'$(date +%s%N)'","severityNumber":9,"severityText":"INFO","body":{"stringValue":"hello"},"traceId":"5b8efff798038103d269b633813fc60c","spanId":"eee19b7ec3c1b174"}]}]}]}'

# Check the gRPC receiver speaks OTLP without an SDK: any grpc-status header back (even an error) proves the service is there
curl -sS --http2-prior-knowledge -X POST -H 'content-type: application/grpc' -D - -o /dev/null http://localhost:4317/opentelemetry.proto.collector.trace.v1.TraceService/Export | grep -i grpc-status

# Generate a valid random traceparent for a manual request
printf 'traceparent: 00-%s-%s-01\n' "$(openssl rand -hex 16)" "$(openssl rand -hex 8)"

# Send a request with a fresh trace and print the IDs so you can search the backend for them
tp="00-$(openssl rand -hex 16)-$(openssl rand -hex 8)-01"; echo "$tp"; curl -sS -o /dev/null -H "traceparent: $tp" https://api.example.com/orders

# Print the spans the Go SDK would export, without a Collector
OTEL_TRACES_EXPORTER=console go run ./cmd/my-app

# Same for Python, one span per line
opentelemetry-instrument --traces_exporter console --metrics_exporter none --logs_exporter none python main.py

# Which instrumentations the Python launcher will activate
opentelemetry-bootstrap -a requirements

# Pods with auto-instrumentation injected by the Operator
kubectl get pods -A -o json | jq -r '.items[] | select(.metadata.annotations["instrumentation.opentelemetry.io/inject-java"]=="true" or .metadata.annotations["instrumentation.opentelemetry.io/inject-python"]=="true") | "\(.metadata.namespace)/\(.metadata.name)"'

# OTEL_ variables the Operator injected into a running pod
kubectl exec -n my-namespace my-pod -c app -- env | grep ^OTEL_ | sort

# Collector pods and their restart counts
kubectl get pods -n observability -l app.kubernetes.io/name=opentelemetry-collector -o custom-columns='POD:.metadata.name,RESTARTS:.status.containerStatuses[0].restartCount,REASON:.status.containerStatuses[0].lastState.terminated.reason'

# Collector config as deployed, from its ConfigMap (the Helm chart stores it under the key "relay")
kubectl get cm -n observability otel-collector -o jsonpath='{.data.relay}' > deployed.yaml && otelcol validate --config=deployed.yaml

# Reload Collector pods after a ConfigMap change (they do not watch the file)
kubectl rollout restart deploy/otel-collector -n observability

# Errors and forbidden responses in the Collector log in the last 10 minutes
kubectl logs -n observability deploy/otel-collector --since=10m | grep -E '"level":"(error|warn)"' | head

# Search Tempo for the trace you just generated
curl -sS "https://tempo.example.com/api/traces/${tp:3:32}" -H "X-Scope-OrgID: my-tenant" | jq '.batches[].resource'

# Search Jaeger for recent traces from one service
curl -sS 'http://localhost:16686/api/traces?service=my-app&limit=5' | jq '.data[].traceID'

#!/usr/bin/env bash
# usage: otel-ingest-check.sh [collector-host] ; needs curl and the Collector's 8888 port reachable
set -euo pipefail
host=${1:-localhost}
accepted() { curl -sf "http://$host:8888/metrics" | awk '/^otelcol_receiver_accepted_spans/ {s += $2} END {print s + 0}'; }
before=$(accepted)
tid=$(openssl rand -hex 16); sid=$(openssl rand -hex 8); now=$(date +%s%N)
body=$(printf '{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"ingest-check"}}]},"scopeSpans":[{"spans":[{"traceId":"%s","spanId":"%s","name":"ingest-check","kind":1,"startTimeUnixNano":"%s","endTimeUnixNano":"%s"}]}]}]}' "$tid" "$sid" "$now" "$now")
code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST "http://$host:4318/v1/traces" -H 'Content-Type: application/json' -d "$body")
[[ $code == 200 ]] || { printf 'receiver returned %s\n' "$code" >&2; exit 1; }
sleep 2
after=$(accepted)
printf 'trace %s accepted; receiver counter %s -> %s\n' "$tid" "$before" "$after"
(( after > before )) || { printf 'counter did not move: is 8888 the right Collector?\n' >&2; exit 1; }

#!/usr/bin/env bash
# usage: otel-collector-report.sh [collector-host]
set -euo pipefail
host=${1:-localhost}
m=$(curl -sf --max-time 5 "http://$host:8888/metrics") || { echo "cannot scrape $host:8888" >&2; exit 1; }
health=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://$host:13133" || true)
printf 'health_check: %s\n\n' "${health:-unreachable}"
printf '%-34s %-28s %12s\n' METRIC COMPONENT VALUE
printf '%s\n' "$m" | awk '
  /^otelcol_(receiver_refused|exporter_send_failed|exporter_enqueue_failed|processor_refused|exporter_queue_size|exporter_queue_capacity)/ {
    name = $1; sub(/\{.*/, "", name)
    comp = ""; if (match($0, /(receiver|exporter|processor)="[^"]*"/)) comp = substr($0, RSTART, RLENGTH)
    if ($2 + 0 > 0) printf "%-34s %-28s %12d\n", name, comp, $2
  }'
printf '\nrss: %s MiB\n' "$(printf '%s\n' "$m" | awk '/^otelcol_process_memory_rss/ {printf "%d", $2 / 1048576}')"

#!/usr/bin/env python3
"""usage: otel-test-trace.py [http://localhost:4318]"""
import json, secrets, sys, time, urllib.request

endpoint = (sys.argv[1] if len(sys.argv) > 1 else "http://localhost:4318").rstrip("/")
trace_id, parent_id, child_id = secrets.token_hex(16), secrets.token_hex(8), secrets.token_hex(8)
now = time.time_ns()

def span(span_id, name, start, end, parent=None, kind=2):
    s = {"traceId": trace_id, "spanId": span_id, "name": name, "kind": kind,
         "startTimeUnixNano": str(start), "endTimeUnixNano": str(end),
         "attributes": [{"key": "http.request.method", "value": {"stringValue": "GET"}}]}
    if parent:
        s["parentSpanId"] = parent
    return s

payload = {"resourceSpans": [{
    "resource": {"attributes": [{"key": "service.name", "value": {"stringValue": "test-trace"}}]},
    "scopeSpans": [{"scope": {"name": "manual"}, "spans": [
        span(parent_id, "GET /orders", now - 300_000_000, now),
        span(child_id, "SELECT orders", now - 250_000_000, now - 50_000_000, parent=parent_id, kind=3),
    ]}]}]}
req = urllib.request.Request(f"{endpoint}/v1/traces", data=json.dumps(payload).encode(),
                             headers={"Content-Type": "application/json"}, method="POST")
with urllib.request.urlopen(req, timeout=10) as resp:
    print(resp.status, resp.read().decode())
print(f"trace_id={trace_id}")