Quick Start: Elasticsearch + OpenTelemetry Collector

$composeContent = @"
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.10.1
    environment:
      - discovery.type=single-node
    ports:
      - "39200:9200"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9200"]
      interval: 3s
      timeout: 3s
      retries: 15

  opentelemetry-collector:
    image: otel/opentelemetry-collector-contrib:latest
    environment:
      - ELASTIC_API_KEY=elasticsearch-apikey
    command: --config /etc/otel-collector-config.yml
    volumes:
      - ./otel-collector-config.yml:/etc/otel-collector-config.yml
    ports:
      - "4317:4317"
      - "4318:4318"
    depends_on:
      elasticsearch:
        condition: service_healthy
"@

$composeFile = "docker-compose.yml"
$composeContent | Out-File -FilePath $composeFile -Encoding UTF8

$otelConfig = @"
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  memory_limiter:
    check_interval: 1s
    limit_mib: 4000
    spike_limit_mib: 800

# natively that setup do not support substitute values for creating indexes like that '[service.name]-logs-yyyy.MM'
# So we need to fill in elasticsearch.index.prefix and elasticsearch.index.suffix or use logstash
  resource:
    attributes:
      - action: upsert
        key: "elasticsearch.index.prefix"
        from_attribute: "service.name"

  batch:
    timeout: 10s
    send_batch_size: 10000
    send_batch_max_size: 11000

exporters:
  elasticsearch:
    endpoint: "http://elasticsearch:9200"
    headers:
      Authorization: "ApiKey ${ELASTIC_API_KEY}"
    logs_index: "logs"
    logs_dynamic_index:
      enabled: true
    mapping:
      mode: "ecs"
    timeout: 5s

service:
  pipelines:
    logs:
      receivers: [otlp]
      processors: [memory_limiter, resource, batch]
      exporters: [elasticsearch]
"@

$otelConfigFile = "otel-collector-config.yml"
$otelConfig | Out-File -FilePath $otelConfigFile -Encoding UTF8

Write-Host "`n--- Bringing up Docker Compose services ---"

# Down any existing services
docker compose -p otel-poc -f $composeFile down

docker compose -p otel-poc -f $composeFile up -d

# Optional: Wait for Elasticsearch / Collector to be healthy
Write-Host "`nWaiting up to 60 seconds for Elasticsearch to become healthy..."
$healthCheckPassed = $false
for ($i = 1; $i -le 20; $i++) {
    # Check if ES responds on port 39200
    try {
        $result = Invoke-RestMethod -Uri "http://localhost:39200" -UseBasicParsing -ErrorAction Stop
        if ($result.name) {
            Write-Host "Elasticsearch is healthy."
            $healthCheckPassed = $true
            break
        }
    }
    catch {
        Write-Host "Still waiting..."
    }
    Start-Sleep -Seconds 3
}

if (-not $healthCheckPassed) {
    Write-Host "`nElasticsearch did not become healthy in time. Exiting..."
    exit 1
}

Write-Host "`n--- Sending test log to the Collector ---"

# Adjust $otlpEndpoint or add Authorization headers as needed
$otlpEndpoint = "http://localhost:4318/v1/logs"
$headers = @{
    "Content-Type" = "application/json"
    # "Authorization" = "Bearer otlp-collector-token"  # If needed
}

$logPayload = @"
{
  "resourceLogs": [
    {
      "resource": {
        "attributes": [
          {
            "key": "service.name",
            "value": { "stringValue": "myservice-" }
          },
          {
            "key": "elasticsearch.index.prefix",
            "value": { "stringValue": "myservice-" }
          },
          {
            "key": "elasticsearch.index.suffix",
            "value": { "stringValue": "-data" }
          }
        ]
      },
      "scopeLogs": [
        {
          "scope": {
            "name": "test-logger",
            "version": "1.0"
          },
          "logRecords": [
            {
              "timeUnixNano": "1673452395000000000",
              "severityNumber": 9,
              "severityText": "ERROR",
              "body": { "stringValue": "This is a test log from PowerShell" },
              "attributes": [
                {
                  "key": "attribute_key",
                  "value": { "stringValue": "attribute_value" }
                }
              ]
            }
          ]
        }
      ]
    }
  ]
}
"@

try {
    $response = Invoke-RestMethod -Uri $otlpEndpoint -Method Post -Headers $headers -Body $logPayload
    Write-Host "Collector Response: $response"
} catch {
    Write-Host "Error sending log: $($_.Exception.Message)"
    exit 1
}

# --- STEP 5: Query Elasticsearch to verify log ingestion ---
# <# We need to wait a bit for the log to be ingested #>
# Start-Sleep -Seconds 60

Write-Host "`n--- Querying Elasticsearch to see if our log is present ---"

$esQueryBody = @{
  query = @{
    match_all = @{}
  }
}

try {
    $searchResponse = Invoke-RestMethod `
        -Method Post `
        -Uri "http://localhost:39200/myservice-logs-data/_search?pretty" `
        -ContentType "application/json" `
        -Body ($esQueryBody | ConvertTo-Json)

    $hits = $searchResponse.hits.hits

    if ($hits.Count -gt 0) {
        Write-Host "`nFound $($hits.Count) log(s) in Elasticsearch!"
        Write-Host "Here are the hits:"
        $hits | ForEach-Object { $_._source | ConvertTo-Json -Depth 10 }
    }
    else {
        Write-Host "`nNo logs found in Elasticsearch index 'test-logs'."
    }
}
catch {
    Write-Host "Error querying Elasticsearch: $($_.Exception.Message)"
}

Write-Host "`nDone!"