Centralized Logging with OpenTelemetry and OpenSearch stack on Kubernetes

$ helm repo add opensearch https://opensearch-project.github.io/helm-charts/

$ helm repo update

$ helm search repo opensearch

NAME                                    CHART VERSION   APP VERSION     DESCRIPTION
opensearch/opensearch                   2.23.0          2.16.0          A Helm chart for OpenSearch
opensearch/opensearch-dashboards        2.21.0          2.16.0          A Helm chart for OpenSearch Dashboards
opensearch/data-prepper                 0.1.0           2.8.0           A Helm chart for Data Prepper

$ helm show values opensearch/opensearch > opensearch-opensearch-values.yaml

# Please re-try with a minimum 8 character password and must contain at least one uppercase letter,
# one lowercase letter, one digit, and one special character that is strong.
# Password strength can be tested here: https://lowe.github.io/tryzxcvbn
extraEnvs:
  - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD
    value: your-password
  - name: DISABLE_INSTALL_DEMO_CONFIG
    value: "false"

secretMounts:
  - name: esnode-certs
    secretName: esnode-certs
    path: /usr/share/opensearch/config/certs
    defaultMode: "0600"

image:
  repository: "opensearchproject/opensearch"
  # override image tag, which is .Chart.AppVersion by default
  tag: "2.16.0"
  pullPolicy: "IfNotPresent"

resources:
  limits:
    cpu: "400m"
    memory: "2048Mi"
  requests:
    cpu: "200m"
    memory: "1024Mi"

# ERROR: [1] bootstrap checks failed
#[1]: max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
#sysctlVmMaxMapCount: 262144

nodeSelector:
  agentpool: linuxnodes

$ helm -n nsa2 install opensearch opensearch/opensearch \
-f nsa2-opensearch-opensearch-values.yaml

NAME: opensearch
LAST DEPLOYED: Mon Sep  9 20:14:13 2024
NAMESPACE: nsa2
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
Watch all cluster members come up.
  $ kubectl get pods --namespace=nsa2 -l app.kubernetes.io/component=opensearch-cluster-master -w

$ kubectl -n nsa2 port-forward service/opensearch-cluster-master 9200:9200

$ curl --insecure -u admin:$OS_ADMIN_PASSWORD https://localhost:9200

{
  "name" : "opensearch-cluster-master-0",
  "cluster_name" : "opensearch-cluster",
  "cluster_uuid" : "6U4IvOgoQbeYO1ONU8z4pg",
  "version" : {
    "distribution" : "opensearch",
    "number" : "2.16.0",
    "build_type" : "tar",
    "build_hash" : "f84a26e76807ea67a69822c37b1a1d89e7177d9b",
    "build_date" : "2024-08-06T20:30:45.209655408Z",
    "build_snapshot" : false,
    "lucene_version" : "9.11.1",
    "minimum_wire_compatibility_version" : "7.10.0",
    "minimum_index_compatibility_version" : "7.0.0"
  },
  "tagline" : "The OpenSearch Project: https://opensearch.org/"
}

$ helm -n nsa2 uninstall opensearch

$ kubectl -n nsa2 delete pvc -l app.kubernetes.io/instance=opensearch

$ helm show values opensearch/opensearch-dashboards > opensearch-dashboards-values.yaml

resources:
  requests:
    cpu: "100m"
    memory: "512M"
  limits:
    cpu: "400m"
    memory: "2048M"

nodeSelector:
#  agentpool: linuxnodes
  agentpool: depnodes

$ helm -n nsa2 install opensearch-dashboards opensearch/opensearch-dashboards \
  -f nsa2-opensearch-dashboards-values.yaml

$ helm -n nsa2 uninstall opensearch-dashboards

$ kubectl -n nsa2 port-forward service/opensearch-dashboards 5601:5601

GET /_cat/indices?v

$ kubectl -n nsa2 port-forward svc/opensearch-cluster-master 9200:9200

{
  "cluster_permissions": [
    "cluster_composite_ops",
    "indices_monitor"
  ],
  "index_permissions": [{
    "index_patterns": [
      "nsa2*"
    ],
    "dls": "",
    "fls": [],
    "masked_fields": [],
    "allowed_actions": [
      "indices:admin/create", "read", "write"
    ]
  }],
  "tenant_permissions": [{
    "tenant_patterns": [
      "human_resources"
    ],
    "allowed_actions": [
      "kibana_all_read"
    ]
  }]
}

# OS_ADMIN_PASSWORD is the password for the admin user

$ curl -X PUT "https://localhost:9200/_plugins/_security/api/roles/nsa2_role" \
  -H 'Content-Type: application/json' -d @nsa2-role.json \
  -k -u admin:$OS_ADMIN_PASSWORD

{"status":"CREATED","message":"'nsa2_role' created."}

$ curl -X DELETE "https://localhost:9200/_plugins/_security/api/roles/nsa2_role" \
  -k -u admin:$OS_ADMIN_PASSWORD

{"status":"OK","message":"'nsa2_role' deleted."}

{
  "password": "your-password",
  "backend_roles": ["nsa2_role"],
  "attributes": {
    "created_by": "nsa2",
    "privider": "alexamy"
  }
}

$ curl -X PUT "https://localhost:9200/_plugins/_security/api/internalusers/nsa2" \
  -H 'Content-Type: application/json' -\
  d @nsa2-user.json -k -u admin:$OS_ADMIN_PASSWORD

{"status":"CREATED","message":"'nsa2' created."}

{
  "backend_roles" : [],
  "hosts" : [ "*" ],
  "users" : [ "nsa2" ]
}

$ curl -X PUT "https://localhost:9200/_plugins/_security/api/rolesmapping/nsa2_role" \
  -H 'Content-Type: application/json' \
  -d @nsa2-rolesmapping.json \
  -k -u admin:$OS_ADMIN_PASSWORD

{"status":"CREATED","message":"'nsa2_role' created."}

$ helm show values opensearch/data-prepper > opensearch-data-prepper-values.yaml

#image:
#  tag: ""

#config:

pipelineConfig:
  config:
   otel-logs-pipeline:
     workers: 5
     delay: 10
     source:
       otel_logs_source:
         #port: 21892
         #path: /v1/logs
         ssl: false
     buffer:
       bounded_blocking:
     sink:
       - opensearch:
           hosts: ["https://opensearch-cluster-master:9200"]
           username: "admin"
           password: "your-password"
           insecure: true
#           cert: /etc/opensearch/certs/esnode-ca.pem
#           verify_hostname: false
           index_type: custom
           index: nsa2-%{yyyy.MM.dd}
           #max_retries: 20
           bulk_size: 4

resources:
  limits:
    cpu: 400m
    memory: 1024Mi
  requests:
    cpu: 100m
    memory: 128Mi


# Additional volumes on the output Deployment definition.
volumes:
# - name: opensearch-certs
 - name: esnode-certs
   secret:
     secretName: esnode-certs
     optional: false

# Additional volumeMounts on the output Deployment definition.
volumeMounts:
# - name: opensearch-certs
 - name: esnode-certs
   mountPath: "/etc/opensearch/certs"
   readOnly: true
# - name: foo
#   mountPath: "/etc/foo"
#   readOnly: true

nodeSelector:
#  agentpool: depnodes
  agentpool: depnodes

$ helm -n nsa2 install opensearch-data-prepper opensearch/data-prepper \
  -f nsa2-opensearch-data-prepper-values.yaml

$ helm -n nsa2 uninstall opensearch-data-prepper

# to generate logs with stack trace
$  curl -X POST http://localhost:8080/log/INVALID -d "this is error message"

# to generate normal logs
$ curl http://localhost:8080/error-logs/notify

serviceName: nsa2-opentelemetry-example AND severityText: INFO

log.attributes.exception@stacktrace: *LoggingExampleServiceImpl*