Data Prepper 는 OpenSearch 프로젝트의 수집 · 변환 파이프라인이다 (Logstash 역할). 현행 2.16.0 이고 GitHub 릴리스(opensearch-project/data-prepper) 의 tar 를 받아 /opt/data-prepper 에 푼다. 아래는 2025년 2.x 에서 테스트한 설정이다.
$DATA_PREPPER_HOME/config/data-prepper-config.yaml# Data Prepper Configuration File
# SSL 설정 (true: 활성화, false: 비활성화)
ssl: false
# SSL 키스토어 설정 (SSL 활성화 시 사용)
#key_store_file_path: "/path/to/keystore.p12"
#key_store_password: "your-keystore-password"
#private_key_password: "your-private-key-password"
# Data Prepper 서버가 바인딩할 포트 번호
server_port: 4900
# 메트릭 레지스트리 설정 (예: Prometheus)
metric_registries:
- Prometheus
# 메트릭 태그 설정 (옵션)
metric_tags:
environment: "production"
region: "us-east-1"
# 프로세서 종료 대기 시간 (초)
processor_shutdown_timeout: 30
# 싱크 종료 대기 시간 (초)
sink_shutdown_timeout: 30
# 파이프라인 종료 설정 (옵션)
pipeline_shutdown:
timeout_seconds: 30
wait_for_termination: true
# 플러그인 확장 설정 (옵션)
extensions:
aws:
secrets:
my-secret-config:
secret_id: "secret-id"
region: "us-east-1"
# 피어 포워더 설정 (분산 환경에서 데이터 전달)
peer_forwarder:
enabled: false
# HTTP 인증 설정 (옵션)
authentication:
enabled: false
username: "admin"
password: "${REDACTED}"
# 이벤트 처리 관련 설정 (옵션)
eventConfiguration:
some_event_setting: true
# 회로 차단기 설정 예시 (옵션)
circuit_breakers:
enabled: true
failure_threshold: 5
recovery_timeout_seconds: 60
data-prepper:
serverPort: 4900
ssl: false
pipeline:
worker_threads: 4
delay: 1000
read_batch_delay: 100
process_workers: 2
/opt/data-prepper/pipelines/pipeline.yamlopensearch-log-pipeline:
workers: 2
delay: 3000
source:
file:
path:
- /var/log/opensearch/*.log
start_at: beginning
record_offset: true
format: plain # 'file_format' 대신 'format' 사용
poll_interval: 1000
processor:
- grok:
match:
log:
- "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:loglevel}\\s*\\]\\[%{DATA:class}\\s*\\]\\s*\\[%{DATA:node}\\]\\s*%{GREEDYDATA:message}"
- date:
from_time_received: false
destination: "@timestamp"
source: "timestamp"
formats:
- "yyyy-MM-dd'T'HH:mm:ss,SSS"
sink:
- opensearch:
hosts:
- "https://localhost:9200"
username: "admin"
password: "${REDACTED}"
index: "opensearch-logs-%{+yyyy.MM.dd}"
index_type: custom
ssl:
insecure: true
/opt/data-prepper/bin/data-prepper \
--config /opt/data-prepper/config/data-prepper-config.yaml \
--pipeline /opt/data-prepper/pipelines/pipeline.yaml
[Unit]
Description=OpenSearch Data Prepper Service
After=network.target
[Service]
Type=simple
User=opensearch
Group=opensearch
ExecStart=/opt/opensearch-data-prepper/bin/data-prepper
Restart=on-failure
RestartSec=5
StandardOutput=syslog
StandardError=syslog
SyslogIdentifier=data-prepper
[Install]
WantedBy=multi-user.target
curl -u admin:PASSWORD -k "http://localhost:9200/_cat/indices?v"
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
green open opensearch-logs-+2025.10.22 G58_iwMEQ4aXsUWIgCOuQA 1 1 275 0 93.1kb 46.5kb
green open top_queries-2025.10.22-55842 rI4TBkGcRtyyVK1iKI468A 1 1 12 0 189.2kb 76.5kb
green open .opensearch-observability x-krnUOZQFSp5ivMS4_q4Q 1 1 0 0 416b 208b
green open .plugins-ml-config dgpba7ikRMi5Pm4kPh_9Sw 1 1 1 0 8.1kb 4kb
green open .ql-datasources FS7URgvdSrO4HRKiyq4cug 1 1 0 0 416b 208b
green open security-auditlog-2025.10.21 iYtG_RoUS5SVImhJlnfLkw 1 1 62 0 227.3kb 113.6kb
green open .kibana_92668751_admin_1 KaiY1alUQKmptZSn5_he1g 1 1 2 0 20.1kb 10kb
green open security-auditlog-2025.10.22 q5m3dlUUSKykKyvMev2ESg 1 1 17 0 299.7kb 149.8kb
green open top_queries-2025.10.21-55841 uHO16WSBQMW0feBStVXq_g 1 1 67 4 355.6kb 177.8kb
green open .opendistro_security Y2cwN2XoSSes-6XooAoQtg 1 1 9 0 161.7kb 80.8kb
green open .kibana_1 MRvsEEmIRnqtUwX1dt0PTA 1 1 0 0 416b 208b