Skip to content
Snippets Groups Projects
Commit 3aad404a authored by Helmut's avatar Helmut
Browse files

Updating README.md - Minor Changes

parent 5fdd68a3
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
README.md
+ 1
− 2
View file @ 3aad404a
...@@ -11,7 +11,7 @@ NGinx Service definiert wird. Nebenbei demonstriert der NGINX Stack das Sharing ...@@ -11,7 +11,7 @@ NGinx Service definiert wird. Nebenbei demonstriert der NGINX Stack das Sharing
* [Docker: Share Compose File Configurations](https://docs.docker.com/compose/extends/) * [Docker: Share Compose File Configurations](https://docs.docker.com/compose/extends/)
## Installation ## Installation
Voraussetzung: Einloggen im Netz der TH Nürnberg bzw. eine VPN Verbindung zu öffnen. Voraussetzung: Einloggen im Netz der TH Nürnberg bzw. eine VPN Verbindung öffnen.
### Lokaler Build ### Lokaler Build
Das Projekt kann mit Docker selbst gebaut werden. Es ist nur docker-compose erforderlich. Alle nötigen Build Tools werden bereitgestellt und es muss keine Java, NodeJS oder Angular Installation lokal vorhanden sein. Das Projekt kann mit Docker selbst gebaut werden. Es ist nur docker-compose erforderlich. Alle nötigen Build Tools werden bereitgestellt und es muss keine Java, NodeJS oder Angular Installation lokal vorhanden sein.
...@@ -22,7 +22,6 @@ Mit dem nachfolgenden docker-compose build werden der NGinx und der Fluentd cont ...@@ -22,7 +22,6 @@ Mit dem nachfolgenden docker-compose build werden der NGinx und der Fluentd cont
## Starten der Demo ## Starten der Demo
#### Starten des Elasticsearch Stacks #### Starten des Elasticsearch Stacks
Zum Starten folgenden Befehl ausführen:
docker stack deploy --compose-file docker-compose-es-stack.yml ElastikSearchStack docker stack deploy --compose-file docker-compose-es-stack.yml ElastikSearchStack
......
docker-compose-es-stack.yml 0 → 100644
+ 45
− 0
View file @ 3aad404a
version: "3.7"
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.8.0
container_name: elasticsearch
environment:
- xpack.security.enabled=false
- discovery.type=single-node
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536
hard: 65536
cap_add:
- IPC_LOCK
volumes:
- elasticsearch-data:/usr/share/elasticsearch/data
#ports:
# - 9200:9200
# - 9300:9300
networks:
- logger-network
kibana:
container_name: kibana
image: docker.elastic.co/kibana/kibana:7.8.0
environment:
- ELASTICSEARCH_HOSTS=http://elasticsearch:9200
ports:
- 5601:5601
networks:
- logger-network
depends_on:
- elasticsearch
networks:
logger-network:
external: true
driver: overlay
volumes:
elasticsearch-data:
driver: local
docker-compose.yml 0 → 100644
+ 22
− 0
View file @ 3aad404a
version: "3.7"
services:
logging:
build: myfluentd
restart: unless-stopped
image: myfluentd
volumes:
- ./myfluentd/conf:/fluentd/etc
- ./log:/log
ports:
- "24224:24224"
- "24224:24224/udp"
networks:
- logger-network
networks:
logger-network:
external: true
driver: overlay
myfluentd/Dockerfile 0 → 100644
+ 18
− 0
View file @ 3aad404a
FROM fluent/fluentd:v1.10
# Use root account to use fluent-gem
USER root
# below RUN includes plugin as examples elasticsearch is not required
# you may customize including plugins as you wish
RUN id && echo " --->" Starting Grep Pluign Installation \
&& gem install fluent-plugin-grep \
&& gem install fluent-plugin-elasticsearch \
&& gem install fluent-plugin-rewrite-tag-filter \
&& gem sources --clear-all
COPY conf/fluent.conf /fluentd/etc/
# COPY entrypoint.sh /bin/
USER fluent
# RUN id && echo " --> Starting fluentd"
myfluentd/conf/fluent.conf 0 → 100644
+ 312
− 0
View file @ 3aad404a
<system>
log_level debug
</system>
# All docker images should be armed with tag option in the logging section
#logging:
# driver: "fluentd"
# options:
# tag: "nginx.logs"
# fluentd-address: "localhost:24224"
# fluentd-async-connect: "true"
<source>
@type forward
port 24224
bind 0.0.0.0
#tag nginx_test
</source>
#
# For debugging - enbable tag nginx_test in above source command
#
#<match nginx_test>
# @type copy
# <store>
# @type stdout
# </store>
#</match>
<match database.logs>
@type copy
<store>
@type file
path /log/${tag}
# Read configuring buffers read: https://docs.fluentd.org/configuration/buffer-section#buffering-parameters
<buffer tag,time>
@type file
path /log/buffer/database
# Timeframe for collecting chunks before flushing
# For production use 3600 or higher
# Setup: Write new Logfiles every hour
timekey 60
# Output plugin will write chunks after timekey_wait seconds later after timekey expiration
timekey_wait 10
# If gzip is set, Fluentd compresses data records before writing to buffer chunks.
compress gzip
</buffer>
</store>
<store>
@type stdout
</store>
<store>
@type elasticsearch
port 9200
host elasticsearch
logstash_format true
suppress_type_name true
logstash_prefix database
logstash_dateformat %Y%m%d
include_tag_key true
<buffer>
flush_interval 1
</buffer>
</store>
</match>
<filter es.logs>
@type parser
key_name log
reserve_data true
remove_key_name_field true
# hash_value_field parsed
<parse>
@type json
</parse>
</filter>
<match es.logs>
@type copy
<store>
@type file
path /log/${tag}
# Read configuring buffers read: https://docs.fluentd.org/configuration/buffer-section#buffering-parameters
<buffer tag,time>
@type file
path /log/buffer/es_stack
# Timeframe for collecting chunks before flushing
# Setup: Flush chunks to Logfiles every minute
timekey 60
# Output plugin will write chunks after timekey_wait seconds later after timekey expiration
timekey_wait 10
# If gzip is set, Fluentd compresses data records before writing to buffer chunks.
compress gzip
</buffer>
</store>
<store>
@type stdout
</store>
<store>
@type elasticsearch
port 9200
host elasticsearch
logstash_format true
suppress_type_name true
logstash_prefix es_stack
logstash_dateformat %Y%m%d
include_tag_key true
<buffer>
flush_interval 1
</buffer>
</store>
</match>
<filter payara.logs>
@type parser
key_name log
reserve_data true
remove_key_name_field true
# hash_value_field parsed
<parse>
@type json
</parse>
</filter>
<match payara.logs>
@type copy
<store>
@type file
path /log/${tag}
# Read configuring buffers read: https://docs.fluentd.org/configuration/buffer-section#buffering-parameters
<buffer tag,time>
@type file
path /log/buffer/payara
# Timeframe for collecting chunks before flushing
# For production use 3600 or higher
# Setup: Write new Logfiles every hour
timekey 60
# Output plugin will write chunks after timekey_wait seconds later after timekey expiration
timekey_wait 10
# If gzip is set, Fluentd compresses data records before writing to buffer chunks.
compress gzip
</buffer>
</store>
<store>
@type stdout
</store>
<store>
@type elasticsearch
port 9200
host elasticsearch
logstash_format true
suppress_type_name true
logstash_prefix fluentd_payara
logstash_dateformat %Y%m%d
include_tag_key true
<buffer>
flush_interval 1
</buffer>
</store>
</match>
#
# Filter Nginx Log Entries by using rewrite_tag_filter
# Log record starts with an Date-String "2020/08/17 12:23:35" -> Type: Nginx Error log -> Tag: error.nginx.logs
# Log record starts with a String "\"ngx_time_local\": -> Log Type: Nginx Access Log -> Tag: access.nginx.logs
#
<match nginx.logs>
@type rewrite_tag_filter
<rule>
#
# Nginx Error logs cannot be configured to send data in JSON format
# Typical Nginx Error log record: "log":"2020/08/17 12:23:35 [error] 21#21: *14 open() \"/usr/share/nginx/html/xxx\" failed (2: No such file or directory)" ..
# In short when Log records starts wiht a Date the record will be tagged as a Nginx Errorlog
#
key log
pattern /^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}/
tag error.nginx.logs
</rule>
<rule>
#
# Assuming Nginx is sending JSON formatted Access Logs:
# - Nginx Logformat is configured in nginx.conf:
# log_format json_combined escape=json
# '"ngx_time_local":"$time_local",
# ..
# Typical Nginx Access log record: "log":"\"ngx_time_local\":\"28/Aug/2020:08:59:36 +0000\", \"remote_user\":\"\", \"remote_addr\":\"172.18.0.1\", \"request\":\"GET /xxx HTTP/1.1\", ..
# Log Records starting with a JSON escapced ngx_time_local string will be taggesd as a Nginx Accesslog
#
key log
#pattern /^\"ngx_time_local\":/
pattern /"ngx_time_local"/
tag access.nginx.logs
</rule>
</match>
<filter error.nginx.logs>
@type parser
#
# As Nginx Errorlog are not formatted in JSON Format a Regex Formatter will be used
#
key_name log
format /^(?<time>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?<log_level>\w+)\] (?<pid>\d+).(?<tid>\d+): (?<message>.*)$/
</filter>
<match error.nginx.logs>
@type copy
<store>
@type file
path /log/${tag}
# Read configuring buffers read: https://docs.fluentd.org/configuration/buffer-section#buffering-parameters
<buffer tag,time>
@type file
path /log/buffer/error_nginx
# Timeframe for collecting chunks before flushing
# For production use 3600 or higher
# Setup: Write new Logfiles every hour
timekey 60
# Output plugin will write chunks after timekey_wait seconds later after timekey expiration
timekey_wait 10
# If gzip is set, Fluentd compresses data records before writing to buffer chunks.
compress gzip
</buffer>
</store>
<store>
@type stdout
</store>
<store>
@type elasticsearch
port 9200
host elasticsearch
logstash_format true
suppress_type_name true
logstash_prefix fluentd_nginx_error
logstash_dateformat %Y%m%d
include_tag_key true
<buffer>
flush_interval 1
</buffer>
</store>
</match>
#
# This filter is not needed anymore as we configured Nginx to send Accesslogs in JSON format
# If we disable this we need to re-enable this filter
#
#<filter access.nginx.logs>
# @type parser
# key_name log
# format nginx
#</filter>
#
#
#
# reserve_data : Keeps the original key-value pair in the parsed result.
#
# remove_key_name_field : Removes key_name field when parsing is succeeded.
# This removes the log: entry from input data - in short removes the nested JSON parsing
# input data: {"key":"value","log":"{\"user\":1,\"num\":2}"}
# output data: {"key":"value","user":1,"num":2}
# This leads to simple parsing in our elasticsearch instance and we don't need to pre-create
# an ES index with "type":"nested" for "log" field
#
<filter access.nginx.logs>
@type parser
key_name log
reserve_data true
remove_key_name_field true
# hash_value_field parsed
<parse>
@type json
</parse>
</filter>
<match access.nginx.logs>
@type copy
<store>
@type file
path /log/${tag}
# Read configuring buffers read: https://docs.fluentd.org/configuration/buffer-section#buffering-parameters
append true
<buffer tag,time>
@type file
path /log/buffer/access_nginx
# Timeframe for collecting chunks before flushing
# Setup: Write new Logfiles every 10 minutes
timekey 600
# Output plugin will write chunks after timekey_wait seconds later after timekey expiration
timekey_wait 10
# If gzip is set, Fluentd compresses data records before writing to buffer chunks.
compress gzip
</buffer>
</store>
<store>
@type stdout
</store>
<store>
@type elasticsearch
port 9200
host elasticsearch
logstash_format true
suppress_type_name true
logstash_prefix fluentd_nginx_log
logstash_dateformat %Y%m%d
include_tag_key true
<buffer>
flush_interval 1
</buffer>
</store>
</match>
mynginx/Dockerfile 0 → 100644
+ 4
− 0
View file @ 3aad404a
FROM nginx
RUN chmod +w /etc/nginx/nginx.conf
COPY nginx.conf /etc/nginx/nginx.conf
RUN cat /etc/nginx/nginx.conf
mynginx/nginx.conf 0 → 100644
+ 47
− 0
View file @ 3aad404a
user nginx;
worker_processes auto;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
# resolver 127.0.0.11 ipv6=off;
# resolver_timeout 10s;
# log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
log_format json_combined escape=json '{'
'"ngx_time_local":"$time_local", '
'"ngx_remote_user":"$remote_user", '
'"ngx_remote_addr":"$remote_addr", '
'"ngx_request":"$request", '
'"ngx_status": "$status", '
'"ngx_body_bytes_sent":"$body_bytes_sent", '
'"ngx_request_time":"$request_time", '
'"ngx_http_referrer":"$http_referer", '
'"ngx_http_user_agent":"$http_user_agent" '
'}';
# access_log syslog:server=logging:22224,tag=nginx_access json_combined;
access_log /var/log/nginx/access.log json_combined;
# error_log syslog:server=logging:21224,tag=nginx_error,severity=info;
error_log /var/log/nginx/error.log info;
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
include /etc/nginx/conf.d/*.conf;
}
\ No newline at end of file
triggerLogEntry.bat 0 → 100644
+ 1
− 0
View file @ 3aad404a
docker run --log-driver=fluentd --log-opt tag="nginx.logs" --log-opt fluentd-address="127.0.0.1:24224" ubuntu echo "{\"ngx_time_local\":\"28/Aug/2020:09:42:10 +0000\", \"ngx_remote_user\":\"Helmut\", \"ngx_remote_addr\":\"172.18.0.1\", \"ngx_request\":\"GET /Page_NOT_found HTTP/1.1\", \"ngx_status\":\"404\" }"
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment