From be8b086a414033b03c53162062230dfd91c822b4 Mon Sep 17 00:00:00 2001
From: Jonas Theis <work@jonastheis.de>
Date: Fri, 7 Feb 2020 16:05:53 +0100
Subject: [PATCH] Remote logging with ELK (#235)
* Add docker compose to run ELK stack with proper configuration loaded
Adapted from https://github.com/deviantony/docker-elk
* Add remotelog plugin and post events via UDP to ELK stack #211
* Fix linter errors
* Remove last octet from host IP
* Adjust to requested changes
* Add background worker
* reorder remote log shutdown order
* update to latest hive.go
* go mod tidy
Co-authored-by: Luca Moser <moser.luca@gmail.com>
---
config.json | 139 +++++++++---------
go.mod | 9 +-
go.sum | 21 +--
main.go | 3 +
packages/shutdown/order.go | 2 +-
plugins/remotelog/plugin.go | 93 ++++++++++++
plugins/remotelog/server/.env | 1 +
.../remotelog/server/config/elasticsearch.yml | 5 +
plugins/remotelog/server/config/kibana.yml | 6 +
.../server/config/logstash/logstash.yml | 4 +
.../config/logstash/pipeline/logstash.conf | 26 ++++
plugins/remotelog/server/docker-compose.yml | 64 ++++++++
12 files changed, 279 insertions(+), 94 deletions(-)
create mode 100644 plugins/remotelog/plugin.go
create mode 100644 plugins/remotelog/server/.env
create mode 100644 plugins/remotelog/server/config/elasticsearch.yml
create mode 100644 plugins/remotelog/server/config/kibana.yml
create mode 100644 plugins/remotelog/server/config/logstash/logstash.yml
create mode 100644 plugins/remotelog/server/config/logstash/pipeline/logstash.conf
create mode 100644 plugins/remotelog/server/docker-compose.yml
diff --git a/config.json b/config.json
index f9eb7c4d..43e4801b 100644
--- a/config.json
+++ b/config.json
@@ -1,68 +1,71 @@
-{
- "analysis": {
- "client": {
- "serverAddress": "ressims.iota.cafe:188"
- },
- "server": {
- "port": 0
- },
- "httpServer": {
- "bindAddress": "0.0.0.0:80"
- }
- },
- "autopeering": {
- "entryNodes": [
- "V8LYtWWcPYYDTTXLeIEFjJEuWlsjDiI0+Pq/Cx9ai6g=@116.202.49.178:14626"
- ],
- "port": 14626
- },
- "dashboard": {
- "bindAddress": "127.0.0.1:8081",
- "dev": false,
- "basic_auth": {
- "enabled": false,
- "username": "goshimmer",
- "password": "goshimmer"
- }
- },
- "database": {
- "directory": "mainnetdb"
- },
- "gossip": {
- "port": 14666
- },
- "graph": {
- "bindAddress": "127.0.0.1:8082",
- "domain": "",
- "networkName": "GoShimmer",
- "socketIOPath": "socket.io-client/dist/socket.io.js",
- "webrootPath": "IOTAtangle/webroot"
- },
- "logger": {
- "level": "info",
- "disableCaller": false,
- "disableStacktrace": false,
- "encoding": "console",
- "outputPaths": [
- "stdout",
- "goshimmer.log"
- ],
- "disableEvents": true
- },
- "network": {
- "bindAddress": "0.0.0.0",
- "externalAddress": "auto"
- },
- "node": {
- "disablePlugins": [],
- "enablePlugins": []
- },
- "webapi": {
- "auth": {
- "password": "goshimmer",
- "privateKey": "",
- "username": "goshimmer"
- },
- "bindAddress": "127.0.0.1:8080"
- }
-}
+{
+ "analysis": {
+ "client": {
+ "serverAddress": "ressims.iota.cafe:188"
+ },
+ "server": {
+ "port": 0
+ },
+ "httpServer": {
+ "bindAddress": "0.0.0.0:80"
+ }
+ },
+ "autopeering": {
+ "entryNodes": [
+ "V8LYtWWcPYYDTTXLeIEFjJEuWlsjDiI0+Pq/Cx9ai6g=@116.202.49.178:14626"
+ ],
+ "port": 14626
+ },
+ "dashboard": {
+ "bindAddress": "127.0.0.1:8081",
+ "dev": false,
+ "basic_auth": {
+ "enabled": false,
+ "username": "goshimmer",
+ "password": "goshimmer"
+ }
+ },
+ "database": {
+ "directory": "mainnetdb"
+ },
+ "gossip": {
+ "port": 14666
+ },
+ "graph": {
+ "bindAddress": "127.0.0.1:8082",
+ "domain": "",
+ "networkName": "GoShimmer",
+ "socketIOPath": "socket.io-client/dist/socket.io.js",
+ "webrootPath": "IOTAtangle/webroot"
+ },
+ "logger": {
+ "level": "info",
+ "disableCaller": false,
+ "disableStacktrace": false,
+ "encoding": "console",
+ "outputPaths": [
+ "stdout",
+ "goshimmer.log"
+ ],
+ "disableEvents": true,
+ "remotelog": {
+ "serverAddress": "remotelog.goshimmer.iota.cafe:5213"
+ }
+ },
+ "network": {
+ "bindAddress": "0.0.0.0",
+ "externalAddress": "auto"
+ },
+ "node": {
+ "disablePlugins": [],
+ "enablePlugins": []
+ },
+ "webapi": {
+ "auth": {
+ "password": "goshimmer",
+ "privateKey": "",
+ "username": "goshimmer"
+ },
+ "bindAddress": "127.0.0.1:8080"
+ }
+}
diff --git a/go.mod b/go.mod
index 8629f723..af0a116b 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,6 @@ require (
github.com/dgraph-io/badger/v2 v2.0.1
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/dgryski/go-farm v0.0.0-20191112170834-c2139c5d712b // indirect
- github.com/gdamore/tcell v1.3.0
github.com/gobuffalo/envy v1.8.1 // indirect
github.com/gobuffalo/logger v1.0.3 // indirect
github.com/gobuffalo/packr/v2 v2.7.1
@@ -14,18 +13,15 @@ require (
github.com/googollee/go-engine.io v1.4.3-0.20190924125625-798118fc0dd2
github.com/googollee/go-socket.io v1.4.3-0.20191204093753-683f8725b6d0
github.com/gorilla/websocket v1.4.1
- github.com/iotaledger/hive.go v0.0.0-20200121213505-28904d5f037c
+ github.com/iotaledger/hive.go v0.0.0-20200207144536-27b18f10f09e
github.com/iotaledger/iota.go v1.0.0-beta.14
github.com/labstack/echo v3.3.10+incompatible
github.com/labstack/gommon v0.3.0 // indirect
- github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
github.com/magiconair/properties v1.8.1
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mattn/go-isatty v0.0.11 // indirect
- github.com/mattn/go-runewidth v0.0.7 // indirect
github.com/pelletier/go-toml v1.6.0 // indirect
github.com/pkg/errors v0.9.1
- github.com/rivo/tview v0.0.0-20191229165609-1ee8d9874dcf
github.com/rogpeppe/go-internal v1.5.2 // indirect
github.com/spf13/afero v1.2.2 // indirect
github.com/spf13/cast v1.3.1 // indirect
@@ -37,9 +33,10 @@ require (
github.com/valyala/fasttemplate v1.1.0 // indirect
go.uber.org/atomic v1.5.1
go.uber.org/zap v1.13.0
- golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d
+ golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d // indirect
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553
golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 // indirect
+ golang.org/x/text v0.3.2 // indirect
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 // indirect
gopkg.in/ini.v1 v1.51.1 // indirect
gopkg.in/yaml.v2 v2.2.7 // indirect
diff --git a/go.sum b/go.sum
index 769edd98..a943eb06 100644
--- a/go.sum
+++ b/go.sum
@@ -9,7 +9,6 @@ git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGy
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
@@ -54,10 +53,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
-github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
-github.com/gdamore/tcell v1.3.0 h1:r35w0JBADPZCVQijYebl6YMWWtHRqVEGt7kL2eBADRM=
-github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -128,8 +123,8 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/iotaledger/hive.go v0.0.0-20200121213505-28904d5f037c h1:3T0MLyZIL74kYLqVrmv1xQlwE2ktS1IO8kjM+NyXEMU=
-github.com/iotaledger/hive.go v0.0.0-20200121213505-28904d5f037c/go.mod h1:S+v90R3u4Rqe4VoOg4DhiZrAKlKZhz2UFKuK/Neqa2o=
+github.com/iotaledger/hive.go v0.0.0-20200207144536-27b18f10f09e h1:eM0/mEG3ZPflZRJxOn0cc+/8ZZIpirEAyhzF4rzTKjM=
+github.com/iotaledger/hive.go v0.0.0-20200207144536-27b18f10f09e/go.mod h1:wj3bFHlcX0NiEOWu5+WOg/MI/5N7PKCFnyaziaylB64=
github.com/iotaledger/iota.go v1.0.0-beta.9/go.mod h1:F6WBmYd98mVjAmmPVYhnxg8NNIWCjjH8VWT9qvv3Rc8=
github.com/iotaledger/iota.go v1.0.0-beta.14 h1:Oeb28MfBuJEeXcGrLhTCJFtbsnc8y1u7xidsAmiOD5A=
github.com/iotaledger/iota.go v1.0.0-beta.14/go.mod h1:F6WBmYd98mVjAmmPVYhnxg8NNIWCjjH8VWT9qvv3Rc8=
@@ -156,9 +151,6 @@ github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8
github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
-github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
-github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac=
-github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -169,9 +161,6 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
-github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -213,10 +202,6 @@ github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rivo/tview v0.0.0-20191229165609-1ee8d9874dcf h1:rh73WIukDlFIRqk1lk76or+LExEjTci2789EDvDD67U=
-github.com/rivo/tview v0.0.0-20191229165609-1ee8d9874dcf/go.mod h1:/rBeY22VG2QprWnEqG57IBC8biVu3i0DOIjRLc9I8H0=
-github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY=
-github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -383,10 +368,8 @@ golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191018095205-727590c5006e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 h1:1/DFK4b7JH8DmkqhUk48onnSfrPzImPoVxuomtbT2nk=
golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/main.go b/main.go
index 18fe0554..2f947172 100644
--- a/main.go
+++ b/main.go
@@ -12,6 +12,7 @@ import (
"github.com/iotaledger/goshimmer/plugins/gracefulshutdown"
"github.com/iotaledger/goshimmer/plugins/graph"
"github.com/iotaledger/goshimmer/plugins/metrics"
+ "github.com/iotaledger/goshimmer/plugins/remotelog"
"github.com/iotaledger/goshimmer/plugins/spa"
"github.com/iotaledger/goshimmer/plugins/tangle"
"github.com/iotaledger/goshimmer/plugins/tipselection"
@@ -35,6 +36,8 @@ func main() {
node.Run(
node.Plugins(
cli.PLUGIN,
+ remotelog.PLUGIN,
+
autopeering.PLUGIN,
gossip.PLUGIN,
tangle.PLUGIN,
diff --git a/packages/shutdown/order.go b/packages/shutdown/order.go
index 06594b8c..b8347d1e 100644
--- a/packages/shutdown/order.go
+++ b/packages/shutdown/order.go
@@ -2,6 +2,7 @@ package shutdown
const (
ShutdownPriorityTangle = iota
+ ShutdownPriorityRemoteLog
ShutdownPrioritySolidifier
ShutdownPriorityBundleProcessor
ShutdownPriorityAnalysis
@@ -13,5 +14,4 @@ const (
ShutdownPriorityTangleSpammer
ShutdownPrioritySPA
ShutdownPriorityBadgerGarbageCollection
- ShutdownPriorityStatusScreen
)
diff --git a/plugins/remotelog/plugin.go b/plugins/remotelog/plugin.go
new file mode 100644
index 00000000..0ca02c50
--- /dev/null
+++ b/plugins/remotelog/plugin.go
@@ -0,0 +1,93 @@
+// remotelog is a plugin that enables log messages being sent via UDP to a central ELK stack for debugging.
+// It is disabled by default and when enabled, additionally, logger.disableEvents=false in config.json needs to be set.
+// The destination can be set via logger.remotelog.serverAddress.
+// All events according to logger.level in config.json are sent.
+package remotelog
+
+import (
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "net"
+ "runtime"
+ "time"
+
+ "github.com/iotaledger/goshimmer/packages/parameter"
+ "github.com/iotaledger/goshimmer/packages/shutdown"
+ "github.com/iotaledger/goshimmer/plugins/autopeering/local"
+ "github.com/iotaledger/hive.go/daemon"
+ "github.com/iotaledger/hive.go/events"
+ "github.com/iotaledger/hive.go/logger"
+ "github.com/iotaledger/hive.go/node"
+ "github.com/iotaledger/hive.go/workerpool"
+)
+
+type logMessage struct {
+ NodeId string `json:"nodeId"`
+ Level string `json:"level"`
+ Name string `json:"name"`
+ Msg string `json:"msg"`
+ Timestamp time.Time `json:"timestamp"`
+}
+
+const (
+ CFG_SERVER_ADDRESS = "logger.remotelog.serverAddress"
+ CFG_DISABLE_EVENTS = "logger.disableEvents"
+ PLUGIN_NAME = "RemoteLog"
+)
+
+var (
+ PLUGIN = node.NewPlugin(PLUGIN_NAME, node.Disabled, configure, run)
+ log *logger.Logger
+ conn net.Conn
+ myID string
+ workerPool *workerpool.WorkerPool
+)
+
+func configure(plugin *node.Plugin) {
+ log = logger.NewLogger(PLUGIN_NAME)
+
+ if parameter.NodeConfig.GetBool(CFG_DISABLE_EVENTS) {
+ log.Fatalf("%s in config.json needs to be false so that events can be captured!", CFG_DISABLE_EVENTS)
+ return
+ }
+
+ c, err := net.Dial("udp", parameter.NodeConfig.GetString(CFG_SERVER_ADDRESS))
+ if err != nil {
+ log.Fatalf("Could not create UDP socket to '%s'. %v", parameter.NodeConfig.GetString(CFG_SERVER_ADDRESS), err)
+ return
+ }
+ conn = c
+
+ if local.GetInstance() != nil {
+ myID = hex.EncodeToString(local.GetInstance().ID().Bytes())
+ }
+
+ workerPool = workerpool.New(func(task workerpool.Task) {
+ sendLogMsg(task.Param(0).(logger.Level), task.Param(1).(string), task.Param(2).(string))
+
+ task.Return(nil)
+ }, workerpool.WorkerCount(runtime.NumCPU()), workerpool.QueueSize(1000))
+}
+
+func run(plugin *node.Plugin) {
+ logEvent := events.NewClosure(func(level logger.Level, name string, msg string) {
+ workerPool.TrySubmit(level, name, msg)
+ })
+
+ daemon.BackgroundWorker(PLUGIN_NAME, func(shutdownSignal <-chan struct{}) {
+ logger.Events.AnyMsg.Attach(logEvent)
+ workerPool.Start()
+ <-shutdownSignal
+ log.Infof("Stopping %s ...", PLUGIN_NAME)
+ logger.Events.AnyMsg.Detach(logEvent)
+ workerPool.Stop()
+ log.Infof("Stopping %s ... done", PLUGIN_NAME)
+ }, shutdown.ShutdownPriorityRemoteLog)
+}
+
+func sendLogMsg(level logger.Level, name string, msg string) {
+ m := logMessage{myID, level.CapitalString(), name, msg, time.Now()}
+ b, _ := json.Marshal(m)
+ fmt.Fprint(conn, string(b))
+}
diff --git a/plugins/remotelog/server/.env b/plugins/remotelog/server/.env
new file mode 100644
index 00000000..68aaea9a
--- /dev/null
+++ b/plugins/remotelog/server/.env
@@ -0,0 +1 @@
+ELK_VERSION=7.5.2
\ No newline at end of file
diff --git a/plugins/remotelog/server/config/elasticsearch.yml b/plugins/remotelog/server/config/elasticsearch.yml
new file mode 100644
index 00000000..1d27fb92
--- /dev/null
+++ b/plugins/remotelog/server/config/elasticsearch.yml
@@ -0,0 +1,5 @@
+## Default Elasticsearch configuration from Elasticsearch base image.
+## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml
+#
+cluster.name: "docker-cluster"
+network.host: 0.0.0.0
\ No newline at end of file
diff --git a/plugins/remotelog/server/config/kibana.yml b/plugins/remotelog/server/config/kibana.yml
new file mode 100644
index 00000000..6d7ef28a
--- /dev/null
+++ b/plugins/remotelog/server/config/kibana.yml
@@ -0,0 +1,6 @@
+## Default Kibana configuration from Kibana base image.
+## https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.js
+#
+server.name: kibana
+server.host: "0"
+elasticsearch.hosts: [ "http://elasticsearch:9200" ]
\ No newline at end of file
diff --git a/plugins/remotelog/server/config/logstash/logstash.yml b/plugins/remotelog/server/config/logstash/logstash.yml
new file mode 100644
index 00000000..705bcc94
--- /dev/null
+++ b/plugins/remotelog/server/config/logstash/logstash.yml
@@ -0,0 +1,4 @@
+## Default Logstash configuration from Logstash base image.
+## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml
+#
+http.host: "0.0.0.0"
\ No newline at end of file
diff --git a/plugins/remotelog/server/config/logstash/pipeline/logstash.conf b/plugins/remotelog/server/config/logstash/pipeline/logstash.conf
new file mode 100644
index 00000000..fcc2ccbc
--- /dev/null
+++ b/plugins/remotelog/server/config/logstash/pipeline/logstash.conf
@@ -0,0 +1,26 @@
+input {
+ udp {
+ port => 5213
+ }
+}
+
+filter {
+ mutate {
+ split => ["host", "."]
+ add_field => { "shortHostname" => "%{[host][0]}.%{[host][1]}.%{[host][2]}.x" }
+ }
+ mutate {
+ rename => ["shortHostname", "host" ]
+ }
+
+ json {
+ source => "message"
+ target => "log"
+ }
+}
+
+output {
+ elasticsearch {
+ hosts => "elasticsearch:9200"
+ }
+}
\ No newline at end of file
diff --git a/plugins/remotelog/server/docker-compose.yml b/plugins/remotelog/server/docker-compose.yml
new file mode 100644
index 00000000..f3c9db88
--- /dev/null
+++ b/plugins/remotelog/server/docker-compose.yml
@@ -0,0 +1,64 @@
+version: '3.7'
+
+services:
+ elasticsearch:
+ container_name: elasticsearch
+ image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
+ volumes:
+ - type: bind
+ source: ./config/elasticsearch.yml
+ target: /usr/share/elasticsearch/config/elasticsearch.yml
+ read_only: true
+ - type: volume
+ source: elasticsearch
+ target: /usr/share/elasticsearch/data
+ environment:
+ ES_JAVA_OPTS: "-Xmx256m -Xms256m"
+ # Use single node discovery in order to disable production mode and avoid bootstrap checks
+ # see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
+ discovery.type: single-node
+ networks:
+ - elk
+
+ logstash:
+ container_name: logstash
+ image: docker.elastic.co/logstash/logstash:${ELK_VERSION}
+ volumes:
+ - type: bind
+ source: ./config/logstash/logstash.yml
+ target: /usr/share/logstash/config/logstash.yml
+ read_only: true
+ - type: bind
+ source: ./config/logstash/pipeline
+ target: /usr/share/logstash/pipeline
+ read_only: true
+ ports:
+ - "5213:5213/udp"
+ environment:
+ LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+ networks:
+ - elk
+ depends_on:
+ - elasticsearch
+
+ kibana:
+ container_name: kibana
+ image: docker.elastic.co/kibana/kibana:${ELK_VERSION}
+ volumes:
+ - type: bind
+ source: ./config/kibana.yml
+ target: /usr/share/kibana/config/kibana.yml
+ read_only: true
+ ports:
+ - "5601:5601"
+ networks:
+ - elk
+ depends_on:
+ - elasticsearch
+
+networks:
+ elk:
+ driver: bridge
+
+volumes:
+ elasticsearch:
\ No newline at end of file
--
GitLab