From 3de819a0e979e93d8e38cc78fdda7deb05817056 Mon Sep 17 00:00:00 2001 From: James Read Date: Fri, 12 Apr 2024 00:23:40 +0100 Subject: [PATCH] feature: add prometheus metrics (#268) * feature: add prometheus metrics * bugfix: ixed issue with path --- cmd/OliveTin/main.go | 18 +++---------- go.mod | 7 ++++- go.sum | 12 +++++++++ internal/config/config.go | 10 +++++++ internal/config/config_reloader.go | 37 ++++++++++++++++++++++++++ internal/entityfiles/entityfiles.go | 4 +-- internal/executor/executor.go | 12 +++++++++ internal/grpcapi/grpcApi.go | 2 +- internal/httpservers/httpServer.go | 4 +++ internal/httpservers/prometheus.go | 19 +++++++++++++ internal/httpservers/singleFrontend.go | 10 +++++++ internal/installationinfo/init.go | 14 +++++----- internal/stringvariables/entities.go | 4 +-- internal/stringvariables/map.go | 28 ++++++++++++++++--- 14 files changed, 150 insertions(+), 31 deletions(-) create mode 100644 internal/config/config_reloader.go create mode 100644 internal/httpservers/prometheus.go diff --git a/cmd/OliveTin/main.go b/cmd/OliveTin/main.go index d097e6e..e59b9a4 100644 --- a/cmd/OliveTin/main.go +++ b/cmd/OliveTin/main.go @@ -8,6 +8,7 @@ import ( "github.com/OliveTin/OliveTin/internal/entityfiles" "github.com/OliveTin/OliveTin/internal/executor" grpcapi "github.com/OliveTin/OliveTin/internal/grpcapi" + "github.com/OliveTin/OliveTin/internal/httpservers" "github.com/OliveTin/OliveTin/internal/installationinfo" "github.com/OliveTin/OliveTin/internal/oncalendarfile" "github.com/OliveTin/OliveTin/internal/oncron" @@ -16,13 +17,10 @@ import ( updatecheck "github.com/OliveTin/OliveTin/internal/updatecheck" "github.com/OliveTin/OliveTin/internal/websocket" - "github.com/OliveTin/OliveTin/internal/httpservers" - config "github.com/OliveTin/OliveTin/internal/config" "github.com/fsnotify/fsnotify" "github.com/spf13/viper" "os" - "path" ) var ( @@ -98,11 +96,11 @@ func initViperConfig(configDir string) { if e.Op == fsnotify.Write { log.Info("Config file changed:", e.String()) - reloadConfig() + config.Reload(cfg) } }) - reloadConfig() + config.Reload(cfg) } func initInstallationInfo() { @@ -130,16 +128,6 @@ func warnIfPuidGuid() { } } -func reloadConfig() { - if err := viper.UnmarshalExact(&cfg); err != nil { - log.Errorf("Config unmarshal error %+v", err) - os.Exit(1) - } - - cfg.SetDir(path.Dir(viper.ConfigFileUsed())) - cfg.Sanitize() -} - func main() { log.WithFields(log.Fields{ "configDir": cfg.GetDir(), diff --git a/go.mod b/go.mod index da75fa8..bec5b3b 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.4.1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 + github.com/prometheus/client_golang v1.19.0 github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.15.0 @@ -33,10 +34,12 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/bufbuild/protocompile v0.9.0 // indirect github.com/bufbuild/protovalidate-go v0.6.0 // indirect github.com/bufbuild/protoyaml-go v0.1.8 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/cristalhq/acmd v0.11.2 // indirect @@ -87,11 +90,13 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pkg/profile v1.7.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/quasilyte/go-ruleguard v0.4.2 // indirect github.com/quasilyte/gogrep v0.5.0 // indirect github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/rs/cors v1.10.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/afero v1.9.3 // indirect diff --git a/go.sum b/go.sum index f05d847..ab2846b 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bufbuild/buf v1.30.1 h1:QFtanwsXodoGFAwzXFXGXpzBkb7N2u8ZDyA3jWB4Pbs= github.com/bufbuild/buf v1.30.1/go.mod h1:7W8DJnj76wQa55EA3z2CmDxS0/nsHh8FqtE00dyDAdA= github.com/bufbuild/protocompile v0.9.0 h1:DI8qLG5PEO0Mu1Oj51YFPqtx6I3qYXUAhJVJ/IzAVl0= @@ -61,6 +63,8 @@ github.com/bufbuild/protoyaml-go v0.1.8/go.mod h1:R8vE2+l49bSiIExP4VJpxOXleHE+FD github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= @@ -304,7 +308,15 @@ github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDj github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/quasilyte/go-ruleguard v0.4.2 h1:htXcXDK6/rO12kiTHKfHuqR4kr3Y4M0J0rOL6CH/BYs= github.com/quasilyte/go-ruleguard v0.4.2/go.mod h1:GJLgqsLeo4qgavUoL8JeGFNS7qcisx3awV/w9eWTmNI= github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo= diff --git a/internal/config/config.go b/internal/config/config.go index 329c209..2fe1d91 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -64,6 +64,11 @@ type AccessControlList struct { Permissions PermissionsList } +type PrometheusConfig struct { + Enabled bool + DefaultGoMetrics bool +} + // Config is the global config used through the whole app. type Config struct { UseSingleHTTPFrontend bool @@ -72,6 +77,7 @@ type Config struct { ListenAddressWebUI string ListenAddressRestActions string ListenAddressGrpcActions string + ListenAddressPrometheus string ExternalRestAddress string LogLevel string LogDebugOptions LogDebugOptions @@ -103,6 +109,7 @@ type Config struct { InsecureAllowDumpVars bool InsecureAllowDumpSos bool InsecureAllowDumpActionMap bool + Prometheus PrometheusConfig usedConfigDir string } @@ -131,6 +138,7 @@ func DefaultConfig() *Config { config.ListenAddressRestActions = "localhost:1338" config.ListenAddressGrpcActions = "localhost:1339" config.ListenAddressWebUI = "localhost:1340" + config.ListenAddressPrometheus = "localhost:1341" config.ExternalRestAddress = "." config.LogLevel = "INFO" config.CheckForUpdates = true @@ -145,6 +153,8 @@ func DefaultConfig() *Config { config.InsecureAllowDumpVars = false config.InsecureAllowDumpSos = false config.InsecureAllowDumpActionMap = false + config.Prometheus.Enabled = false + config.Prometheus.DefaultGoMetrics = false return &config } diff --git a/internal/config/config_reloader.go b/internal/config/config_reloader.go new file mode 100644 index 0000000..4fc7a31 --- /dev/null +++ b/internal/config/config_reloader.go @@ -0,0 +1,37 @@ +package config + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + log "github.com/sirupsen/logrus" + "os" + "path" + + "github.com/spf13/viper" +) + +var ( + metricConfigActionCount = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "olivetin_config_action_count", + Help: "Then number of actions in the config file", + }) + + metricConfigReloadedCount = promauto.NewCounter(prometheus.CounterOpts{ + Name: "olivetin_config_reloaded_count", + Help: "The number of times the config has been reloaded", + }) +) + +func Reload(cfg *Config) { + if err := viper.UnmarshalExact(&cfg); err != nil { + log.Errorf("Config unmarshal error %+v", err) + os.Exit(1) + } + + metricConfigReloadedCount.Inc() + metricConfigActionCount.Set(float64(len(cfg.Actions))) + + cfg.SetDir(path.Dir(viper.ConfigFileUsed())) + cfg.Sanitize() +} diff --git a/internal/entityfiles/entityfiles.go b/internal/entityfiles/entityfiles.go index b20885a..fd56c13 100644 --- a/internal/entityfiles/entityfiles.go +++ b/internal/entityfiles/entityfiles.go @@ -174,13 +174,13 @@ func updateEvmFromFile(entityname string, data []map[string]string) { sv.RemoveKeysThatStartWith("entities." + entityname) - sv.Contents["entities."+entityname+".count"] = fmt.Sprintf("%v", count) + sv.Set("entities."+entityname+".count", fmt.Sprintf("%v", count)) for i, mapp := range data { prefix := "entities." + entityname + "." + fmt.Sprintf("%v", i) for k, v := range mapp { - sv.Contents[prefix+"."+k] = v + sv.Set(prefix+"."+k, v) } } } diff --git a/internal/executor/executor.go b/internal/executor/executor.go index 1bee85e..58177ad 100644 --- a/internal/executor/executor.go +++ b/internal/executor/executor.go @@ -7,6 +7,9 @@ import ( "github.com/google/uuid" log "github.com/sirupsen/logrus" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "bytes" "context" "fmt" @@ -17,6 +20,13 @@ import ( "time" ) +var ( + metricActionsRequested = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "olivetin_actions_requested_count", + Help: "The actions requested count", + }) +) + // Executor represents a helper class for executing commands. It's main method // is ExecRequest type Executor struct { @@ -229,6 +239,8 @@ func stepRequestAction(req *ExecutionRequest) bool { } } + metricActionsRequested.Inc() + req.logEntry.ActionTitle = sv.ReplaceEntityVars(req.EntityPrefix, req.Action.Title) req.logEntry.ActionIcon = req.Action.Icon req.logEntry.ActionId = req.Action.ID diff --git a/internal/grpcapi/grpcApi.go b/internal/grpcapi/grpcApi.go index 4987fb5..7d53d4d 100644 --- a/internal/grpcapi/grpcApi.go +++ b/internal/grpcapi/grpcApi.go @@ -302,7 +302,7 @@ func (api *oliveTinAPI) DumpVars(ctx ctx.Context, req *pb.DumpVarsRequest) (*pb. } res.Alert = "Dumping variables has been enabled in the configuration. Please set InsecureAllowDumpVars = false again after you don't need it anymore" - res.Contents = sv.Contents + res.Contents = sv.GetAll() return res, nil } diff --git a/internal/httpservers/httpServer.go b/internal/httpservers/httpServer.go index 19622fa..d41f7d8 100644 --- a/internal/httpservers/httpServer.go +++ b/internal/httpservers/httpServer.go @@ -13,5 +13,9 @@ func StartServers(cfg *config.Config) { go StartSingleHTTPFrontend(cfg) } + if cfg.Prometheus.Enabled { + go StartPrometheus(cfg) + } + startRestAPIServer(cfg) } diff --git a/internal/httpservers/prometheus.go b/internal/httpservers/prometheus.go new file mode 100644 index 0000000..6425145 --- /dev/null +++ b/internal/httpservers/prometheus.go @@ -0,0 +1,19 @@ +package httpservers + +import ( + "net/http" + + config "github.com/OliveTin/OliveTin/internal/config" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func StartPrometheus(cfg *config.Config) { + if !cfg.Prometheus.DefaultGoMetrics { + prometheus.Unregister(collectors.NewGoCollector()) + } + + http.Handle("/", promhttp.Handler()) + http.ListenAndServe(cfg.ListenAddressPrometheus, nil) +} diff --git a/internal/httpservers/singleFrontend.go b/internal/httpservers/singleFrontend.go index a90b471..786e476 100644 --- a/internal/httpservers/singleFrontend.go +++ b/internal/httpservers/singleFrontend.go @@ -62,6 +62,16 @@ func StartSingleHTTPFrontend(cfg *config.Config) { webuiProxy.ServeHTTP(w, r) }) + if cfg.Prometheus.Enabled { + promURL, _ := url.Parse("http://" + cfg.ListenAddressPrometheus) + promProxy := httputil.NewSingleHostReverseProxy(promURL) + + mux.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) { + log.Debugf("prom req: %q", r.URL) + promProxy.ServeHTTP(w, r) + }) + } + srv := &http.Server{ Addr: cfg.ListenAddressSingleHTTPFrontend, Handler: mux, diff --git a/internal/installationinfo/init.go b/internal/installationinfo/init.go index 42fca73..fc72254 100644 --- a/internal/installationinfo/init.go +++ b/internal/installationinfo/init.go @@ -6,11 +6,11 @@ import ( ) func init() { - sv.Contents["OliveTin.build.commit"] = Build.Commit - sv.Contents["OliveTin.build.version"] = Build.Version - sv.Contents["OliveTin.build.date"] = Build.Date - sv.Contents["OliveTin.runtime.os"] = Runtime.OS - sv.Contents["OliveTin.runtime.os.pretty"] = Runtime.OSReleasePrettyName - sv.Contents["OliveTin.runtime.arch"] = Runtime.Arch - sv.Contents["OliveTin.runtime.incontainer"] = fmt.Sprintf("%v", Runtime.InContainer) + sv.Set("OliveTin.build.commit", Build.Commit) + sv.Set("OliveTin.build.version", Build.Version) + sv.Set("OliveTin.build.date", Build.Date) + sv.Set("OliveTin.runtime.os", Runtime.OS) + sv.Set("OliveTin.runtime.os.pretty", Runtime.OSReleasePrettyName) + sv.Set("OliveTin.runtime.arch", Runtime.Arch) + sv.Set("OliveTin.runtime.incontainer", fmt.Sprintf("%v", Runtime.InContainer)) } diff --git a/internal/stringvariables/entities.go b/internal/stringvariables/entities.go index e24224e..a6aaa97 100644 --- a/internal/stringvariables/entities.go +++ b/internal/stringvariables/entities.go @@ -31,9 +31,9 @@ func ReplaceEntityVars(prefix string, source string) string { } func RemoveKeysThatStartWith(search string) { - for k, _ := range Contents { + for k, _ := range contents { if strings.HasPrefix(k, search) { - delete(Contents, k) + delete(contents, k) } } } diff --git a/internal/stringvariables/map.go b/internal/stringvariables/map.go index 8bdd555..79c95ac 100644 --- a/internal/stringvariables/map.go +++ b/internal/stringvariables/map.go @@ -9,14 +9,26 @@ package stringvariables -var Contents map[string]string +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + contents map[string]string + + metricSvCount = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "olivetin_sv_count", + Help: "The number entries in the sv map", + }) +) func init() { - Contents = make(map[string]string) + contents = make(map[string]string) } func Get(key string) string { - v, ok := Contents[key] + v, ok := contents[key] if !ok { return "" @@ -24,3 +36,13 @@ func Get(key string) string { return v } } + +func GetAll() map[string]string { + return contents +} + +func Set(key string, value string) { + contents[key] = value + + metricSvCount.Set(float64(len(contents))) +}