diff --git a/CHANGELOG.md b/CHANGELOG.md index e52dbff..1b7fc63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.1.0] - 2019-10-12 +## [0.2.0] - 2024-11-05 + +- Added almost every possible metric + +## [0.1.0] - 2024-10-24 - Initial release -[0.1.0]: https://git.opensocial.at/nupplaPhil/friendica-exporter/releases/tag/v0.1.0 \ No newline at end of file +[0.1.0]: https://git.friendi.ca/friendica/friendica-exporter/releases/tag/v0.1.0 +[0.2.0]: https://git.friendi.ca/friendica/friendica-exporter/releases/tag/v0.1.0 \ No newline at end of file diff --git a/README.md b/README.md index 0b4932f..cf62044 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ The following tags are available: ### Pre-built binaries -The [releases](https://git.friendi.ca/friendica-exporter/releases) page contains pre-built binaries for AMD64 and ARM64 linux. +The [releases](https://git.friendi.ca/friendica/friendica-exporter/releases) page contains pre-built binaries for AMD64 and ARM64 linux. ### Build from Source @@ -146,20 +146,38 @@ scrape_configs: These metrics are exported by `friendica-exporter`: -| name | description | -|----------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| friendica_cron_last_execution | Contains information about the last execution of cron of this Instance | -| friendica_exporter_info | Information about the friendica-exporter | -| friendica_up | Indicates if the metrics could be scraped by the exporter | -| friendica_update_available | Contains information whether a system update is available (0 = no, 1 = yes). The available_version level contains the latest available Friendica version, whereas the version level contains the current installed Friendica version. | -| friendica_update_db_status | Contains information whether a database update was successful (0 = no, 1 = yes) | -| friendica_update_status | Contains information whether a system update was successful (1 = no, 0 = yes) | -| friendica_users_active_half_year | Number of active users of the last half year | -| friendica_users_active_month | Number of active users of the last month | -| friendica_users_active_week | Number of active users of the last week | -| friendica_users_pending | Number of pending users | -| friendica_users_total | Number of total users of this Instance | -| friendica_worker_jpm | Number of jobs per Minute | -| friendica_worker_last_execution | Contains information about the last execution of worker of this Instance | -| friendica_worker_tasks_active | Number of active worker tasks of this Instance | -| friendica_worker_tasks_total | Number of worker tasks of this Instance | +| name | description | +|---------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| friendica_cron_last_execution | Contains information about the last execution of cron of this Instance | +| friendica_exporter_info | Information about the friendica-exporter | +| friendica_packets_inbound | Number of packets inbound. | +| friendica_packets_outbound | Number of packets outbound. | +| friendica_php_info | Contains meta information about PHP as labels. Value is always 1. | +| friendica_php_memory_limit_bytes | Configured PHP memory limit in bytes. | +| friendica_php_post_max_allowed_packet | Configured maximum allowed packet length to send to or receive from the server. | +| friendica_php_post_max_size_bytes | Configured maximum post size in bytes. | +| friendica_php_upload_max_size_bytes | Configured maximum upload size in bytes. | +| friendica_posts_inbound | Number of posts inbound. | +| friendica_posts_outbound | Number of posts outbound. | +| friendica_reports_closed | Number of closed reports. | +| friendica_reports_newest | Contains the datetime about the newest report of this Instance. | +| friendica_reports_open | Number of open reports. | +| friendica_server_info | Contains meta information about Server as labels. Value is always 1. | +| friendica_up | Indicates if the metrics could be scraped by the exporter | +| friendica_update_available | Contains information whether a system update is available (0 = no, 1 = yes). The available_version level contains the latest available Friendica version, whereas the version level contains the current installed Friendica version. | +| friendica_update_db_status | Contains information whether a database update was successful (0 = no, 1 = yes) | +| friendica_update_status | Contains information whether a system update was successful (1 = no, 0 = yes) | +| friendica_users_active_half_year | Number of active users of the last half year | +| friendica_users_active_month | Number of active users of the last month | +| friendica_users_active_week | Number of active users of the last week | +| friendica_users_pending | Number of pending users | +| friendica_users_total | Number of total users of this Instance | +| friendica_worker_jpm | Number of jobs per Minute | +| friendica_worker_last_execution | Contains information about the last execution of worker of this Instance | +| friendica_worker_tasks_active | Number of active worker tasks of this Instance | +| friendica_worker_tasks_total | Number of worker tasks of this Instance | + + +# Thanks + +Special thanks goes to https://github.com/xperimental and his contributors, because a lot of code is based on https://github.com/xperimental/nextcloud-exporter \ No newline at end of file diff --git a/contrib/prometheus.alert.yml b/contrib/prometheus.alert.yml new file mode 100644 index 0000000..3b64c94 --- /dev/null +++ b/contrib/prometheus.alert.yml @@ -0,0 +1,72 @@ +groups: + # --- Recording rules (no subqueries) --- + - name: friendica-exporter.records + rules: + # Current backlog per instance (sum over all priority levels) + - record: friendica:worker_backlog + expr: sum by (instance) (friendica_worker_tasks_total{job="friendica"}) + + # Backlog change over 10 minutes (using offset) + - record: friendica:worker_backlog_10m_delta + expr: friendica:worker_backlog - friendica:worker_backlog offset 10m + + # Backlog change over 2 hours (using offset) + - record: friendica:worker_backlog_2h_delta + expr: friendica:worker_backlog - friendica:worker_backlog offset 2h + + # --- Alerts --- + - name: friendica-exporter.alerts + rules: + + # 1) Exporter must be available (target up AND exporter can read Friendica) + - alert: FriendicaExporterUnavailable + expr: | + (up{job="friendica"} == 0) + OR (friendica_up{job="friendica"} == 0) + for: 5m + labels: + severity: critical + service: friendica + component: exporter + tier: app + annotations: + summary: "Friendica exporter unavailable on {{ $labels.instance }}" + description: "Target down or exporter cannot read Friendica (friendica_up=0) for >5m." + + # 2) Worker must be active (<15 minutes since last execution), with JPM fallback + - alert: FriendicaWorkerStale + expr: | + ( + time() + - max by (instance) (friendica_worker_last_execution{job="friendica"}) + > 15 * 60 + ) + OR + ( + max_over_time(friendica_worker_jpm{job="friendica", frequency="1 minute"}[15m]) <= 0 + ) + for: 5m + labels: + severity: warning + service: friendica + component: worker + tier: app + annotations: + summary: "Friendica worker inactive (>15m) on {{ $labels.instance }}" + description: "Last worker execution is older than 15 minutes or 1-minute JPM stayed 0 for the last 15 minutes." + + # 3) Backlog grows without relief (deadlock / stall) + # A: backlog rose >200 over 2h; B: no 10-min window with a decrease in the last 2h + - alert: FriendicaWorkerBacklogMonotonicIncrease + expr: | + (friendica:worker_backlog_2h_delta > 200) + AND (min_over_time(friendica:worker_backlog_10m_delta[2h]) >= 0) + for: 15m + labels: + severity: critical + service: friendica + component: worker + tier: app + annotations: + summary: "Backlog grows without decreases on {{ $labels.instance }}" + description: "Outstanding worker tasks rose by >200 in the last 2h and never dropped in any 10-minute slice → likely deadlock or stalled workers. Δ2h={{ $value }}." diff --git a/internal/metrics/collector.go b/internal/metrics/collector.go index e32d735..7a84495 100644 --- a/internal/metrics/collector.go +++ b/internal/metrics/collector.go @@ -6,6 +6,7 @@ import ( "friendica-exporter/serverinfo" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" + "strconv" ) const ( @@ -65,10 +66,62 @@ var ( metricPrefix+"users_active_half_year", "Number of active users of the last half year.", nil, nil) + postsInbound = prometheus.NewDesc( + metricPrefix+"posts_inbound", + "Number of posts inbound.", + []string{"type"}, nil) + postsOutbound = prometheus.NewDesc( + metricPrefix+"posts_outbound", + "Number of posts outbound.", + []string{"type"}, nil) + packetsInbound = prometheus.NewDesc( + metricPrefix+"packets_inbound", + "Number of packets inbound.", + []string{"protocol"}, nil) + packetsOutbound = prometheus.NewDesc( + metricPrefix+"packets_outbound", + "Number of packets outbound.", + []string{"protocol"}, nil) usersPending = prometheus.NewDesc( metricPrefix+"users_pending", "Number of pending users.", nil, nil) + reportsNewest = prometheus.NewDesc( + metricPrefix+"reports_newest", + "Contains the datetime about the newest report of this Instance.", + []string{"datetime"}, nil) + reportsOpen = prometheus.NewDesc( + metricPrefix+"reports_open", + "Number of open reports.", + nil, nil) + reportsClosed = prometheus.NewDesc( + metricPrefix+"reports_closed", + "Number of closed reports.", + nil, nil) + serverInfoDesc = prometheus.NewDesc( + metricPrefix+"server_info", + "Contains meta information about Server as labels. Value is always 1.", + []string{"version"}, nil) + phpInfoDesc = prometheus.NewDesc( + metricPrefix+"php_info", + "Contains meta information about PHP as labels. Value is always 1.", + []string{"version"}, nil) + phpMemoryLimitDesc = prometheus.NewDesc( + metricPrefix+"php_memory_limit_bytes", + "Configured PHP memory limit in bytes.", + nil, nil) + phpMaxUploadSizeDesc = prometheus.NewDesc( + metricPrefix+"php_upload_max_size_bytes", + "Configured maximum upload size in bytes.", + nil, nil) + phpMaxPostSizeDesc = prometheus.NewDesc( + metricPrefix+"php_post_max_size_bytes", + "Configured maximum post size in bytes.", + nil, nil) + databaseMaxAllowedPacketDesc = prometheus.NewDesc( + metricPrefix+"php_post_max_allowed_packet", + "Configured maximum allowed packet length to send to or receive from the server.", + nil, nil) ) type friendicaCollector struct { @@ -113,6 +166,19 @@ func (c *friendicaCollector) Describe(ch chan<- *prometheus.Desc) { ch <- usersActiveMonth ch <- usersActiveHalfYear ch <- usersPending + ch <- postsInbound + ch <- postsOutbound + ch <- packetsInbound + ch <- packetsOutbound + ch <- reportsNewest + ch <- reportsOpen + ch <- reportsClosed + ch <- serverInfoDesc + ch <- phpInfoDesc + ch <- phpMaxUploadSizeDesc + ch <- phpMaxPostSizeDesc + ch <- phpMemoryLimitDesc + ch <- databaseMaxAllowedPacketDesc } func (c *friendicaCollector) Collect(ch chan<- prometheus.Metric) { @@ -165,6 +231,22 @@ func readMetrics(ch chan<- prometheus.Metric, status *serverinfo.ServerInfo) err return err } + if err := collectPosts(ch, status); err != nil { + return err + } + + if err := collectPacketsPerDirection(ch, status, packetsInbound); err != nil { + return err + } + + if err := collectPacketsPerDirection(ch, status, packetsOutbound); err != nil { + return err + } + + if err := collectPhpMetrics(ch, status); err != nil { + return err + } + return nil } @@ -222,6 +304,14 @@ func collectSimpleMetrics(ch chan<- prometheus.Metric, status *serverinfo.Server desc: usersPending, value: float64(status.Users.Pending), }, + { + desc: reportsOpen, + value: float64(status.Reports.Open), + }, + { + desc: reportsClosed, + value: float64(status.Reports.Closed), + }, } for _, m := range metrics { @@ -359,5 +449,178 @@ func collectLastExecutions(ch chan<- prometheus.Metric, status *serverinfo.Serve } ch <- metric + reportsInfo := status.Reports + + metric, err = prometheus.NewConstMetric(reportsNewest, prometheus.GaugeValue, float64(reportsInfo.Newest.Timestamp), reportsInfo.Newest.DateTime) + if err != nil { + return fmt.Errorf("error creating metric for %s: %w", reportsNewest, err) + } + ch <- metric + return nil +} + +type postMetric struct { + desc *prometheus.Desc + value float64 + postType string +} + +func collectPosts(ch chan<- prometheus.Metric, status *serverinfo.ServerInfo) error { + metrics := []postMetric{ + { + desc: postsInbound, + value: float64(status.Posts.Inbound.Posts), + postType: "posts", + }, + { + desc: postsInbound, + value: float64(status.Posts.Inbound.Comments), + postType: "comments", + }, + { + desc: postsOutbound, + value: float64(status.Posts.Outbound.Posts), + postType: "posts", + }, + { + desc: postsOutbound, + value: float64(status.Posts.Outbound.Comments), + postType: "comments", + }, + } + + for _, m := range metrics { + metric, err := prometheus.NewConstMetric(m.desc, prometheus.GaugeValue, m.value, m.postType) + if err != nil { + return fmt.Errorf("error creating metric for %s: %w", m.desc, err) + } + ch <- metric + } + + return nil +} + +type packetMetrics struct { + desc *prometheus.Desc + value float64 + protocol string +} + +func collectPacketsPerDirection(ch chan<- prometheus.Metric, status *serverinfo.ServerInfo, desc *prometheus.Desc) error { + var packetInfo serverinfo.PacketCounts + if desc == packetsInbound { + packetInfo = status.Packets.Inbound + } else { + packetInfo = status.Packets.Outbound + } + + metrics := []packetMetrics{ + { + desc: desc, + value: float64(packetInfo.ActivityPub), + protocol: "apub", + }, + { + desc: desc, + value: float64(packetInfo.DFRN), + protocol: "dfrn", + }, + { + desc: desc, + value: float64(packetInfo.Feed), + protocol: "feed", + }, + { + desc: desc, + value: float64(packetInfo.Diaspora), + protocol: "dspr", + }, + { + desc: desc, + value: float64(packetInfo.Mail), + protocol: "mail", + }, + { + desc: desc, + value: float64(packetInfo.OStatus), + protocol: "stat", + }, + { + desc: desc, + value: float64(packetInfo.Bluesky), + protocol: "bsky", + }, + { + desc: desc, + value: float64(packetInfo.Thumblr), + protocol: "tmbl", + }, + } + + for _, m := range metrics { + metric, err := prometheus.NewConstMetric(m.desc, prometheus.GaugeValue, m.value, m.protocol) + if err != nil { + return fmt.Errorf("error creating metric for %s: %w", m.desc, err) + } + ch <- metric + } + + return nil +} + +func collectPhpMetrics(ch chan<- prometheus.Metric, status *serverinfo.ServerInfo) error { + + metric, err := prometheus.NewConstMetric(phpInfoDesc, prometheus.GaugeValue, 1, status.Server.PHP.Version) + if err != nil { + return fmt.Errorf("error creating metric for %s: %w", phpInfoDesc, err) + } + ch <- metric + + metric, err = prometheus.NewConstMetric(serverInfoDesc, prometheus.GaugeValue, 1, status.Server.Version) + if err != nil { + return fmt.Errorf("error creating metric for %s: %w", serverInfoDesc, err) + } + ch <- metric + + metricVal, err := strconv.ParseFloat(status.Server.Database.MaxAllowedPacket, 64) + if err != nil { + return fmt.Errorf("error converting value to byte for %s: %w", databaseMaxAllowedPacketDesc, err) + } + metric, err = prometheus.NewConstMetric(databaseMaxAllowedPacketDesc, prometheus.GaugeValue, metricVal) + if err != nil { + return fmt.Errorf("error creating metric for %s: %w", databaseMaxAllowedPacketDesc, err) + } + ch <- metric + + metrics := []struct { + desc *prometheus.Desc + value string + }{ + { + desc: phpMemoryLimitDesc, + value: status.Server.PHP.MemoryLimit, + }, + { + desc: phpMaxPostSizeDesc, + value: status.Server.PHP.PostMaxSize, + }, + { + desc: phpMaxUploadSizeDesc, + value: status.Server.PHP.UploadMaxFilesize, + }, + } + + for _, m := range metrics { + metricVal, err := ConvertMemoryToBytes(m.value) + if err != nil { + return fmt.Errorf("error converting value to byte for %s: %w", m.desc, err) + } + metric, err = prometheus.NewConstMetric(m.desc, prometheus.GaugeValue, float64(metricVal)) + if err != nil { + return fmt.Errorf("error creating metric for %s: %w", m.desc, err) + } + ch <- metric + } + return nil } diff --git a/internal/metrics/converter.go b/internal/metrics/converter.go new file mode 100644 index 0000000..850b999 --- /dev/null +++ b/internal/metrics/converter.go @@ -0,0 +1,51 @@ +package metrics + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" +) + +// ConvertMemoryToBytes converts a memory limit string into bytes. +// Supported units are: K (Kilobytes), M (Megabytes), G (Gigabytes), case-insensitive. +// Anything else is interpreted as bytes. +func ConvertMemoryToBytes(memory string) (uint64, error) { + // Trim whitespace + memory = strings.TrimSpace(memory) + + // Regular expression to match the pattern (number + optional unit) + re := regexp.MustCompile(`(?i)^([\d.]+)([KMG]?)$`) + matches := re.FindStringSubmatch(memory) + if matches == nil { + return 0, errors.New("invalid memory format") + } + + // Extract the numeric part and unit + numberStr := matches[1] + unit := strings.ToUpper(matches[2]) + + // Parse the number, truncate fractional parts as PHP does + number, err := strconv.ParseFloat(numberStr, 64) + if err != nil { + return 0, fmt.Errorf("invalid numeric value: %v", err) + } + numberInt := uint64(number) // Truncate fractional part by casting to uint64 + + // Convert to bytes based on the unit + var multiplier uint64 + switch unit { + case "K": + multiplier = 1024 // Kilobytes + case "M": + multiplier = 1024 * 1024 // Megabytes + case "G": + multiplier = 1024 * 1024 * 1024 // Gigabytes + default: + multiplier = 1 // Bytes + } + + // Calculate the result + return numberInt * multiplier, nil +} diff --git a/internal/metrics/converter_test.go b/internal/metrics/converter_test.go new file mode 100644 index 0000000..ba906c6 --- /dev/null +++ b/internal/metrics/converter_test.go @@ -0,0 +1,124 @@ +package metrics + +import ( + "errors" + "friendica-exporter/internal/testutil" + "testing" +) + +func TestConvertMemoryToBytes(t *testing.T) { + tt := []struct { + desc string + memory string + wantErr error + wantEqual uint64 + }{ + { + desc: "1 kilobyte (upper case)", + memory: "1K", + wantErr: nil, + wantEqual: 1024, + }, + { + desc: "1 kilobyte (lower case)", + memory: "1k", + wantErr: nil, + wantEqual: 1024, + }, + { + desc: "1 megabyte (upper case)", + memory: "1M", + wantErr: nil, + wantEqual: 1048576, + }, + { + desc: "1 megabyte (lower case)", + memory: "1m", + wantErr: nil, + wantEqual: 1048576, + }, + { + desc: "1 gigabyte (upper case)", + memory: "1G", + wantErr: nil, + wantEqual: 1073741824, + }, + { + desc: "1 gigabyte (lower case)", + memory: "1g", + wantErr: nil, + wantEqual: 1073741824, + }, + { + desc: "2 Kilobytes", + memory: "2K", + wantErr: nil, + wantEqual: 2048, + }, + { + desc: "5 gigabytes", + memory: "5g", + wantErr: nil, + wantEqual: 5368709120, + }, + { + desc: "0.5 Megabyte", + memory: "0.5M", + wantErr: nil, + wantEqual: 0, + }, + { + desc: "10.7 Kilobytes", + memory: "10.7k", + wantErr: nil, + wantEqual: 10240, + }, + { + desc: "12345234 Bytes", + memory: "12345234", + wantErr: nil, + wantEqual: 12345234, + }, + { + desc: "Invalid", + memory: "invalid", + wantErr: errors.New("invalid memory format"), + wantEqual: 0, + }, + { + desc: "1234X", + memory: "1234X", + wantErr: errors.New("invalid memory format"), + wantEqual: 0, + }, + { + desc: "512 Megabytes", + memory: "512M", + wantErr: nil, + wantEqual: 536870912, + }, + { + desc: "CopyPasteError", + memory: "512M512M", + wantErr: errors.New("invalid memory format"), + wantEqual: 0, + }, + } + + for _, tc := range tt { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + equal, err := ConvertMemoryToBytes(tc.memory) + + if !testutil.EqualErrorMessage(err, tc.wantErr) { + t.Errorf("got error %q, want %q", err, tc.wantErr) + } + + if equal != tc.wantEqual { + t.Errorf("got equal %v, want %v", equal, tc.wantEqual) + } + }) + } +} diff --git a/serverinfo/serverinfo.go b/serverinfo/serverinfo.go index 8eb9d51..e6fac74 100644 --- a/serverinfo/serverinfo.go +++ b/serverinfo/serverinfo.go @@ -5,6 +5,7 @@ type ServerInfo struct { Cron Cron `json:"cron"` Worker Worker `json:"worker"` Users Users `json:"users"` + Posts Posts `json:"posts"` Packets Packets `json:"packets"` Reports Reports `json:"reports"` Update Update `json:"update"` @@ -45,7 +46,7 @@ type Worker struct { LastExecution DateTimeTimestamp `json:"lastExecution"` JPM JPM `json:"jpm"` Active WorkerCount `json:"active"` - Defferd []int64 `json:"defferd"` + Deferred []int64 `json:"deferrd"` Total WorkerCount `json:"total"` } @@ -78,6 +79,8 @@ type PacketCounts struct { OStatus int64 `json:"stat"` Feed int64 `json:"feed"` Mail int64 `json:"mail"` + Bluesky int64 `json:"bsky"` + Thumblr int64 `json:"tmbl"` } // Packets contains statistics of inbound and outbound packages @@ -105,7 +108,7 @@ type Update struct { type PHP struct { Version string `json:"version"` UploadMaxFilesize string `json:"upload_max_filesize"` - PostMaxSize string `json:"post_max_filesize"` + PostMaxSize string `json:"post_max_size"` MemoryLimit string `json:"memory_limit"` }