From d32b00ff3171ee03479e8f7421e87e7779ee9a18 Mon Sep 17 00:00:00 2001 From: "CanbiZ (MickLesk)" <47820557+MickLesk@users.noreply.github.com> Date: Wed, 11 Feb 2026 12:19:30 +0100 Subject: [PATCH] Add weekly reports, cleanup, and dashboard UI Introduce weekly summary reports and a cleanup job, enhance dashboard UI, and adjust telemetry/build settings. - Add REPO_SOURCE to misc/api.func and include repo_source in telemetry payloads. - Implement weekly report generation/scheduling in alerts.go: new data types, HTML/plain templates, scheduler, SendWeeklyReport/TestWeeklyReport, and email/HTML helpers. - Add Cleaner (misc/data/cleanup.go) to detect and mark stuck installations as 'unknown' with scheduling and manual trigger APIs. - Enhance dashboard backend/frontend (misc/data/dashboard.go): optional days filter (allow 'All'), increase fetch page size, simplify fetchRecords, add quick filter buttons, detail & health modals, improved styles and chart options, and client-side record detail view. - Update Dockerfile (misc/data/Dockerfile): rename binaries to telemetry-service and build migrate from ./migration/migrate.go; copy adjusted in final image. - Add migration tooling (misc/data/migration/migrate.sh and migration.go) and other small service changes. These changes add operational reporting and cleanup capabilities, improve observability and UX of the dashboard, and align build and telemetry identifiers for the service. --- misc/api.func | 22 +- misc/data/Dockerfile | 6 +- misc/data/alerts.go | 600 ++++++++++++++++++++++++++++- misc/data/cleanup.go | 173 +++++++++ misc/data/dashboard.go | 627 +++++++++++++++++++++++++++++-- misc/data/entrypoint.sh | 74 ++-- misc/data/go.mod | 6 +- misc/data/go.sum | 8 +- misc/data/migration/migrate.sh | 67 ++++ misc/data/migration/migration.go | 492 ++++++++++++++++++++++++ misc/data/service.go | 82 ++-- 11 files changed, 2047 insertions(+), 110 deletions(-) create mode 100644 misc/data/cleanup.go create mode 100644 misc/data/migration/migrate.sh create mode 100644 misc/data/migration/migration.go diff --git a/misc/api.func b/misc/api.func index d910b9884..7d02a7912 100644 --- a/misc/api.func +++ b/misc/api.func @@ -37,6 +37,10 @@ TELEMETRY_URL="https://telemetry.community-scripts.org/telemetry" # Timeout for telemetry requests (seconds) TELEMETRY_TIMEOUT=5 +# Repository source identifier (auto-transformed by CI on promotion to ProxmoxVE) +# DO NOT CHANGE - this is used by the telemetry service to route data to the correct collection +REPO_SOURCE="community-scripts/ProxmoxVED" + # ============================================================================== # SECTION 1: ERROR CODE DESCRIPTIONS # ============================================================================== @@ -350,7 +354,8 @@ post_to_api() { "gpu_vendor": "${gpu_vendor}", "gpu_model": "${gpu_model}", "gpu_passthrough": "${gpu_passthrough}", - "ram_speed": "${ram_speed}" + "ram_speed": "${ram_speed}", + "repo_source": "${REPO_SOURCE}" } EOF ) @@ -421,7 +426,8 @@ post_to_api_vm() { "os_type": "${var_os:-}", "os_version": "${var_version:-}", "pve_version": "${pve_version}", - "method": "${METHOD:-default}" + "method": "${METHOD:-default}", + "repo_source": "${REPO_SOURCE}" } EOF ) @@ -541,7 +547,8 @@ post_update_to_api() { "gpu_vendor": "${gpu_vendor}", "gpu_model": "${gpu_model}", "gpu_passthrough": "${gpu_passthrough}", - "ram_speed": "${ram_speed}" + "ram_speed": "${ram_speed}", + "repo_source": "${REPO_SOURCE}" } EOF ) @@ -671,7 +678,8 @@ post_tool_to_api() { "error": "${error}", "error_category": "${error_category}", "install_duration": ${duration:-0}, - "pve_version": "${pve_version}" + "pve_version": "${pve_version}", + "repo_source": "${REPO_SOURCE}" } EOF ) @@ -734,7 +742,8 @@ post_addon_to_api() { "error_category": "${error_category}", "install_duration": ${duration:-0}, "os_type": "${os_type}", - "os_version": "${os_version}" + "os_version": "${os_version}", + "repo_source": "${REPO_SOURCE}" } EOF ) @@ -820,7 +829,8 @@ post_update_to_api_extended() { "error_category": "${error_category}", "install_duration": ${duration:-0}, "gpu_vendor": "${gpu_vendor}", - "gpu_passthrough": "${gpu_passthrough}" + "gpu_passthrough": "${gpu_passthrough}", + "repo_source": "${REPO_SOURCE}" } EOF ) diff --git a/misc/data/Dockerfile b/misc/data/Dockerfile index d9228dafe..3d58795c0 100644 --- a/misc/data/Dockerfile +++ b/misc/data/Dockerfile @@ -3,13 +3,13 @@ WORKDIR /src COPY go.mod go.sum* ./ RUN go mod download 2>/dev/null || true COPY . . -RUN go build -trimpath -ldflags "-s -w" -o /out/telemetry-ingest . -RUN go build -trimpath -ldflags "-s -w" -o /out/migrate migrate.go +RUN go build -trimpath -ldflags "-s -w" -o /out/telemetry-service . +RUN go build -trimpath -ldflags "-s -w" -o /out/migrate ./migration/migrate.go FROM alpine:3.23 RUN apk add --no-cache ca-certificates tzdata WORKDIR /app -COPY --from=build /out/telemetry-ingest /app/telemetry-ingest +COPY --from=build /out/telemetry-service /app/telemetry-service COPY --from=build /out/migrate /app/migrate COPY entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh /app/migrate diff --git a/misc/data/alerts.go b/misc/data/alerts.go index 198e7a599..01f1a57f1 100644 --- a/misc/data/alerts.go +++ b/misc/data/alerts.go @@ -25,16 +25,54 @@ type AlertConfig struct { FailureThreshold float64 // Alert when failure rate exceeds this (e.g., 20.0 = 20%) CheckInterval time.Duration // How often to check Cooldown time.Duration // Minimum time between alerts + + // Weekly Report settings + WeeklyReportEnabled bool // Enable weekly summary reports + WeeklyReportDay time.Weekday // Day to send report (0=Sunday, 1=Monday, etc.) + WeeklyReportHour int // Hour to send report (0-23) +} + +// WeeklyReportData contains aggregated weekly statistics +type WeeklyReportData struct { + CalendarWeek int + Year int + StartDate time.Time + EndDate time.Time + TotalInstalls int + SuccessCount int + FailedCount int + SuccessRate float64 + TopApps []AppStat + TopFailedApps []AppStat + ComparedToPrev WeekComparison + OsDistribution map[string]int + TypeDistribution map[string]int +} + +// AppStat represents statistics for a single app +type AppStat struct { + Name string + Total int + Failed int + FailureRate float64 +} + +// WeekComparison shows changes compared to previous week +type WeekComparison struct { + InstallsChange int // Difference in total installs + InstallsPercent float64 // Percentage change + FailRateChange float64 // Change in failure rate (percentage points) } // Alerter handles alerting functionality type Alerter struct { - cfg AlertConfig - lastAlertAt time.Time - mu sync.Mutex - pb *PBClient - lastStats alertStats - alertHistory []AlertEvent + cfg AlertConfig + lastAlertAt time.Time + lastWeeklyReport time.Time + mu sync.Mutex + pb *PBClient + lastStats alertStats + alertHistory []AlertEvent } type alertStats struct { @@ -74,6 +112,12 @@ func (a *Alerter) Start() { go a.monitorLoop() log.Printf("INFO: alert monitoring started (threshold: %.1f%%, interval: %v)", a.cfg.FailureThreshold, a.cfg.CheckInterval) + + // Start weekly report scheduler if enabled + if a.cfg.WeeklyReportEnabled { + go a.weeklyReportLoop() + log.Printf("INFO: weekly report scheduler started (day: %s, hour: %02d:00)", a.cfg.WeeklyReportDay, a.cfg.WeeklyReportHour) + } } func (a *Alerter) monitorLoop() { @@ -162,13 +206,21 @@ This is an automated alert from the telemetry service. } func (a *Alerter) sendEmail(subject, body string) error { + return a.sendEmailWithType(subject, body, "text/plain") +} + +func (a *Alerter) sendHTMLEmail(subject, body string) error { + return a.sendEmailWithType(subject, body, "text/html") +} + +func (a *Alerter) sendEmailWithType(subject, body, contentType string) error { // Build message var msg bytes.Buffer msg.WriteString(fmt.Sprintf("From: %s\r\n", a.cfg.SMTPFrom)) msg.WriteString(fmt.Sprintf("To: %s\r\n", strings.Join(a.cfg.SMTPTo, ", "))) msg.WriteString(fmt.Sprintf("Subject: %s\r\n", subject)) msg.WriteString("MIME-Version: 1.0\r\n") - msg.WriteString("Content-Type: text/plain; charset=UTF-8\r\n") + msg.WriteString(fmt.Sprintf("Content-Type: %s; charset=UTF-8\r\n", contentType)) msg.WriteString("\r\n") msg.WriteString(body) @@ -264,4 +316,538 @@ This is an automated test message. // Helper for timeout context func newTimeoutContext(d time.Duration) (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), d) +} + +// weeklyReportLoop checks periodically if it's time to send the weekly report +func (a *Alerter) weeklyReportLoop() { + // Check every hour + ticker := time.NewTicker(1 * time.Hour) + defer ticker.Stop() + + for range ticker.C { + a.checkAndSendWeeklyReport() + } +} + +// checkAndSendWeeklyReport sends the weekly report if it's the right time +func (a *Alerter) checkAndSendWeeklyReport() { + now := time.Now() + + // Check if it's the right day and hour + if now.Weekday() != a.cfg.WeeklyReportDay || now.Hour() != a.cfg.WeeklyReportHour { + return + } + + a.mu.Lock() + // Check if we already sent a report this week + _, lastWeek := a.lastWeeklyReport.ISOWeek() + _, currentWeek := now.ISOWeek() + if a.lastWeeklyReport.Year() == now.Year() && lastWeek == currentWeek { + a.mu.Unlock() + return + } + a.mu.Unlock() + + // Send the weekly report + if err := a.SendWeeklyReport(); err != nil { + log.Printf("ERROR: failed to send weekly report: %v", err) + } +} + +// SendWeeklyReport generates and sends the weekly summary email +func (a *Alerter) SendWeeklyReport() error { + if !a.cfg.Enabled || a.cfg.SMTPHost == "" { + return fmt.Errorf("alerting not configured") + } + + ctx, cancel := newTimeoutContext(30 * time.Second) + defer cancel() + + // Get data for the past week + reportData, err := a.fetchWeeklyReportData(ctx) + if err != nil { + return fmt.Errorf("failed to fetch weekly data: %w", err) + } + + // Generate email content + subject := fmt.Sprintf("[ProxmoxVED] Weekly Report - Week %d, %d", reportData.CalendarWeek, reportData.Year) + body := a.generateWeeklyReportHTML(reportData) + + if err := a.sendHTMLEmail(subject, body); err != nil { + return fmt.Errorf("failed to send email: %w", err) + } + + a.mu.Lock() + a.lastWeeklyReport = time.Now() + a.alertHistory = append(a.alertHistory, AlertEvent{ + Timestamp: time.Now(), + Type: "weekly_report", + Message: fmt.Sprintf("Weekly report KW %d/%d sent", reportData.CalendarWeek, reportData.Year), + }) + a.mu.Unlock() + + log.Printf("INFO: weekly report KW %d/%d sent successfully", reportData.CalendarWeek, reportData.Year) + return nil +} + +// fetchWeeklyReportData collects data for the weekly report +func (a *Alerter) fetchWeeklyReportData(ctx context.Context) (*WeeklyReportData, error) { + // Calculate the previous week's date range (Mon-Sun) + now := time.Now() + + // Find last Monday + daysToLastMonday := int(now.Weekday() - time.Monday) + if daysToLastMonday < 0 { + daysToLastMonday += 7 + } + // Go back to the Monday of LAST week + lastMonday := now.AddDate(0, 0, -daysToLastMonday-7) + lastMonday = time.Date(lastMonday.Year(), lastMonday.Month(), lastMonday.Day(), 0, 0, 0, 0, lastMonday.Location()) + lastSunday := lastMonday.AddDate(0, 0, 6) + lastSunday = time.Date(lastSunday.Year(), lastSunday.Month(), lastSunday.Day(), 23, 59, 59, 0, lastSunday.Location()) + + // Get calendar week + year, week := lastMonday.ISOWeek() + + // Fetch current week's data (7 days) + currentData, err := a.pb.FetchDashboardData(ctx, 7) + if err != nil { + return nil, fmt.Errorf("failed to fetch current week data: %w", err) + } + + // Fetch previous week's data for comparison (14 days, we'll compare) + prevData, err := a.pb.FetchDashboardData(ctx, 14) + if err != nil { + // Non-fatal, just log + log.Printf("WARN: could not fetch previous week data: %v", err) + prevData = nil + } + + // Build report data + report := &WeeklyReportData{ + CalendarWeek: week, + Year: year, + StartDate: lastMonday, + EndDate: lastSunday, + TotalInstalls: currentData.TotalInstalls, + SuccessCount: currentData.SuccessCount, + FailedCount: currentData.FailedCount, + OsDistribution: make(map[string]int), + TypeDistribution: make(map[string]int), + } + + // Calculate success rate + if report.TotalInstalls > 0 { + report.SuccessRate = float64(report.SuccessCount) / float64(report.TotalInstalls) * 100 + } + + // Top 5 installed apps + for i, app := range currentData.TopApps { + if i >= 5 { + break + } + report.TopApps = append(report.TopApps, AppStat{ + Name: app.App, + Total: app.Count, + }) + } + + // Top 5 failed apps + for i, app := range currentData.FailedApps { + if i >= 5 { + break + } + report.TopFailedApps = append(report.TopFailedApps, AppStat{ + Name: app.App, + Total: app.TotalCount, + Failed: app.FailedCount, + FailureRate: app.FailureRate, + }) + } + + // OS distribution + for _, os := range currentData.OsDistribution { + report.OsDistribution[os.Os] = os.Count + } + + // Type distribution (LXC vs VM) + for _, t := range currentData.TypeStats { + report.TypeDistribution[t.Type] = t.Count + } + + // Calculate comparison to previous week + if prevData != nil { + // Previous week stats (subtract current from 14-day total) + prevInstalls := prevData.TotalInstalls - currentData.TotalInstalls + prevFailed := prevData.FailedCount - currentData.FailedCount + prevSuccess := prevData.SuccessCount - currentData.SuccessCount + + if prevInstalls > 0 { + prevFailRate := float64(prevFailed) / float64(prevInstalls) * 100 + currentFailRate := 100 - report.SuccessRate + + report.ComparedToPrev.InstallsChange = report.TotalInstalls - prevInstalls + if prevInstalls > 0 { + report.ComparedToPrev.InstallsPercent = float64(report.TotalInstalls-prevInstalls) / float64(prevInstalls) * 100 + } + report.ComparedToPrev.FailRateChange = currentFailRate - prevFailRate + _ = prevSuccess // suppress unused warning + } + } + + return report, nil +} + +// generateWeeklyReportHTML creates the HTML email body for the weekly report +func (a *Alerter) generateWeeklyReportHTML(data *WeeklyReportData) string { + var b strings.Builder + + // HTML Email Template + b.WriteString(` + + + + + + + + +
+ + + + + + + + + + + + + + + + +`) + + // Week comparison + if data.ComparedToPrev.InstallsChange != 0 || data.ComparedToPrev.FailRateChange != 0 { + installIcon := "📈" + installColor := "#16a34a" + if data.ComparedToPrev.InstallsChange < 0 { + installIcon = "📉" + installColor = "#dc2626" + } + failIcon := "✅" + failColor := "#16a34a" + if data.ComparedToPrev.FailRateChange > 0 { + failIcon = "⚠️" + failColor = "#dc2626" + } + + b.WriteString(` + + +`) + } + + // Top 5 Installed Scripts + b.WriteString(` + + +`) + + // Top 5 Failed Scripts + b.WriteString(` + + +`) + + // Type Distribution + if len(data.TypeDistribution) > 0 { + b.WriteString(` + + +`) + } + + // OS Distribution + if len(data.OsDistribution) > 0 { + b.WriteString(` + + +`) + } + + // Footer + b.WriteString(` + + + +
+

📊 Weekly Telemetry Report

+

ProxmoxVE Helper Scripts

+
+ + + + + +
+Calendar Week
+Week `) + b.WriteString(fmt.Sprintf("%d, %d", data.CalendarWeek, data.Year)) + b.WriteString(` +		 +Period
+`) + b.WriteString(fmt.Sprintf("%s – %s", data.StartDate.Format("Jan 02"), data.EndDate.Format("Jan 02, 2006"))) + b.WriteString(` +
+
+ + + + + + + +
+
+
`) + b.WriteString(fmt.Sprintf("%d", data.TotalInstalls)) + b.WriteString(`
+
Total
+
+		 +
+
`) + b.WriteString(fmt.Sprintf("%d", data.SuccessCount)) + b.WriteString(`
+
Successful
+
+		 +
+
`) + b.WriteString(fmt.Sprintf("%d", data.FailedCount)) + b.WriteString(`
+
Failed
+
+		 +
+
`) + b.WriteString(fmt.Sprintf("%.1f%%", data.SuccessRate)) + b.WriteString(`
+
Success Rate
+
+
+
+ + + + + +
+vs. Previous Week
+`) + b.WriteString(installIcon) + b.WriteString(fmt.Sprintf(" %+d installations (%.1f%%)", data.ComparedToPrev.InstallsChange, data.ComparedToPrev.InstallsPercent)) + b.WriteString(` +		 +Failure Rate Change
+`) + b.WriteString(failIcon) + b.WriteString(fmt.Sprintf(" %+.1f percentage points", data.ComparedToPrev.FailRateChange)) + b.WriteString(` +
+
+

🏆 Top 5 Installed Scripts

+ +`) + if len(data.TopApps) > 0 { + for i, app := range data.TopApps { + bgColor := "#ffffff" + if i%2 == 0 { + bgColor = "#f8fafc" + } + b.WriteString(fmt.Sprintf(` + + +`, bgColor, i+1, app.Name, app.Total)) + } + } else { + b.WriteString(``) + } + b.WriteString(`
+%d +%s +%d installs
No data available
+
+

⚠️ Top 5 Scripts with Highest Failure Rates

+ +`) + if len(data.TopFailedApps) > 0 { + for i, app := range data.TopFailedApps { + bgColor := "#ffffff" + if i%2 == 0 { + bgColor = "#fef2f2" + } + rateColor := "#dc2626" + if app.FailureRate < 20 { + rateColor = "#ea580c" + } + if app.FailureRate < 10 { + rateColor = "#ca8a04" + } + b.WriteString(fmt.Sprintf(` + + + +`, bgColor, app.Name, app.Failed, app.Total, rateColor, app.FailureRate)) + } + } else { + b.WriteString(``) + } + b.WriteString(`
+%s +%d / %d failed +%.1f%% +
🎉 No failures this week!
+
+

📦 Distribution by Type

+ + +`) + for t, count := range data.TypeDistribution { + percent := float64(count) / float64(data.TotalInstalls) * 100 + b.WriteString(fmt.Sprintf(``, count, strings.ToUpper(t), percent)) + } + b.WriteString(` +
+
+
%d
+
%s (%.1f%%)
+
+
+
+

🐧 Top Operating Systems

+ +`) + // Sort OS by count + type osEntry struct { + name string + count int + } + var osList []osEntry + for name, count := range data.OsDistribution { + osList = append(osList, osEntry{name, count}) + } + for i := 0; i < len(osList); i++ { + for j := i + 1; j < len(osList); j++ { + if osList[j].count > osList[i].count { + osList[i], osList[j] = osList[j], osList[i] + } + } + } + for i, os := range osList { + if i >= 5 { + break + } + percent := float64(os.count) / float64(data.TotalInstalls) * 100 + barWidth := int(percent * 2) // Scale for visual + if barWidth > 100 { + barWidth = 100 + } + b.WriteString(fmt.Sprintf(` + + + +`, os.name, barWidth, os.count, percent)) + } + b.WriteString(`
%s +
+
+
+		%d (%.1f%%)
+
+

+Generated `) + b.WriteString(time.Now().Format("Jan 02, 2006 at 15:04 MST")) + b.WriteString(`
+ProxmoxVE Helper Scripts — +This is an automated report from the telemetry service. +

+
+
+ +`) + + return b.String() +} + +// generateWeeklyReportEmail creates the plain text email body (kept for compatibility) +func (a *Alerter) generateWeeklyReportEmail(data *WeeklyReportData) string { + var b strings.Builder + + b.WriteString("ProxmoxVE Helper Scripts - Weekly Telemetry Report\n") + b.WriteString("==================================================\n\n") + + b.WriteString(fmt.Sprintf("Calendar Week: %d, %d\n", data.CalendarWeek, data.Year)) + b.WriteString(fmt.Sprintf("Period: %s - %s\n\n", + data.StartDate.Format("Jan 02, 2006"), + data.EndDate.Format("Jan 02, 2006"))) + + b.WriteString("OVERVIEW\n") + b.WriteString("--------\n") + b.WriteString(fmt.Sprintf("Total Installations: %d\n", data.TotalInstalls)) + b.WriteString(fmt.Sprintf("Successful: %d\n", data.SuccessCount)) + b.WriteString(fmt.Sprintf("Failed: %d\n", data.FailedCount)) + b.WriteString(fmt.Sprintf("Success Rate: %.1f%%\n\n", data.SuccessRate)) + + if data.ComparedToPrev.InstallsChange != 0 || data.ComparedToPrev.FailRateChange != 0 { + b.WriteString("vs. Previous Week:\n") + b.WriteString(fmt.Sprintf(" Installations: %+d (%.1f%%)\n", data.ComparedToPrev.InstallsChange, data.ComparedToPrev.InstallsPercent)) + b.WriteString(fmt.Sprintf(" Failure Rate: %+.1f pp\n\n", data.ComparedToPrev.FailRateChange)) + } + + b.WriteString("TOP 5 INSTALLED SCRIPTS\n") + b.WriteString("-----------------------\n") + for i, app := range data.TopApps { + if i >= 5 { + break + } + b.WriteString(fmt.Sprintf("%d. %-25s %5d installs\n", i+1, app.Name, app.Total)) + } + b.WriteString("\n") + + b.WriteString("TOP 5 FAILED SCRIPTS\n") + b.WriteString("--------------------\n") + if len(data.TopFailedApps) > 0 { + for i, app := range data.TopFailedApps { + if i >= 5 { + break + } + b.WriteString(fmt.Sprintf("%d. %-20s %3d/%3d failed (%.1f%%)\n", + i+1, app.Name, app.Failed, app.Total, app.FailureRate)) + } + } else { + b.WriteString("No failures this week!\n") + } + b.WriteString("\n") + + b.WriteString("---\n") + b.WriteString(fmt.Sprintf("Generated: %s\n", time.Now().Format("Jan 02, 2006 15:04 MST"))) + b.WriteString("This is an automated report from the telemetry service.\n") + + return b.String() +} + +// TestWeeklyReport sends a test weekly report email +func (a *Alerter) TestWeeklyReport() error { + return a.SendWeeklyReport() } \ No newline at end of file diff --git a/misc/data/cleanup.go b/misc/data/cleanup.go new file mode 100644 index 000000000..c072bef3d --- /dev/null +++ b/misc/data/cleanup.go @@ -0,0 +1,173 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "net/url" + "time" +) + +// CleanupConfig holds configuration for the cleanup job +type CleanupConfig struct { + Enabled bool + CheckInterval time.Duration // How often to run cleanup + StuckAfterHours int // Consider "installing" as stuck after X hours +} + +// Cleaner handles cleanup of stuck installations +type Cleaner struct { + cfg CleanupConfig + pb *PBClient +} + +// NewCleaner creates a new cleaner instance +func NewCleaner(cfg CleanupConfig, pb *PBClient) *Cleaner { + return &Cleaner{ + cfg: cfg, + pb: pb, + } +} + +// Start begins the cleanup loop +func (c *Cleaner) Start() { + if !c.cfg.Enabled { + log.Println("INFO: cleanup job disabled") + return + } + + go c.cleanupLoop() + log.Printf("INFO: cleanup job started (interval: %v, stuck after: %d hours)", c.cfg.CheckInterval, c.cfg.StuckAfterHours) +} + +func (c *Cleaner) cleanupLoop() { + // Run immediately on start + c.runCleanup() + + ticker := time.NewTicker(c.cfg.CheckInterval) + defer ticker.Stop() + + for range ticker.C { + c.runCleanup() + } +} + +// runCleanup finds and updates stuck installations +func (c *Cleaner) runCleanup() { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + // Find stuck records + stuckRecords, err := c.findStuckInstallations(ctx) + if err != nil { + log.Printf("WARN: cleanup - failed to find stuck installations: %v", err) + return + } + + if len(stuckRecords) == 0 { + log.Printf("INFO: cleanup - no stuck installations found") + return + } + + log.Printf("INFO: cleanup - found %d stuck installations", len(stuckRecords)) + + // Update each record + updated := 0 + for _, record := range stuckRecords { + if err := c.markAsUnknown(ctx, record.ID); err != nil { + log.Printf("WARN: cleanup - failed to update record %s: %v", record.ID, err) + continue + } + updated++ + } + + log.Printf("INFO: cleanup - updated %d stuck installations to 'unknown'", updated) +} + +// StuckRecord represents a minimal record for cleanup +type StuckRecord struct { + ID string `json:"id"` + NSAPP string `json:"nsapp"` + Created string `json:"created"` +} + +// findStuckInstallations finds records that are stuck in "installing" status +func (c *Cleaner) findStuckInstallations(ctx context.Context) ([]StuckRecord, error) { + if err := c.pb.ensureAuth(ctx); err != nil { + return nil, err + } + + // Calculate cutoff time + cutoff := time.Now().Add(-time.Duration(c.cfg.StuckAfterHours) * time.Hour) + cutoffStr := cutoff.Format("2006-01-02 15:04:05") + + // Build filter: status='installing' AND created < cutoff + filter := url.QueryEscape(fmt.Sprintf("status='installing' && created<'%s'", cutoffStr)) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + fmt.Sprintf("%s/api/collections/%s/records?filter=%s&perPage=100", + c.pb.baseURL, c.pb.devColl, filter), + nil, + ) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+c.pb.token) + + resp, err := c.pb.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result struct { + Items []StuckRecord `json:"items"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + + return result.Items, nil +} + +// markAsUnknown updates a record's status to "unknown" +func (c *Cleaner) markAsUnknown(ctx context.Context, recordID string) error { + update := TelemetryStatusUpdate{ + Status: "unknown", + Error: "Installation timed out - no completion status received", + } + return c.pb.UpdateTelemetryStatus(ctx, recordID, update) +} + +// RunNow triggers an immediate cleanup run (for testing/manual trigger) +func (c *Cleaner) RunNow() (int, error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + stuckRecords, err := c.findStuckInstallations(ctx) + if err != nil { + return 0, fmt.Errorf("failed to find stuck installations: %w", err) + } + + updated := 0 + for _, record := range stuckRecords { + if err := c.markAsUnknown(ctx, record.ID); err != nil { + log.Printf("WARN: cleanup - failed to update record %s: %v", record.ID, err) + continue + } + updated++ + } + + return updated, nil +} + +// GetStuckCount returns the current number of stuck installations +func (c *Cleaner) GetStuckCount(ctx context.Context) (int, error) { + records, err := c.findStuckInstallations(ctx) + if err != nil { + return 0, err + } + return len(records), nil +} diff --git a/misc/data/dashboard.go b/misc/data/dashboard.go index 6b28625e4..ab899be94 100644 --- a/misc/data/dashboard.go +++ b/misc/data/dashboard.go @@ -111,12 +111,17 @@ func (p *PBClient) FetchDashboardData(ctx context.Context, days int) (*Dashboard data := &DashboardData{} - // Calculate date filter - since := time.Now().AddDate(0, 0, -days).Format("2006-01-02 00:00:00") - filter := url.QueryEscape(fmt.Sprintf("created >= '%s'", since)) + // Calculate date filter (days=0 means all entries) + var filter string + if days > 0 { + since := time.Now().AddDate(0, 0, -days).Format("2006-01-02 00:00:00") + filter = url.QueryEscape(fmt.Sprintf("created >= '%s'", since)) + } else { + filter = "" // No filter = all entries + } // Fetch all records for the period - records, err := p.fetchRecords(ctx, filter, 500) + records, err := p.fetchRecords(ctx, filter) if err != nil { return nil, err } @@ -292,17 +297,22 @@ type TelemetryRecord struct { Created string `json:"created"` } -func (p *PBClient) fetchRecords(ctx context.Context, filter string, limit int) ([]TelemetryRecord, error) { +func (p *PBClient) fetchRecords(ctx context.Context, filter string) ([]TelemetryRecord, error) { var allRecords []TelemetryRecord page := 1 - perPage := 100 + perPage := 500 for { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, - fmt.Sprintf("%s/api/collections/%s/records?filter=%s&sort=-created&page=%d&perPage=%d", - p.baseURL, p.targetColl, filter, page, perPage), - nil, - ) + var url string + if filter != "" { + url = fmt.Sprintf("%s/api/collections/%s/records?filter=%s&sort=-created&page=%d&perPage=%d", + p.baseURL, p.devColl, filter, page, perPage) + } else { + url = fmt.Sprintf("%s/api/collections/%s/records?sort=-created&page=%d&perPage=%d", + p.baseURL, p.devColl, page, perPage) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -325,7 +335,7 @@ func (p *PBClient) fetchRecords(ctx context.Context, filter string, limit int) ( allRecords = append(allRecords, result.Items...) - if len(allRecords) >= limit || len(allRecords) >= result.TotalItems { + if len(allRecords) >= result.TotalItems { break } page++ @@ -746,6 +756,36 @@ func DashboardHTML() string { color: #fff; } + .quickfilter { + display: flex; + gap: 4px; + background: var(--bg-tertiary); + padding: 4px; + border-radius: 8px; + border: 1px solid var(--border-color); + } + + .filter-btn { + background: transparent; + border: none; + color: var(--text-secondary); + padding: 6px 12px; + border-radius: 6px; + font-size: 13px; + font-weight: 500; + transition: all 0.2s; + } + + .filter-btn:hover { + background: var(--bg-secondary); + color: var(--text-primary); + } + + .filter-btn.active { + background: var(--accent-blue); + color: #fff; + } + .stats-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); @@ -951,6 +991,98 @@ func DashboardHTML() string { background: var(--bg-secondary); } + .admin-btn { + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + color: var(--text-primary); + padding: 4px 8px; + border-radius: 4px; + font-size: 12px; + cursor: pointer; + margin-left: 8px; + } + + .admin-btn:hover { + background: var(--accent-blue); + color: #fff; + border-color: var(--accent-blue); + } + + .admin-btn:disabled { + opacity: 0.5; + cursor: not-allowed; + } + + .footer-btn { + background: transparent; + border: none; + color: var(--accent-blue); + cursor: pointer; + font-size: 12px; + padding: 0; + margin-right: 8px; + } + + .footer-btn:hover { + text-decoration: underline; + } + + .health-modal { + max-width: 400px; + } + + .health-status { + display: flex; + align-items: center; + gap: 12px; + padding: 16px; + border-radius: 8px; + margin-bottom: 12px; + } + + .health-status.ok { + background: rgba(63, 185, 80, 0.1); + border: 1px solid var(--accent-green); + } + + .health-status.error { + background: rgba(248, 81, 73, 0.1); + border: 1px solid var(--accent-red); + } + + .health-status .icon { + font-size: 32px; + } + + .health-status .details { + flex: 1; + } + + .health-status .title { + font-weight: 600; + font-size: 16px; + } + + .health-status .subtitle { + font-size: 12px; + color: var(--text-secondary); + margin-top: 4px; + } + + .health-info { + font-size: 12px; + color: var(--text-secondary); + padding: 12px; + background: var(--bg-tertiary); + border-radius: 6px; + } + + .health-info div { + display: flex; + justify-content: space-between; + padding: 4px 0; + } + .pve-version-card { background: var(--bg-secondary); border: 1px solid var(--border-color); @@ -1104,6 +1236,171 @@ func DashboardHTML() string { color: var(--text-secondary); font-size: 14px; } + + /* Detail Modal Styles */ + .modal-overlay { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(0, 0, 0, 0.7); + display: flex; + justify-content: center; + align-items: center; + z-index: 1000; + opacity: 0; + visibility: hidden; + transition: opacity 0.2s, visibility 0.2s; + } + + .modal-overlay.active { + opacity: 1; + visibility: visible; + } + + .modal-content { + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: 12px; + width: 90%; + max-width: 700px; + max-height: 90vh; + overflow-y: auto; + transform: scale(0.9); + transition: transform 0.2s; + } + + .modal-overlay.active .modal-content { + transform: scale(1); + } + + .modal-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 20px 24px; + border-bottom: 1px solid var(--border-color); + position: sticky; + top: 0; + background: var(--bg-secondary); + z-index: 10; + } + + .modal-header h2 { + font-size: 20px; + font-weight: 600; + display: flex; + align-items: center; + gap: 10px; + } + + .modal-close { + background: none; + border: none; + color: var(--text-secondary); + font-size: 24px; + cursor: pointer; + padding: 4px 8px; + border-radius: 4px; + } + + .modal-close:hover { + background: var(--bg-tertiary); + color: var(--text-primary); + } + + .modal-body { + padding: 24px; + } + + .detail-section { + margin-bottom: 24px; + } + + .detail-section:last-child { + margin-bottom: 0; + } + + .detail-section-header { + display: flex; + align-items: center; + gap: 8px; + font-size: 12px; + font-weight: 600; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.5px; + margin-bottom: 12px; + padding-bottom: 8px; + border-bottom: 1px solid var(--border-color); + } + + .detail-section-header svg { + opacity: 0.7; + } + + .detail-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 16px; + } + + .detail-item { + background: var(--bg-tertiary); + border-radius: 8px; + padding: 12px 16px; + } + + .detail-item .label { + font-size: 11px; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.3px; + margin-bottom: 4px; + } + + .detail-item .value { + font-size: 15px; + font-weight: 500; + word-break: break-word; + } + + .detail-item .value.mono { + font-family: 'SF Mono', 'Consolas', monospace; + font-size: 13px; + } + + .detail-item.full-width { + grid-column: 1 / -1; + } + + .detail-item .value.status-success { color: var(--accent-green); } + .detail-item .value.status-failed { color: var(--accent-red); } + .detail-item .value.status-installing { color: var(--accent-yellow); } + + .error-box { + background: rgba(248, 81, 73, 0.1); + border: 1px solid rgba(248, 81, 73, 0.3); + border-radius: 8px; + padding: 16px; + font-family: 'SF Mono', 'Consolas', monospace; + font-size: 13px; + color: var(--accent-red); + white-space: pre-wrap; + word-break: break-word; + max-height: 200px; + overflow-y: auto; + } + + tr.clickable-row { + cursor: pointer; + transition: background 0.15s; + } + + tr.clickable-row:hover { + background: rgba(88, 166, 255, 0.1) !important; + } @@ -1116,13 +1413,13 @@ func DashboardHTML() string { Telemetry Dashboard
- +
+ + + + + +
- Health Check • - Metrics • + API
+ + + + + +