diff --git a/misc/api.func b/misc/api.func
index 062be163f..a8f851617 100644
--- a/misc/api.func
+++ b/misc/api.func
@@ -87,17 +87,17 @@ detect_repo_source() {
# Map detected owner/repo to canonical repo_source value
case "$owner_repo" in
- community-scripts/ProxmoxVE) REPO_SOURCE="ProxmoxVE" ;;
- community-scripts/ProxmoxVED) REPO_SOURCE="ProxmoxVED" ;;
- "")
- # No URL detected — use hardcoded fallback
- # CI sed transforms this on promotion: ProxmoxVED → ProxmoxVE
- REPO_SOURCE="ProxmoxVED"
- ;;
- *)
- # Fork or unknown repo
- REPO_SOURCE="external"
- ;;
+ community-scripts/ProxmoxVE) REPO_SOURCE="ProxmoxVE" ;;
+ community-scripts/ProxmoxVED) REPO_SOURCE="ProxmoxVED" ;;
+ "")
+ # No URL detected — use hardcoded fallback
+ # CI sed transforms this on promotion: ProxmoxVED → ProxmoxVE
+ REPO_SOURCE="ProxmoxVED"
+ ;;
+ *)
+ # Fork or unknown repo
+ REPO_SOURCE="external"
+ ;;
esac
export REPO_SOURCE
diff --git a/misc/data/.gitignore b/misc/data/.gitignore
new file mode 100644
index 000000000..03cf14d6e
--- /dev/null
+++ b/misc/data/.gitignore
@@ -0,0 +1,34 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+telemetry-service
+migration/migrate
+
+# Test binary, built with `go test -c`
+*.test
+
+# Code coverage profiles and other test artifacts
+*.out
+coverage.*
+*.coverprofile
+profile.cov
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+# env file
+.env
+
+# Editor/IDE
+# .idea/
+# .vscode/
diff --git a/misc/data/Dockerfile b/misc/data/Dockerfile
index 3d58795c0..7201849b2 100644
--- a/misc/data/Dockerfile
+++ b/misc/data/Dockerfile
@@ -24,7 +24,7 @@ ENV ENABLE_REQUEST_LOGGING="false"
# Cache config (optional)
ENV ENABLE_CACHE="true"
-ENV CACHE_TTL_SECONDS="60"
+ENV CACHE_TTL_SECONDS="300"
ENV ENABLE_REDIS="false"
# ENV REDIS_URL="redis://localhost:6379"
diff --git a/misc/data/LICENSE b/misc/data/LICENSE
new file mode 100644
index 000000000..1fae4b9c8
--- /dev/null
+++ b/misc/data/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Community Scripts
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/misc/data/README.md b/misc/data/README.md
new file mode 100644
index 000000000..0211ab9f1
--- /dev/null
+++ b/misc/data/README.md
@@ -0,0 +1,81 @@
+# Telemetry Service
+
+A standalone Go microservice that collects anonymous telemetry data from [ProxmoxVE](https://github.com/community-scripts/ProxmoxVE) and [ProxmoxVED](https://github.com/community-scripts/ProxmoxVED) script installations.
+
+## Overview
+
+This service acts as a telemetry ingestion layer between the bash installation scripts and a PocketBase backend. When users run scripts from the ProxmoxVE/ProxmoxVED repositories, optional anonymous usage data is sent here for aggregation and analysis.
+
+**What gets collected:**
+- Script name and installation status (success/failed)
+- Container/VM type and resource allocation (CPU, RAM, disk)
+- OS type and version
+- Proxmox VE version
+- Anonymous session ID (randomly generated UUID)
+
+**What is NOT collected:**
+- IP addresses (not logged, not stored)
+- Hostnames or domain names
+- User credentials or personal information
+- Hardware identifiers (MAC addresses, serial numbers)
+- Network configuration or internal IPs
+- Any data that could identify a person or system
+
+**What this enables:**
+- Understanding which scripts are most popular
+- Identifying scripts with high failure rates
+- Tracking resource allocation trends
+- Improving script quality based on real-world data
+
+## Features
+
+- **Telemetry Ingestion** - Receives and validates telemetry data from bash scripts
+- **PocketBase Integration** - Stores data in PocketBase collections
+- **Rate Limiting** - Configurable per-IP rate limiting to prevent abuse
+- **Caching** - In-memory or Redis-backed caching support
+- **Email Alerts** - SMTP-based alerts when failure rates exceed thresholds
+- **Dashboard** - Built-in HTML dashboard for telemetry visualization
+- **Migration Tool** - Migrate data from external sources to PocketBase
+
+## Architecture
+
+```
+┌─────────────────┐ ┌───────────────────┐ ┌────────────┐
+│ Bash Scripts │────▶│ Telemetry Service │────▶│ PocketBase │
+│ (ProxmoxVE/VED) │ │ (this repo) │ │ Database │
+└─────────────────┘ └───────────────────┘ └────────────┘
+```
+
+## Project Structure
+
+```
+├── service.go # Main service, HTTP handlers, rate limiting
+├── cache.go # In-memory and Redis caching
+├── alerts.go # SMTP alert system
+├── dashboard.go # Dashboard HTML generation
+├── migration/
+│ ├── migrate.go # Data migration tool
+│ └── migrate.sh # Migration shell script
+├── Dockerfile # Container build
+├── entrypoint.sh # Container entrypoint with migration support
+└── go.mod # Go module definition
+```
+
+## Related Projects
+
+- [ProxmoxVE](https://github.com/community-scripts/ProxmoxVE) - Proxmox VE Helper Scripts
+- [ProxmoxVED](https://github.com/community-scripts/ProxmoxVED) - Proxmox VE Helper Scripts (Dev)
+
+## Privacy & Compliance
+
+This service is designed with privacy in mind and is **GDPR/DSGVO compliant**:
+
+- ✅ **No personal data** - Only anonymous technical metrics are collected
+- ✅ **No IP logging** - Request logging is disabled by default, IPs are never stored
+- ✅ **Transparent** - All collected fields are documented and the code is open source
+- ✅ **No tracking** - Session IDs are randomly generated and cannot be linked to users
+- ✅ **No third parties** - Data is only stored in our self-hosted PocketBase instance
+
+## License
+
+MIT License - see [LICENSE](LICENSE) file.
diff --git a/misc/data/dashboard.go b/misc/data/dashboard.go
index 1ad89b988..fbfff5ff8 100644
--- a/misc/data/dashboard.go
+++ b/misc/data/dashboard.go
@@ -13,6 +13,8 @@ import (
// DashboardData holds aggregated statistics for the dashboard
type DashboardData struct {
TotalInstalls int `json:"total_installs"`
+ TotalAllTime int `json:"total_all_time"` // Total records in DB (not limited)
+ SampleSize int `json:"sample_size"` // How many records were sampled
SuccessCount int `json:"success_count"`
FailedCount int `json:"failed_count"`
InstallingCount int `json:"installing_count"`
@@ -132,10 +134,15 @@ func (p *PBClient) FetchDashboardData(ctx context.Context, days int, repoSource
}
// Fetch all records for the period
- records, err := p.fetchRecords(ctx, filter)
+ result, err := p.fetchRecords(ctx, filter)
if err != nil {
return nil, err
}
+ records := result.Records
+
+ // Set total counts
+ data.TotalAllTime = result.TotalItems // Actual total in database
+ data.SampleSize = len(records) // How many we actually processed
// Aggregate statistics
appCounts := make(map[string]int)
@@ -257,18 +264,18 @@ func (p *PBClient) FetchDashboardData(ctx context.Context, days int, repoSource
data.SuccessRate = float64(data.SuccessCount) / float64(completed) * 100
}
- // Convert maps to sorted slices (top 10)
- data.TopApps = topN(appCounts, 10)
- data.OsDistribution = topNOs(osCounts, 10)
+ // Convert maps to sorted slices (increased limits for better analytics)
+ data.TopApps = topN(appCounts, 20)
+ data.OsDistribution = topNOs(osCounts, 15)
data.MethodStats = topNMethod(methodCounts, 10)
- data.PveVersions = topNPve(pveCounts, 10)
+ data.PveVersions = topNPve(pveCounts, 15)
data.TypeStats = topNType(typeCounts, 10)
// Error analysis
- data.ErrorAnalysis = buildErrorAnalysis(errorPatterns, 10)
+ data.ErrorAnalysis = buildErrorAnalysis(errorPatterns, 15)
- // Failed apps with failure rates
- data.FailedApps = buildFailedApps(appCounts, appFailures, 10)
+ // Failed apps with failure rates (min 10 installs threshold)
+ data.FailedApps = buildFailedApps(appCounts, appFailures, 15)
// Daily stats for chart
data.DailyStats = buildDailyStats(dailySuccess, dailyFailed, days)
@@ -282,10 +289,10 @@ func (p *PBClient) FetchDashboardData(ctx context.Context, days int, repoSource
data.ErrorCategories = buildErrorCategories(errorCatCounts)
// Top tools
- data.TopTools = buildToolStats(toolCounts, 10)
+ data.TopTools = buildToolStats(toolCounts, 15)
// Top addons
- data.TopAddons = buildAddonStats(addonCounts, 10)
+ data.TopAddons = buildAddonStats(addonCounts, 15)
// Average install duration
if durationCount > 0 {
@@ -308,22 +315,30 @@ type TelemetryRecord struct {
Created string `json:"created"`
}
-func (p *PBClient) fetchRecords(ctx context.Context, filter string) ([]TelemetryRecord, error) {
+// fetchRecordsResult contains records and total count
+type fetchRecordsResult struct {
+ Records []TelemetryRecord
+ TotalItems int // Actual total in database (not limited)
+}
+
+func (p *PBClient) fetchRecords(ctx context.Context, filter string) (*fetchRecordsResult, error) {
var allRecords []TelemetryRecord
page := 1
perPage := 500
+ maxRecords := 100000 // Limit to prevent timeout with large datasets
+ totalItems := 0
for {
- var url string
+ var reqURL string
if filter != "" {
- url = fmt.Sprintf("%s/api/collections/%s/records?filter=%s&sort=-created&page=%d&perPage=%d",
+ reqURL = fmt.Sprintf("%s/api/collections/%s/records?filter=%s&sort=-created&page=%d&perPage=%d",
p.baseURL, p.targetColl, filter, page, perPage)
} else {
- url = fmt.Sprintf("%s/api/collections/%s/records?sort=-created&page=%d&perPage=%d",
+ reqURL = fmt.Sprintf("%s/api/collections/%s/records?sort=-created&page=%d&perPage=%d",
p.baseURL, p.targetColl, page, perPage)
}
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
if err != nil {
return nil, err
}
@@ -344,15 +359,24 @@ func (p *PBClient) fetchRecords(ctx context.Context, filter string) ([]Telemetry
}
resp.Body.Close()
+ // Store total on first page
+ if page == 1 {
+ totalItems = result.TotalItems
+ }
+
allRecords = append(allRecords, result.Items...)
- if len(allRecords) >= result.TotalItems {
+ // Stop if we have enough records or reached the end
+ if len(allRecords) >= maxRecords || len(allRecords) >= result.TotalItems {
break
}
page++
}
- return allRecords, nil
+ return &fetchRecordsResult{
+ Records: allRecords,
+ TotalItems: totalItems,
+ }, nil
}
func topN(m map[string]int, n int) []AppCount {
@@ -531,11 +555,12 @@ func buildErrorAnalysis(patterns map[string]map[string]bool, n int) []ErrorGroup
func buildFailedApps(total, failed map[string]int, n int) []AppFailure {
result := make([]AppFailure, 0)
+ minInstalls := 10 // Minimum installations to be considered (avoid noise from rare apps)
for app, failCount := range failed {
totalCount := total[app]
- if totalCount == 0 {
- continue
+ if totalCount < minInstalls {
+ continue // Skip apps with too few installations
}
rate := float64(failCount) / float64(totalCount) * 100
@@ -673,37 +698,49 @@ func DashboardHTML() string {
- Telemetry Dashboard - ProxmoxVE Helper Scripts
-
+ Analytics - Proxmox VE Helper-Scripts
+
+
+
+
-