From 313da7c00cfbb4ba4f1224376300f103fe5175c8 Mon Sep 17 00:00:00 2001 From: "CanbiZ (MickLesk)" <47820557+MickLesk@users.noreply.github.com> Date: Mon, 9 Feb 2026 16:06:44 +0100 Subject: [PATCH] Switch telemetry to ingest service Replace direct PocketBase integration with a fire-and-forget telemetry ingest endpoint and tighten validation. misc/api.func: point to telemetry.community-scripts.org, add TELEMETRY_TIMEOUT, use DIAGNOSTICS=no opt-out, include random_id/NSAPP/status in payloads, unify LXC/VM POSTs, avoid blocking or failing scripts, remove PocketBase record lookup/patch logic. misc/data/service.go: update TelemetryIn/TelemetryOut schemas to match new payload, add stricter sanitization and enum/range validation, adjust hashing/deduplication usage, and update request logging to reflect nsapp/status. Overall: safer, non-blocking telemetry with improved schema validation and GDPR-friendly behavior. --- misc/api.func | 207 ++++++++++++++++--------------------------- misc/data/service.go | 153 +++++++++++++++++++++----------- 2 files changed, 177 insertions(+), 183 deletions(-) diff --git a/misc/api.func b/misc/api.func index 4c323a954..6e6224427 100644 --- a/misc/api.func +++ b/misc/api.func @@ -3,11 +3,11 @@ # License: MIT | https://raw.githubusercontent.com/community-scripts/ProxmoxVED/main/LICENSE # ============================================================================== -# API.FUNC - TELEMETRY & DIAGNOSTICS API (PocketBase) +# API.FUNC - TELEMETRY & DIAGNOSTICS API # ============================================================================== # -# Provides functions for sending anonymous telemetry data to PocketBase -# backend at db.community-scripts.org for analytics and diagnostics. +# Provides functions for sending anonymous telemetry data via the community +# telemetry ingest service at telemetry.community-scripts.org. # # Features: # - Container/VM creation statistics @@ -17,26 +17,25 @@ # # Usage: # source <(curl -fsSL .../api.func) -# post_to_api # Report container creation +# post_to_api # Report LXC container creation # post_to_api_vm # Report VM creation # post_update_to_api # Report installation status # # Privacy: # - Only anonymous statistics (no personal data) -# - User can opt-out via diagnostics settings +# - User can opt-out via DIAGNOSTICS=no # - Random UUID for session tracking only +# - Data retention: 30 days # # ============================================================================== # ============================================================================== -# PocketBase Configuration +# Telemetry Configuration # ============================================================================== -PB_URL="http://db.community-scripts.org" -PB_COLLECTION="_dev_telemetry_data" -PB_API_URL="${PB_URL}/api/collections/${PB_COLLECTION}/records" +TELEMETRY_URL="http://telemetry.community-scripts.org/telemetry" -# Store PocketBase record ID for update operations -PB_RECORD_ID="" +# Timeout for telemetry requests (seconds) +TELEMETRY_TIMEOUT=5 # ============================================================================== # SECTION 1: ERROR CODE DESCRIPTIONS @@ -172,8 +171,7 @@ explain_exit_code() { # ------------------------------------------------------------------------------ # post_to_api() # -# - Sends LXC container creation statistics to PocketBase -# - Creates a new record in the _dev_telemetry_data collection +# - Sends LXC container creation statistics to telemetry ingest service # - Only executes if: # * curl is available # * DIAGNOSTICS=yes @@ -186,232 +184,181 @@ explain_exit_code() { # * PVE version # * Status: "installing" # * Random UUID for session tracking -# - Stores PB_RECORD_ID for later updates # - Anonymous telemetry (no personal data) +# - Never blocks or fails script execution # ------------------------------------------------------------------------------ post_to_api() { - if ! command -v curl &>/dev/null; then - return - fi + # Silent fail - telemetry should never break scripts + command -v curl &>/dev/null || return 0 + [[ "${DIAGNOSTICS:-no}" == "no" ]] && return 0 + [[ -z "${RANDOM_UUID:-}" ]] && return 0 - if [[ "${DIAGNOSTICS:-no}" == "no" ]]; then - return - fi + # Set type for later status updates + TELEMETRY_TYPE="lxc" - if [[ -z "${RANDOM_UUID:-}" ]]; then - return - fi - - local pve_version="not found" + local pve_version="" if command -v pveversion &>/dev/null; then - pve_version=$(pveversion | awk -F'[/ ]' '{print $2}') + pve_version=$(pveversion 2>/dev/null | awk -F'[/ ]' '{print $2}') || true fi local JSON_PAYLOAD JSON_PAYLOAD=$( cat </dev/null) || true - - # Extract PocketBase record ID from response for later updates - local http_code body - http_code=$(echo "$RESPONSE" | tail -n1) - body=$(echo "$RESPONSE" | sed '$d') - - if [[ "$http_code" == "200" ]] || [[ "$http_code" == "201" ]]; then - PB_RECORD_ID=$(echo "$body" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) || true - fi + -d "$JSON_PAYLOAD" &>/dev/null || true } # ------------------------------------------------------------------------------ # post_to_api_vm() # -# - Sends VM creation statistics to PocketBase -# - Similar to post_to_api() but for virtual machines (not containers) +# - Sends VM creation statistics to telemetry ingest service # - Reads DIAGNOSTICS from /usr/local/community-scripts/diagnostics file -# - Payload differences: +# - Payload differences from LXC: # * ct_type=2 (VM instead of LXC) # * type="vm" -# * Disk size without 'G' suffix (parsed from DISK_SIZE variable) +# * Disk size without 'G' suffix # - Only executes if DIAGNOSTICS=yes and RANDOM_UUID is set +# - Never blocks or fails script execution # ------------------------------------------------------------------------------ post_to_api_vm() { - if [[ ! -f /usr/local/community-scripts/diagnostics ]]; then - return - fi - DIAGNOSTICS=$(grep -i "^DIAGNOSTICS=" /usr/local/community-scripts/diagnostics | awk -F'=' '{print $2}') - - if ! command -v curl &>/dev/null; then - return + # Read diagnostics setting from file + if [[ -f /usr/local/community-scripts/diagnostics ]]; then + DIAGNOSTICS=$(grep -i "^DIAGNOSTICS=" /usr/local/community-scripts/diagnostics 2>/dev/null | awk -F'=' '{print $2}') || true fi - if [[ "${DIAGNOSTICS:-no}" == "no" ]]; then - return - fi + # Silent fail - telemetry should never break scripts + command -v curl &>/dev/null || return 0 + [[ "${DIAGNOSTICS:-no}" == "no" ]] && return 0 + [[ -z "${RANDOM_UUID:-}" ]] && return 0 - if [[ -z "${RANDOM_UUID:-}" ]]; then - return - fi + # Set type for later status updates + TELEMETRY_TYPE="vm" - local pve_version="not found" + local pve_version="" if command -v pveversion &>/dev/null; then - pve_version=$(pveversion | awk -F'[/ ]' '{print $2}') + pve_version=$(pveversion 2>/dev/null | awk -F'[/ ]' '{print $2}') || true fi + # Remove 'G' suffix from disk size local DISK_SIZE_API="${DISK_SIZE%G}" local JSON_PAYLOAD JSON_PAYLOAD=$( cat </dev/null) || true - - # Extract PocketBase record ID from response for later updates - local http_code body - http_code=$(echo "$RESPONSE" | tail -n1) - body=$(echo "$RESPONSE" | sed '$d') - - if [[ "$http_code" == "200" ]] || [[ "$http_code" == "201" ]]; then - PB_RECORD_ID=$(echo "$body" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) || true - fi + -d "$JSON_PAYLOAD" &>/dev/null || true } # ------------------------------------------------------------------------------ # post_update_to_api() # -# - Reports installation completion status to PocketBase via PATCH +# - Reports installation completion status to telemetry ingest service # - Prevents duplicate submissions via POST_UPDATE_DONE flag # - Arguments: # * $1: status ("done" or "failed") # * $2: exit_code (numeric, default: 1 for failed, 0 for done) -# - Uses PB_RECORD_ID if available, otherwise looks up by random_id # - Payload includes: # * Final status (mapped: "done"→"sucess", "failed"→"failed") # * Error description via explain_exit_code() # * Numeric exit code # - Only executes once per session -# - Silently returns if: -# * curl not available -# * Already reported (POST_UPDATE_DONE=true) -# * DIAGNOSTICS=no +# - Never blocks or fails script execution # ------------------------------------------------------------------------------ post_update_to_api() { - if ! command -v curl &>/dev/null; then - return - fi + # Silent fail - telemetry should never break scripts + command -v curl &>/dev/null || return 0 - # Initialize flag if not set (prevents 'unbound variable' error with set -u) + # Prevent duplicate submissions POST_UPDATE_DONE=${POST_UPDATE_DONE:-false} + [[ "$POST_UPDATE_DONE" == "true" ]] && return 0 - if [[ "$POST_UPDATE_DONE" == "true" ]]; then - return 0 - fi - - if [[ "${DIAGNOSTICS:-no}" == "no" ]]; then - return - fi - - if [[ -z "${RANDOM_UUID:-}" ]]; then - return - fi + [[ "${DIAGNOSTICS:-no}" == "no" ]] && return 0 + [[ -z "${RANDOM_UUID:-}" ]] && return 0 local status="${1:-failed}" local raw_exit_code="${2:-1}" - local exit_code error pb_status + local exit_code=0 error="" pb_status - # Map status to PocketBase select values: installing, sucess, failed, unknown + # Map status to telemetry values: installing, sucess, failed, unknown case "$status" in done | success | sucess) pb_status="sucess" exit_code=0 error="" ;; - failed) pb_status="failed" ;; - *) pb_status="unknown" ;; + failed) + pb_status="failed" + ;; + *) + pb_status="unknown" + ;; esac - # For failed status, resolve exit code and error description + # For failed/unknown status, resolve exit code and error description if [[ "$pb_status" == "failed" ]] || [[ "$pb_status" == "unknown" ]]; then - # If exit_code is numeric, use it; otherwise default to 1 if [[ "$raw_exit_code" =~ ^[0-9]+$ ]]; then exit_code="$raw_exit_code" else exit_code=1 fi error=$(explain_exit_code "$exit_code") - if [[ -z "$error" ]]; then - error="Unknown error" - fi - fi - - # Resolve PocketBase record ID if not already known - local record_id="${PB_RECORD_ID:-}" - - if [[ -z "$record_id" ]]; then - # Look up record by random_id filter - local lookup_url="${PB_API_URL}?filter=(random_id='${RANDOM_UUID}')&fields=id&perPage=1" - local lookup_response - lookup_response=$(curl -s -L "${lookup_url}" 2>/dev/null) || true - - record_id=$(echo "$lookup_response" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) || true - - if [[ -z "$record_id" ]]; then - POST_UPDATE_DONE=true - return - fi + [[ -z "$error" ]] && error="Unknown error" fi local JSON_PAYLOAD JSON_PAYLOAD=$( cat </dev/null || true diff --git a/misc/data/service.go b/misc/data/service.go index 99f322121..08a1cfad9 100644 --- a/misc/data/service.go +++ b/misc/data/service.go @@ -39,36 +39,47 @@ type Config struct { EnableReqLogging bool // default false (GDPR-friendly) } +// TelemetryIn matches payload from api.func (bash client) type TelemetryIn struct { - Script string `json:"script"` - Version string `json:"version"` - Event string `json:"event"` - OsType string `json:"os_type"` - OsVersion string `json:"os_version,omitempty"` + // Required + RandomID string `json:"random_id"` // Session UUID + Type string `json:"type"` // "lxc" or "vm" + NSAPP string `json:"nsapp"` // Application name (e.g., "jellyfin") + Status string `json:"status"` // "installing", "sucess", "failed", "unknown" + + // Container/VM specs + CTType int `json:"ct_type,omitempty"` // 1=unprivileged, 2=privileged/VM + DiskSize int `json:"disk_size,omitempty"` // GB + CoreCount int `json:"core_count,omitempty"` // CPU cores + RAMSize int `json:"ram_size,omitempty"` // MB + + // System info + OsType string `json:"os_type,omitempty"` // "debian", "ubuntu", "alpine", etc. + OsVersion string `json:"os_version,omitempty"` // "12", "24.04", etc. PveVer string `json:"pve_version,omitempty"` - Arch string `json:"arch"` - Method string `json:"method,omitempty"` - Status string `json:"status,omitempty"` - ExitCode int `json:"exit_code,omitempty"` - Error string `json:"error,omitempty"` // must be sanitized/short + + // Optional + Method string `json:"method,omitempty"` // "default", "advanced" + Error string `json:"error,omitempty"` // Error description (max 120 chars) + ExitCode int `json:"exit_code,omitempty"` // 0-255 } +// TelemetryOut is sent to PocketBase (matches _dev_telemetry_data collection) type TelemetryOut struct { - Script string `json:"script"` - Version string `json:"version"` - Event string `json:"event"` - OsType string `json:"os_type"` + RandomID string `json:"random_id"` + Type string `json:"type"` + NSAPP string `json:"nsapp"` + Status string `json:"status"` + CTType int `json:"ct_type,omitempty"` + DiskSize int `json:"disk_size,omitempty"` + CoreCount int `json:"core_count,omitempty"` + RAMSize int `json:"ram_size,omitempty"` + OsType string `json:"os_type,omitempty"` OsVersion string `json:"os_version,omitempty"` PveVer string `json:"pve_version,omitempty"` - Arch string `json:"arch"` Method string `json:"method,omitempty"` - Status string `json:"status,omitempty"` - ExitCode int `json:"exit_code,omitempty"` Error string `json:"error,omitempty"` - - TS int64 `json:"ts"` - IngestDay string `json:"ingest_day"` - Hash string `json:"hash"` + ExitCode int `json:"exit_code,omitempty"` } type PBClient struct { @@ -297,9 +308,21 @@ func getClientIP(r *http.Request, pt *ProxyTrust) net.IP { // -------- Validation (strict allowlist) -------- var ( - allowedEvents = map[string]bool{"install": true, "update": true, "error": true} - allowedOsType = map[string]bool{"pve": true, "lxc": true, "vm": true, "debian": true, "ubuntu": true, "alpine": true} - allowedArch = map[string]bool{"amd64": true, "arm64": true} + // Allowed values for 'type' field + allowedType = map[string]bool{"lxc": true, "vm": true} + + // Allowed values for 'status' field (note: "sucess" is intentional, matches PB schema) + allowedStatus = map[string]bool{"installing": true, "sucess": true, "failed": true, "unknown": true} + + // Allowed values for 'os_type' field + allowedOsType = map[string]bool{ + "debian": true, "ubuntu": true, "alpine": true, "devuan": true, + "fedora": true, "rocky": true, "alma": true, "centos": true, + "opensuse": true, "gentoo": true, "openeuler": true, + } + + // Allowed values for 'method' field + allowedMethod = map[string]bool{"default": true, "advanced": true, "": true} ) func sanitizeShort(s string, max int) string { @@ -317,42 +340,66 @@ func sanitizeShort(s string, max int) string { } func validate(in *TelemetryIn) error { - in.Script = sanitizeShort(in.Script, 64) - in.Version = sanitizeShort(in.Version, 32) - in.Event = sanitizeShort(in.Event, 16) - in.OsType = sanitizeShort(in.OsType, 16) - in.Arch = sanitizeShort(in.Arch, 16) - in.Method = sanitizeShort(in.Method, 32) + // Sanitize all string fields + in.RandomID = sanitizeShort(in.RandomID, 64) + in.Type = sanitizeShort(in.Type, 8) + in.NSAPP = sanitizeShort(in.NSAPP, 64) in.Status = sanitizeShort(in.Status, 16) + in.OsType = sanitizeShort(in.OsType, 32) in.OsVersion = sanitizeShort(in.OsVersion, 32) in.PveVer = sanitizeShort(in.PveVer, 32) + in.Method = sanitizeShort(in.Method, 32) // IMPORTANT: "error" must be short and not contain identifiers/logs in.Error = sanitizeShort(in.Error, 120) - if in.Script == "" || in.Version == "" || in.Event == "" || in.OsType == "" || in.Arch == "" { - return errors.New("missing required fields") + // Required fields + if in.RandomID == "" || in.Type == "" || in.NSAPP == "" || in.Status == "" { + return errors.New("missing required fields: random_id, type, nsapp, status") } - if !allowedEvents[in.Event] { - return errors.New("invalid event") + + // Validate enums + if !allowedType[in.Type] { + return errors.New("invalid type (must be 'lxc' or 'vm')") } - if !allowedOsType[in.OsType] { + if !allowedStatus[in.Status] { + return errors.New("invalid status") + } + + // os_type is optional but if provided must be valid + if in.OsType != "" && !allowedOsType[in.OsType] { return errors.New("invalid os_type") } - if !allowedArch[in.Arch] { - return errors.New("invalid arch") + + // method is optional but if provided must be valid + if !allowedMethod[in.Method] { + return errors.New("invalid method") + } + + // Validate numeric ranges + if in.CTType < 0 || in.CTType > 2 { + return errors.New("invalid ct_type (must be 0, 1, or 2)") + } + if in.DiskSize < 0 || in.DiskSize > 100000 { + return errors.New("invalid disk_size") + } + if in.CoreCount < 0 || in.CoreCount > 256 { + return errors.New("invalid core_count") + } + if in.RAMSize < 0 || in.RAMSize > 1048576 { + return errors.New("invalid ram_size") } - // exit_code only relevant for error, but allow 0..255 if in.ExitCode < 0 || in.ExitCode > 255 { return errors.New("invalid exit_code") } + return nil } +// computeHash generates a hash for deduplication (GDPR-safe, no IP) func computeHash(out TelemetryOut) string { - // hash over non-identifying fields (no IP) to enable dedupe if needed - key := fmt.Sprintf("%s|%s|%s|%s|%s|%s|%d", - out.Script, out.Version, out.Event, out.OsType, out.Arch, out.IngestDay, out.ExitCode, + key := fmt.Sprintf("%s|%s|%s|%s|%d", + out.RandomID, out.NSAPP, out.Type, out.Status, out.ExitCode, ) sum := sha256.Sum256([]byte(key)) return hex.EncodeToString(sum[:]) @@ -447,24 +494,24 @@ func main() { return } - now := time.Now().UTC() + // Map input to PocketBase schema out := TelemetryOut{ - Script: in.Script, - Version: in.Version, - Event: in.Event, + RandomID: in.RandomID, + Type: in.Type, + NSAPP: in.NSAPP, + Status: in.Status, + CTType: in.CTType, + DiskSize: in.DiskSize, + CoreCount: in.CoreCount, + RAMSize: in.RAMSize, OsType: in.OsType, OsVersion: in.OsVersion, PveVer: in.PveVer, - Arch: in.Arch, Method: in.Method, - Status: in.Status, - ExitCode: in.ExitCode, Error: in.Error, - - TS: now.Unix(), - IngestDay: now.Format("2006-01-02"), + ExitCode: in.ExitCode, } - out.Hash = computeHash(out) + _ = computeHash(out) // For future deduplication ctx, cancel := context.WithTimeout(r.Context(), cfg.RequestTimeout) defer cancel() @@ -477,7 +524,7 @@ func main() { } if cfg.EnableReqLogging { - log.Printf("telemetry accepted script=%s event=%s", out.Script, out.Event) + log.Printf("telemetry accepted nsapp=%s status=%s", out.NSAPP, out.Status) } w.WriteHeader(http.StatusAccepted)