mirror of
https://github.com/community-scripts/ProxmoxVED.git
synced 2026-02-25 05:57:26 +00:00
Add telemetry data service and dashboard revamp
Introduce a telemetry data microservice under misc/data: add Dockerfile, entrypoint, migration tools, README, LICENSE and a .gitignore. Increase Docker CACHE_TTL_SECONDS to 300s. Implement extensive dashboard and analytics updates in dashboard.go: add total_all_time and sample_size, return total item counts from fetchRecords (with page/limit handling and a maxRecords guard), raise top-N limits, add a minimum-installs threshold for failed-apps, and numerous UI/style/layout improvements in the embedded DashboardHTML. Minor formatting tweak to misc/api.func.
This commit is contained in:
77
misc/data/migration/import-direct.sh
Normal file
77
misc/data/migration/import-direct.sh
Normal file
@@ -0,0 +1,77 @@
|
||||
#!/bin/sh
|
||||
# Direct SQLite Import - Pure Shell, FAST batch mode!
|
||||
# Imports MongoDB Extended JSON directly into PocketBase SQLite
|
||||
#
|
||||
# Usage:
|
||||
# docker cp import-direct.sh pocketbase:/tmp/
|
||||
# docker cp data.json pocketbase:/tmp/
|
||||
# docker exec -it pocketbase sh -c "cd /tmp && chmod +x import-direct.sh && ./import-direct.sh"
|
||||
|
||||
set -e
|
||||
|
||||
JSON_FILE="${1:-/tmp/data.json}"
|
||||
TABLE="${2:-telemetry}"
|
||||
REPO="${3:-Proxmox VE}"
|
||||
DB="${4:-/app/pb_data/data.db}"
|
||||
BATCH=5000
|
||||
|
||||
echo "========================================================="
|
||||
echo " Direct SQLite Import (Batch Mode)"
|
||||
echo "========================================================="
|
||||
echo "JSON: $JSON_FILE"
|
||||
echo "Table: $TABLE"
|
||||
echo "Repo: $REPO"
|
||||
echo "Batch: $BATCH"
|
||||
echo "---------------------------------------------------------"
|
||||
|
||||
# Install jq if missing
|
||||
command -v jq >/dev/null || apk add --no-cache jq
|
||||
|
||||
# Optimize SQLite for bulk
|
||||
sqlite3 "$DB" "PRAGMA journal_mode=WAL; PRAGMA synchronous=OFF; PRAGMA cache_size=100000;"
|
||||
|
||||
SQL_FILE="/tmp/batch.sql"
|
||||
echo "[INFO] Converting JSON to SQL..."
|
||||
START=$(date +%s)
|
||||
|
||||
# Convert entire JSON to SQL file (much faster than line-by-line sqlite3 calls)
|
||||
{
|
||||
echo "BEGIN TRANSACTION;"
|
||||
jq -r '.[] | @json' "$JSON_FILE" | while read -r r; do
|
||||
CT=$(echo "$r" | jq -r 'if .ct_type|type=="object" then .ct_type["$numberLong"] else .ct_type end // 0')
|
||||
DISK=$(echo "$r" | jq -r 'if .disk_size|type=="object" then .disk_size["$numberLong"] else .disk_size end // 0')
|
||||
CORE=$(echo "$r" | jq -r 'if .core_count|type=="object" then .core_count["$numberLong"] else .core_count end // 0')
|
||||
RAM=$(echo "$r" | jq -r 'if .ram_size|type=="object" then .ram_size["$numberLong"] else .ram_size end // 0')
|
||||
OS=$(echo "$r" | jq -r '.os_type // ""' | sed "s/'/''/g")
|
||||
OSVER=$(echo "$r" | jq -r '.os_version // ""' | sed "s/'/''/g")
|
||||
DIS6=$(echo "$r" | jq -r '.disable_ip6 // "no"' | sed "s/'/''/g")
|
||||
APP=$(echo "$r" | jq -r '.nsapp // "unknown"' | sed "s/'/''/g")
|
||||
METH=$(echo "$r" | jq -r '.method // ""' | sed "s/'/''/g")
|
||||
PVE=$(echo "$r" | jq -r '.pveversion // ""' | sed "s/'/''/g")
|
||||
STAT=$(echo "$r" | jq -r '.status // "unknown"')
|
||||
[ "$STAT" = "done" ] && STAT="success"
|
||||
RID=$(echo "$r" | jq -r '.random_id // ""' | sed "s/'/''/g")
|
||||
TYPE=$(echo "$r" | jq -r '.type // "lxc"' | sed "s/'/''/g")
|
||||
ERR=$(echo "$r" | jq -r '.error // ""' | sed "s/'/''/g")
|
||||
DATE=$(echo "$r" | jq -r 'if .created_at|type=="object" then .created_at["$date"] else .created_at end // ""')
|
||||
ID=$(head -c 100 /dev/urandom | tr -dc 'a-z0-9' | head -c 15)
|
||||
REPO_ESC=$(echo "$REPO" | sed "s/'/''/g")
|
||||
|
||||
echo "INSERT OR IGNORE INTO $TABLE (id,created,updated,ct_type,disk_size,core_count,ram_size,os_type,os_version,disableip6,nsapp,method,pve_version,status,random_id,type,error,repo_source) VALUES ('$ID','$DATE','$DATE',$CT,$DISK,$CORE,$RAM,'$OS','$OSVER','$DIS6','$APP','$METH','$PVE','$STAT','$RID','$TYPE','$ERR','$REPO_ESC');"
|
||||
done
|
||||
echo "COMMIT;"
|
||||
} > "$SQL_FILE"
|
||||
|
||||
MID=$(date +%s)
|
||||
echo "[INFO] SQL generated in $((MID - START))s"
|
||||
echo "[INFO] Importing into SQLite..."
|
||||
|
||||
sqlite3 "$DB" < "$SQL_FILE"
|
||||
|
||||
END=$(date +%s)
|
||||
COUNT=$(wc -l < "$SQL_FILE")
|
||||
rm -f "$SQL_FILE"
|
||||
|
||||
echo "========================================================="
|
||||
echo "Done! ~$((COUNT - 2)) records in $((END - START)) seconds"
|
||||
echo "========================================================="
|
||||
Reference in New Issue
Block a user