0231b72d78
Introduce a telemetry data microservice under misc/data: add Dockerfile, entrypoint, migration tools, README, LICENSE and a .gitignore. Increase Docker CACHE_TTL_SECONDS to 300s. Implement extensive dashboard and analytics updates in dashboard.go: add total_all_time and sample_size, return total item counts from fetchRecords (with page/limit handling and a maxRecords guard), raise top-N limits, add a minimum-installs threshold for failed-apps, and numerous UI/style/layout improvements in the embedded DashboardHTML. Minor formatting tweak to misc/api.func.
78 lines
3.4 KiB
Bash
78 lines
3.4 KiB
Bash
#!/bin/sh
|
|
# Direct SQLite Import - Pure Shell, FAST batch mode!
|
|
# Imports MongoDB Extended JSON directly into PocketBase SQLite
|
|
#
|
|
# Usage:
|
|
# docker cp import-direct.sh pocketbase:/tmp/
|
|
# docker cp data.json pocketbase:/tmp/
|
|
# docker exec -it pocketbase sh -c "cd /tmp && chmod +x import-direct.sh && ./import-direct.sh"
|
|
|
|
set -e
|
|
|
|
JSON_FILE="${1:-/tmp/data.json}"
|
|
TABLE="${2:-telemetry}"
|
|
REPO="${3:-Proxmox VE}"
|
|
DB="${4:-/app/pb_data/data.db}"
|
|
BATCH=5000
|
|
|
|
echo "========================================================="
|
|
echo " Direct SQLite Import (Batch Mode)"
|
|
echo "========================================================="
|
|
echo "JSON: $JSON_FILE"
|
|
echo "Table: $TABLE"
|
|
echo "Repo: $REPO"
|
|
echo "Batch: $BATCH"
|
|
echo "---------------------------------------------------------"
|
|
|
|
# Install jq if missing
|
|
command -v jq >/dev/null || apk add --no-cache jq
|
|
|
|
# Optimize SQLite for bulk
|
|
sqlite3 "$DB" "PRAGMA journal_mode=WAL; PRAGMA synchronous=OFF; PRAGMA cache_size=100000;"
|
|
|
|
SQL_FILE="/tmp/batch.sql"
|
|
echo "[INFO] Converting JSON to SQL..."
|
|
START=$(date +%s)
|
|
|
|
# Convert entire JSON to SQL file (much faster than line-by-line sqlite3 calls)
|
|
{
|
|
echo "BEGIN TRANSACTION;"
|
|
jq -r '.[] | @json' "$JSON_FILE" | while read -r r; do
|
|
CT=$(echo "$r" | jq -r 'if .ct_type|type=="object" then .ct_type["$numberLong"] else .ct_type end // 0')
|
|
DISK=$(echo "$r" | jq -r 'if .disk_size|type=="object" then .disk_size["$numberLong"] else .disk_size end // 0')
|
|
CORE=$(echo "$r" | jq -r 'if .core_count|type=="object" then .core_count["$numberLong"] else .core_count end // 0')
|
|
RAM=$(echo "$r" | jq -r 'if .ram_size|type=="object" then .ram_size["$numberLong"] else .ram_size end // 0')
|
|
OS=$(echo "$r" | jq -r '.os_type // ""' | sed "s/'/''/g")
|
|
OSVER=$(echo "$r" | jq -r '.os_version // ""' | sed "s/'/''/g")
|
|
DIS6=$(echo "$r" | jq -r '.disable_ip6 // "no"' | sed "s/'/''/g")
|
|
APP=$(echo "$r" | jq -r '.nsapp // "unknown"' | sed "s/'/''/g")
|
|
METH=$(echo "$r" | jq -r '.method // ""' | sed "s/'/''/g")
|
|
PVE=$(echo "$r" | jq -r '.pveversion // ""' | sed "s/'/''/g")
|
|
STAT=$(echo "$r" | jq -r '.status // "unknown"')
|
|
[ "$STAT" = "done" ] && STAT="success"
|
|
RID=$(echo "$r" | jq -r '.random_id // ""' | sed "s/'/''/g")
|
|
TYPE=$(echo "$r" | jq -r '.type // "lxc"' | sed "s/'/''/g")
|
|
ERR=$(echo "$r" | jq -r '.error // ""' | sed "s/'/''/g")
|
|
DATE=$(echo "$r" | jq -r 'if .created_at|type=="object" then .created_at["$date"] else .created_at end // ""')
|
|
ID=$(head -c 100 /dev/urandom | tr -dc 'a-z0-9' | head -c 15)
|
|
REPO_ESC=$(echo "$REPO" | sed "s/'/''/g")
|
|
|
|
echo "INSERT OR IGNORE INTO $TABLE (id,created,updated,ct_type,disk_size,core_count,ram_size,os_type,os_version,disableip6,nsapp,method,pve_version,status,random_id,type,error,repo_source) VALUES ('$ID','$DATE','$DATE',$CT,$DISK,$CORE,$RAM,'$OS','$OSVER','$DIS6','$APP','$METH','$PVE','$STAT','$RID','$TYPE','$ERR','$REPO_ESC');"
|
|
done
|
|
echo "COMMIT;"
|
|
} > "$SQL_FILE"
|
|
|
|
MID=$(date +%s)
|
|
echo "[INFO] SQL generated in $((MID - START))s"
|
|
echo "[INFO] Importing into SQLite..."
|
|
|
|
sqlite3 "$DB" < "$SQL_FILE"
|
|
|
|
END=$(date +%s)
|
|
COUNT=$(wc -l < "$SQL_FILE")
|
|
rm -f "$SQL_FILE"
|
|
|
|
echo "========================================================="
|
|
echo "Done! ~$((COUNT - 2)) records in $((END - START)) seconds"
|
|
echo "========================================================="
|