1105 lines
41 KiB
Python
1105 lines
41 KiB
Python
"""
|
|
NFQUEUE Python-Scripting API Router
|
|
Endpoints:
|
|
- POST /scripts -> upload script (multipart): script file, optional requirements file, required 'name' form field
|
|
- GET /scripts -> list scripts + per-script unit mappings/status (combined)
|
|
- GET /scripts/{name} -> download script (binary blob)
|
|
- GET /scripts/{name}/requirements -> download requirements file (binary blob) if present
|
|
- PUT /scripts/{name}/requirements -> upload/replace requirements file (multipart). ALWAYS runs pip install and returns pip output.
|
|
- DELETE /scripts/{name}/requirements -> delete only requirements file and remove venv (cleanup)
|
|
- POST /scripts/{name}/enable -> enable systemd service for script on given qnum
|
|
- POST /scripts/{name}/disable -> disable service for script on qnum
|
|
- DELETE /scripts/{name} -> disable all or a specific qnum service(s) and delete script + venv + requirements
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import re
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
import logging
|
|
from typing import Optional, List, Dict, Any, Tuple, DefaultDict
|
|
from collections import defaultdict
|
|
|
|
from fastapi import APIRouter, UploadFile, File, Form, HTTPException, Query
|
|
from fastapi.responses import FileResponse, JSONResponse
|
|
from pydantic import BaseModel, Field
|
|
|
|
# ---------- Configuration ----------
|
|
SCRIPT_DIR = "/srv/fw-scripts"
|
|
REPO_BACKEND_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
|
EXAMPLE_SCRIPT_DIR = os.path.join(REPO_BACKEND_DIR, "example_scripts")
|
|
VENV_BASE = "/srv/fw-scripts/venvs"
|
|
UNIT_DIR = "/etc/systemd/system" # retained for writing new units, but discovery uses systemctl
|
|
UNIT_PREFIX = "fw-script"
|
|
|
|
# ensure dirs exist
|
|
os.makedirs(SCRIPT_DIR, exist_ok=True)
|
|
os.makedirs(EXAMPLE_SCRIPT_DIR, exist_ok=True)
|
|
os.makedirs(VENV_BASE, exist_ok=True)
|
|
|
|
# ---------- Logging ----------
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s [%(name)s] %(message)s")
|
|
logger = logging.getLogger("script-router")
|
|
|
|
# ---------- Router ----------
|
|
router = APIRouter(prefix="/scripts", tags=["scripts"])
|
|
|
|
# ---------- Name validation ----------
|
|
_NAME_RE = re.compile(r'^[A-Za-z0-9_.-]+$')
|
|
|
|
|
|
def validate_name(name: str) -> None:
|
|
if not name:
|
|
raise ValueError("name must be provided")
|
|
if not _NAME_RE.match(name):
|
|
raise ValueError("invalid name; allowed characters: letters, digits, dot, underscore, hyphen")
|
|
if name in (".", ".."):
|
|
raise ValueError("invalid name")
|
|
|
|
|
|
# ---------- Paths ----------
|
|
def script_path_for(name: str) -> str:
|
|
return os.path.join(SCRIPT_DIR, f"{name}.py")
|
|
|
|
|
|
def requirements_path_for(name: str) -> str:
|
|
return os.path.join(SCRIPT_DIR, f"{name}-requirements.txt")
|
|
|
|
|
|
def example_script_path_for(name: str) -> str:
|
|
return os.path.join(EXAMPLE_SCRIPT_DIR, f"{name}.py")
|
|
|
|
|
|
def example_requirements_path_for(name: str) -> str:
|
|
return os.path.join(EXAMPLE_SCRIPT_DIR, f"{name}-requirements.txt")
|
|
|
|
|
|
def example_deploy_config_path_for(name: str) -> str:
|
|
return os.path.join(EXAMPLE_SCRIPT_DIR, f"{name}.deploy.json")
|
|
|
|
|
|
def venv_path_for(name: str) -> str:
|
|
return os.path.join(VENV_BASE, name)
|
|
|
|
|
|
def venv_python_for(name: str) -> str:
|
|
vpy = os.path.join(venv_path_for(name), "bin", "python")
|
|
if os.path.exists(vpy):
|
|
return vpy
|
|
return "/usr/bin/python3"
|
|
|
|
|
|
def make_service_name(name: str, qnum: int) -> str:
|
|
# safe, deterministic service name
|
|
return f"{UNIT_PREFIX}-{name}-q{qnum}"
|
|
|
|
|
|
def unit_path_for_name(service_name: str) -> str:
|
|
# default location for units we write
|
|
return os.path.join(UNIT_DIR, service_name + ".service")
|
|
|
|
|
|
def is_example_script(name: str) -> bool:
|
|
return os.path.exists(example_script_path_for(name))
|
|
|
|
|
|
def assert_not_example_script(name: str, action: str) -> None:
|
|
if is_example_script(name):
|
|
raise HTTPException(status_code=403, detail=f"example script '{name}' is protected and cannot be {action} via API")
|
|
|
|
|
|
def _read_example_deploy_config(name: str) -> Dict[str, Any]:
|
|
cfg_path = example_deploy_config_path_for(name)
|
|
if not os.path.exists(cfg_path):
|
|
return {}
|
|
try:
|
|
with open(cfg_path, "r", encoding="utf-8") as fh:
|
|
data = json.load(fh)
|
|
if not isinstance(data, dict):
|
|
logger.warning("Deploy config for %s is not an object: %s", name, cfg_path)
|
|
return {}
|
|
return data
|
|
except Exception:
|
|
logger.exception("Failed reading deploy config for %s at %s", name, cfg_path)
|
|
return {}
|
|
|
|
|
|
def deploy_example_scripts_on_startup() -> None:
|
|
"""
|
|
Sync example scripts from EXAMPLE_SCRIPT_DIR into SCRIPT_DIR and auto-deploy units
|
|
if an optional '<name>.deploy.json' contains a queue number.
|
|
|
|
Optional config schema per script:
|
|
{
|
|
"qnum": 42,
|
|
"extra_args": "--flag value",
|
|
"enable_at_boot": true,
|
|
"service_name": "fw-script-<name>-q<qnum>"
|
|
}
|
|
service_name is optional and must start with 'fw-script-'.
|
|
"""
|
|
if not os.path.isdir(EXAMPLE_SCRIPT_DIR):
|
|
return
|
|
|
|
for fn in os.listdir(EXAMPLE_SCRIPT_DIR):
|
|
if not fn.endswith(".py"):
|
|
continue
|
|
|
|
name = fn.rsplit(".", 1)[0]
|
|
src_script = os.path.join(EXAMPLE_SCRIPT_DIR, fn)
|
|
dst_script = script_path_for(name)
|
|
|
|
try:
|
|
shutil.copy2(src_script, dst_script)
|
|
os.chmod(dst_script, 0o700)
|
|
logger.info("Synced example script %s -> %s", src_script, dst_script)
|
|
except Exception:
|
|
logger.exception("Failed syncing example script %s", src_script)
|
|
continue
|
|
|
|
src_req = example_requirements_path_for(name)
|
|
dst_req = requirements_path_for(name)
|
|
if os.path.exists(src_req):
|
|
try:
|
|
shutil.copy2(src_req, dst_req)
|
|
logger.info("Synced example requirements %s -> %s", src_req, dst_req)
|
|
pip_install_requirements(name, dst_req)
|
|
except Exception:
|
|
logger.exception("Failed syncing/installing requirements for example %s", name)
|
|
|
|
cfg = _read_example_deploy_config(name)
|
|
qnum_raw = cfg.get("qnum")
|
|
if qnum_raw is None:
|
|
continue
|
|
|
|
try:
|
|
qnum = int(qnum_raw)
|
|
except (TypeError, ValueError):
|
|
logger.warning("Invalid qnum in deploy config for %s: %r", name, qnum_raw)
|
|
continue
|
|
|
|
if qnum < 0 or qnum > 65535:
|
|
logger.warning("Out-of-range qnum in deploy config for %s: %d", name, qnum)
|
|
continue
|
|
|
|
default_service_name = make_service_name(name, qnum)
|
|
configured_service_name = cfg.get("service_name")
|
|
if configured_service_name:
|
|
if str(configured_service_name).startswith(UNIT_PREFIX + "-"):
|
|
service_name = str(configured_service_name)
|
|
else:
|
|
logger.warning(
|
|
"Ignoring invalid service_name for example %s: %r (must start with '%s-'). Falling back to %s",
|
|
name,
|
|
configured_service_name,
|
|
UNIT_PREFIX,
|
|
default_service_name,
|
|
)
|
|
service_name = default_service_name
|
|
else:
|
|
service_name = default_service_name
|
|
extra_args = cfg.get("extra_args") or ""
|
|
enable_at_boot = bool(cfg.get("enable_at_boot", True))
|
|
|
|
python_path = venv_python_for(name)
|
|
exec_start = f"{python_path} {dst_script} {qnum}"
|
|
if extra_args:
|
|
exec_start += " " + str(extra_args)
|
|
|
|
try:
|
|
write_unit(service_name, exec_start, description=f"FW example script {name} queue {qnum}", enable_at_boot=enable_at_boot)
|
|
time.sleep(0.05)
|
|
start_unit(service_name)
|
|
logger.info("Auto-deployed example script %s as %s", name, service_name)
|
|
except Exception:
|
|
logger.exception("Failed auto-deploying example script %s", name)
|
|
|
|
|
|
# ---------- systemd interaction (systemctl-based, no fallback) ----------
|
|
def _systemctl_unit_name(unit: str) -> str:
|
|
"""Return unit with .service suffix if missing."""
|
|
return unit if unit.endswith(".service") else unit + ".service"
|
|
|
|
|
|
def list_fw_units() -> List[str]:
|
|
"""
|
|
Return list of systemd units (without .service suffix) whose name starts with UNIT_PREFIX-.
|
|
Uses `systemctl list-units` to be canonical.
|
|
"""
|
|
units: List[str] = []
|
|
try:
|
|
p = subprocess.run(
|
|
["systemctl", "list-units", "--type=service", "--all", "--no-legend"],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
timeout=3,
|
|
)
|
|
out = p.stdout or ""
|
|
# each line starts with unit name
|
|
for line in out.splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
cols = line.split()
|
|
unit = cols[0]
|
|
if unit.startswith(UNIT_PREFIX + "-") and unit.endswith(".service"):
|
|
units.append(unit[:-8]) # strip ".service"
|
|
except Exception:
|
|
logger.exception("systemctl list-units failed")
|
|
return units
|
|
|
|
|
|
def get_unit_fragment_path(service_name: str) -> Optional[str]:
|
|
"""
|
|
Use `systemctl show -p FragmentPath --value` to obtain the unit file path (if any).
|
|
Returns None if unknown / empty.
|
|
"""
|
|
unit = _systemctl_unit_name(service_name)
|
|
try:
|
|
p = subprocess.run(["systemctl", "show", "-p", "FragmentPath", "--value", unit],
|
|
capture_output=True, text=True, check=False, timeout=2)
|
|
frag = (p.stdout or "").strip()
|
|
if not frag:
|
|
return None
|
|
return frag
|
|
except Exception:
|
|
logger.exception("systemctl show FragmentPath failed for %s", service_name)
|
|
return None
|
|
|
|
|
|
class UnitParsed(BaseModel):
|
|
service: str
|
|
exec_start: Optional[str] = None
|
|
name: Optional[str] = None
|
|
script_path: Optional[str] = None
|
|
qnum: Optional[int] = None
|
|
extra: Optional[str] = None
|
|
|
|
|
|
class UnitStatus(BaseModel):
|
|
parsed: UnitParsed
|
|
active: bool = False
|
|
|
|
|
|
def parse_unit_execstart(service_name: str) -> UnitParsed:
|
|
"""
|
|
Parser for ExecStart using systemctl --no-pager show.
|
|
- reconstructs argv[] entries if systemd returns a structured blob
|
|
- finds /srv/fw-scripts/<name>.py anywhere in the commandline
|
|
- extracts the first integer after the script as qnum
|
|
Returns a UnitParsed instance.
|
|
"""
|
|
unit = service_name if service_name.endswith(".service") else service_name + ".service"
|
|
|
|
try:
|
|
p = subprocess.run(
|
|
["systemctl", "--no-pager", "show", "-p", "ExecStart", "--value", unit],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
timeout=3,
|
|
)
|
|
raw = (p.stdout or "").strip()
|
|
except Exception:
|
|
logger.exception("systemctl show ExecStart failed for %s", service_name)
|
|
return UnitParsed(service=service_name)
|
|
|
|
if not raw:
|
|
return UnitParsed(service=service_name)
|
|
|
|
exec_start: str = raw
|
|
|
|
# If structured with argv[] entries, extract them (allow newlines).
|
|
if "argv[]=" in raw:
|
|
argv_entries = re.findall(r'argv\[\]=([^;]+)', raw, flags=re.DOTALL)
|
|
argv_parts: List[str] = []
|
|
for a in argv_entries:
|
|
v = a.strip()
|
|
# strip surrounding quotes if present
|
|
if (v.startswith('"') and v.endswith('"')) or (v.startswith("'") and v.endswith("'")):
|
|
v = v[1:-1]
|
|
if v:
|
|
argv_parts.append(v)
|
|
if argv_parts:
|
|
exec_start = " ".join(argv_parts)
|
|
|
|
# Look for /srv/fw-scripts/<name>.py directly
|
|
m = re.search(r'(/srv/fw-scripts/(?P<name>[A-Za-z0-9_.-]+)\.py)\b', exec_start)
|
|
if not m:
|
|
# return raw exec_start but no parsed metadata
|
|
return UnitParsed(service=service_name, exec_start=exec_start)
|
|
|
|
script_path = m.group(1)
|
|
name = m.group("name")
|
|
|
|
# Find first integer token after the script path (queue number)
|
|
after = exec_start[m.end():].strip()
|
|
qnum: Optional[int] = None
|
|
extra: Optional[str] = None
|
|
if after:
|
|
tokens = after.split()
|
|
for i, t in enumerate(tokens):
|
|
try:
|
|
val = int(t)
|
|
if 0 <= val <= 65535:
|
|
qnum = val
|
|
extra = " ".join(tokens[i+1:]) if i + 1 < len(tokens) else ""
|
|
break
|
|
except ValueError:
|
|
continue
|
|
|
|
return UnitParsed(service=service_name, exec_start=exec_start, name=name, script_path=script_path, qnum=qnum, extra=extra)
|
|
|
|
|
|
def is_unit_active(service_name: str) -> bool:
|
|
unit = _systemctl_unit_name(service_name)
|
|
p = subprocess.run(["systemctl", "is-active", "--quiet", unit])
|
|
return p.returncode == 0
|
|
|
|
|
|
def write_unit(service_name: str, exec_start: str, description: str = "", enable_at_boot: bool = False) -> str:
|
|
"""
|
|
Write unit file to default UNIT_DIR and daemon-reload. This writes to disk as systemd expects.
|
|
Returns the path to the unit file written.
|
|
"""
|
|
unit_path = unit_path_for_name(service_name)
|
|
unit_text = f"""[Unit]
|
|
Description={description}
|
|
After=network.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
ExecStart={exec_start}
|
|
Restart=always
|
|
RestartSec=2
|
|
StandardOutput=syslog
|
|
StandardError=syslog
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
"""
|
|
with open(unit_path, "w") as fh:
|
|
fh.write(unit_text)
|
|
subprocess.run(["systemctl", "daemon-reload"], check=True)
|
|
logger.info("Wrote unit %s", unit_path)
|
|
if enable_at_boot:
|
|
try:
|
|
subprocess.run(["systemctl", "enable", _systemctl_unit_name(service_name)], check=True)
|
|
logger.info("Enabled %s at boot", service_name)
|
|
except subprocess.CalledProcessError:
|
|
logger.warning("Failed to enable %s at boot", service_name)
|
|
return unit_path
|
|
|
|
|
|
def start_unit(service_name: str) -> None:
|
|
subprocess.run(["systemctl", "start", _systemctl_unit_name(service_name)], check=True)
|
|
logger.info("Started service %s", service_name)
|
|
|
|
|
|
def stop_unit(service_name: str) -> None:
|
|
subprocess.run(["systemctl", "stop", _systemctl_unit_name(service_name)], check=True)
|
|
logger.info("Stopped service %s", service_name)
|
|
|
|
|
|
def disable_unit(service_name: str) -> None:
|
|
subprocess.run(["systemctl", "disable", _systemctl_unit_name(service_name)], check=False)
|
|
logger.info("Disabled service %s", service_name)
|
|
|
|
|
|
def remove_unit(service_name: str) -> None:
|
|
"""
|
|
Stop + disable the unit, remove the unit file using FragmentPath (if present),
|
|
and daemon-reload. All via systemctl queries (no fallback to scanning /etc).
|
|
"""
|
|
unit = _systemctl_unit_name(service_name)
|
|
try:
|
|
subprocess.run(["systemctl", "stop", unit], check=False)
|
|
except Exception:
|
|
logger.exception("Failed stopping %s", unit)
|
|
try:
|
|
subprocess.run(["systemctl", "disable", unit], check=False)
|
|
except Exception:
|
|
pass
|
|
|
|
frag = get_unit_fragment_path(service_name)
|
|
if frag:
|
|
try:
|
|
if os.path.exists(frag):
|
|
os.remove(frag)
|
|
logger.info("Removed unit file at %s for %s", frag, service_name)
|
|
except Exception:
|
|
logger.exception("Failed to remove unit file %s", frag)
|
|
else:
|
|
logger.debug("No FragmentPath for %s; nothing to delete on-disk", service_name)
|
|
|
|
# ensure systemd reloads units
|
|
try:
|
|
subprocess.run(["systemctl", "daemon-reload"], check=True)
|
|
except Exception:
|
|
logger.exception("daemon-reload failed after removing unit %s", service_name)
|
|
|
|
|
|
# ---------- venv + pip helpers ----------
|
|
def create_venv(name: str, timeout: int = 60) -> str:
|
|
venv_dir = venv_path_for(name)
|
|
if os.path.exists(venv_dir):
|
|
logger.debug("Venv already exists for %s", name)
|
|
return venv_dir
|
|
os.makedirs(os.path.dirname(venv_dir), exist_ok=True)
|
|
logger.info("Creating venv for %s at %s", name, venv_dir)
|
|
try:
|
|
subprocess.run([sys.executable, "-m", "venv", venv_dir], check=True, timeout=timeout, capture_output=True, text=True)
|
|
except subprocess.CalledProcessError as e:
|
|
logger.exception("venv creation failed for %s: %s", name, e.stderr if hasattr(e, "stderr") else str(e))
|
|
raise RuntimeError("venv creation failed: " + (e.stderr or str(e)))
|
|
except subprocess.TimeoutExpired:
|
|
logger.exception("venv creation timed out for %s", name)
|
|
raise RuntimeError("venv creation timed out")
|
|
return venv_dir
|
|
|
|
|
|
def jsonify_cmd_output(out: Dict[str, str]) -> str:
|
|
s = ""
|
|
if out.get("stdout"):
|
|
s += "STDOUT:\n" + out["stdout"] + "\n"
|
|
if out.get("stderr"):
|
|
s += "STDERR:\n" + out["stderr"] + "\n"
|
|
return s.strip()
|
|
|
|
|
|
def pip_install_requirements(name: str, requirements_path: str, timeout: int = 600) -> Dict[str, str]:
|
|
venv_dir = create_venv(name)
|
|
pip_path = os.path.join(venv_dir, "bin", "pip")
|
|
try:
|
|
subprocess.run([pip_path, "install", "--upgrade", "pip"], check=True, capture_output=True, text=True, timeout=300)
|
|
except subprocess.CalledProcessError as e:
|
|
logger.warning("pip upgrade warning for %s: %s", name, getattr(e, "stderr", str(e)))
|
|
try:
|
|
p = subprocess.run([pip_path, "install", "-r", requirements_path, "--no-cache-dir"],
|
|
check=True, capture_output=True, text=True, timeout=timeout)
|
|
logger.info("pip install succeeded for %s", name)
|
|
return {"stdout": p.stdout or "", "stderr": p.stderr or ""}
|
|
except subprocess.CalledProcessError as e:
|
|
out = {"stdout": getattr(e, "stdout", "") or "", "stderr": getattr(e, "stderr", "") or str(e)}
|
|
logger.error("pip install failed for %s: %s", name, out["stderr"][:4000])
|
|
raise RuntimeError(jsonify_cmd_output(out))
|
|
except subprocess.TimeoutExpired:
|
|
logger.error("pip install timed out for %s", name)
|
|
raise RuntimeError("pip install timed out")
|
|
|
|
|
|
# ---------- Models ----------
|
|
class ScriptInfo(BaseModel):
|
|
name: str
|
|
path: str
|
|
|
|
|
|
class UnitMapping(BaseModel):
|
|
service: str
|
|
parsed: UnitParsed
|
|
active: bool = False
|
|
|
|
|
|
class ScriptWithStatus(ScriptInfo):
|
|
"""
|
|
Represents a script plus discovered unit mappings (if any).
|
|
- mappings: list of UnitMapping for that script
|
|
- requirements_exists: whether a requirements file exists on disk
|
|
- is_protected_example: script originates from repository example folder and is immutable via API
|
|
- requirements_is_protected_example: requirements file originates from repository example folder and is immutable via API
|
|
"""
|
|
mappings: List[UnitMapping] = Field(default_factory=list)
|
|
requirements_exists: bool = False
|
|
is_protected_example: bool = False
|
|
requirements_is_protected_example: bool = False
|
|
|
|
|
|
class ScriptUploadResponse(ScriptInfo):
|
|
pip: Optional[Dict[str, str]] = None
|
|
|
|
|
|
class EnableRequest(BaseModel):
|
|
qnum: int = Field(..., description="Queue number (integer)")
|
|
service_name: Optional[str] = None
|
|
extra_args: Optional[str] = None
|
|
enable_at_boot: Optional[bool] = False
|
|
|
|
|
|
class StatusForNameResponse(BaseModel):
|
|
name: str
|
|
mappings: List[UnitMapping]
|
|
|
|
|
|
class OperationResult(BaseModel):
|
|
status: str
|
|
name: str
|
|
qnum: Optional[int] = None
|
|
service: Optional[str] = None
|
|
python: Optional[str] = None
|
|
|
|
|
|
# ---------- Helper: group parsed units by script name ----------
|
|
def _group_units_by_script(units: List[str]) -> Dict[str, List[Tuple[str, UnitParsed, bool]]]:
|
|
"""
|
|
Parse each unit and group by parsed.name (script name). Returns a dict:
|
|
{ script_name: [ (svc, parsed, active), ... ], ... }
|
|
If parsed.name is None, attempt to extract script name from service string using pattern.
|
|
"""
|
|
groups: DefaultDict[str, List[Tuple[str, UnitParsed, bool]]] = defaultdict(list)
|
|
for svc in units:
|
|
parsed = parse_unit_execstart(svc)
|
|
try:
|
|
active = is_unit_active(svc)
|
|
except Exception:
|
|
active = False
|
|
|
|
script_name = parsed.name
|
|
if not script_name:
|
|
# attempt to extract from service name like 'fw-script-<name>-q<N>'
|
|
m = re.match(rf'^{re.escape(UNIT_PREFIX)}-(?P<name>[A-Za-z0-9_.-]+)-q\d+$', svc)
|
|
if m:
|
|
script_name = m.group("name")
|
|
if not script_name:
|
|
# fallback: group under special key so they won't be lost; use svc as key
|
|
script_name = svc
|
|
|
|
groups[script_name].append((svc, parsed, active))
|
|
return groups
|
|
|
|
|
|
# ---------- Endpoints ----------
|
|
# NOTE: combined status info into GET /scripts below (replaces separate /status & /{name}/status endpoints)
|
|
|
|
@router.get("", response_model=List[ScriptWithStatus])
|
|
def list_scripts_with_status() -> List[ScriptWithStatus]:
|
|
"""
|
|
Return list of scripts plus per-script unit mappings/status.
|
|
This combines the former /scripts and /scripts/status endpoints so clients get everything in one call.
|
|
Each entry contains:
|
|
- name, path
|
|
- mappings: list of UnitMapping (service, parsed ExecStart, active flag)
|
|
- requirements_exists: boolean
|
|
- is_protected_example: boolean
|
|
- requirements_is_protected_example: boolean
|
|
"""
|
|
out: List[ScriptWithStatus] = []
|
|
|
|
# build a list of units once for efficiency and group them by parsed script name
|
|
all_units = list_fw_units()
|
|
logger.debug("list_scripts_with_status: discovered %d fw units", len(all_units))
|
|
grouped = _group_units_by_script(all_units)
|
|
|
|
# iterate script files on disk
|
|
for fn in os.listdir(SCRIPT_DIR):
|
|
if not fn.endswith(".py"):
|
|
continue
|
|
name = fn.rsplit(".", 1)[0]
|
|
spath = os.path.join(SCRIPT_DIR, fn)
|
|
|
|
mappings: List[UnitMapping] = []
|
|
# take groups[name] if present
|
|
entries = grouped.get(name, [])
|
|
for svc, parsed, active in entries:
|
|
mappings.append(UnitMapping(service=svc, parsed=parsed, active=active))
|
|
|
|
req_exists = os.path.exists(requirements_path_for(name))
|
|
out.append(
|
|
ScriptWithStatus(
|
|
name=name,
|
|
path=spath,
|
|
mappings=mappings,
|
|
requirements_exists=req_exists,
|
|
is_protected_example=is_example_script(name),
|
|
requirements_is_protected_example=os.path.exists(example_requirements_path_for(name)),
|
|
)
|
|
)
|
|
|
|
logger.debug("Listed %d scripts with status", len(out))
|
|
return out
|
|
|
|
|
|
@router.post("", response_model=ScriptUploadResponse)
|
|
async def upload_script(
|
|
script: UploadFile = File(...),
|
|
name: str = Form(...),
|
|
requirements: Optional[UploadFile] = File(None),
|
|
) -> ScriptUploadResponse:
|
|
"""
|
|
Upload a script with supplied 'name' and optional requirements file.
|
|
On pip/venv install failure, cleanup uploaded files and venv and return 500 with details.
|
|
|
|
NOTE: This endpoint is multipart/form-data (UploadFile), so Swagger UI will show file upload widgets.
|
|
"""
|
|
# validate name
|
|
try:
|
|
validate_name(name)
|
|
except ValueError as e:
|
|
logger.warning("Invalid name provided: %s", name)
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
|
assert_not_example_script(name, "overwritten")
|
|
|
|
spath = script_path_for(name)
|
|
if os.path.exists(spath):
|
|
logger.warning("Upload rejected: script with name already exists: %s", name)
|
|
raise HTTPException(status_code=409, detail="script with that name already exists")
|
|
|
|
data = await script.read()
|
|
try:
|
|
with open(spath, "wb") as fh:
|
|
fh.write(data)
|
|
os.chmod(spath, 0o700)
|
|
logger.info("Saved script for name=%s at %s", name, spath)
|
|
except Exception as e:
|
|
logger.exception("Failed to write script file for %s: %s", name, e)
|
|
raise HTTPException(status_code=500, detail="failed to save script")
|
|
|
|
req_path: Optional[str] = None
|
|
pip_output: Optional[Dict[str, str]] = None
|
|
venv_created = False
|
|
|
|
try:
|
|
if requirements is not None:
|
|
req_data = await requirements.read()
|
|
req_path = requirements_path_for(name)
|
|
with open(req_path, "wb") as fh:
|
|
fh.write(req_data)
|
|
logger.info("Saved requirements for %s at %s", name, req_path)
|
|
try:
|
|
# create venv (if needed) and run pip install; pip output is returned in response
|
|
create_venv(name)
|
|
venv_created = True
|
|
res = pip_install_requirements(name, req_path)
|
|
pip_output = {"stdout": res.get("stdout", ""), "stderr": res.get("stderr", "")}
|
|
logger.info("pip install completed for %s", name)
|
|
except Exception as pip_exc:
|
|
err_msg = str(pip_exc)
|
|
logger.error("pip install error for %s: %s", name, err_msg[:2000])
|
|
# cleanup: remove script, req file, partial venv
|
|
try:
|
|
if os.path.exists(spath):
|
|
os.remove(spath)
|
|
if req_path and os.path.exists(req_path):
|
|
os.remove(req_path)
|
|
if venv_created and os.path.isdir(venv_path_for(name)):
|
|
shutil.rmtree(venv_path_for(name), ignore_errors=True)
|
|
except Exception:
|
|
logger.exception("Cleanup after pip failure partially failed for %s", name)
|
|
# Return pip failure as 500 with message
|
|
raise HTTPException(status_code=500, detail=f"pip install failed: {err_msg}")
|
|
else:
|
|
# If no requirements provided on upload, we simply keep uploaded script. (Per earlier conversation you can require requirements always if you want.)
|
|
logger.debug("No requirements uploaded with script %s", name)
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.exception("Unexpected error during upload for %s: %s", name, e)
|
|
try:
|
|
if os.path.exists(spath):
|
|
os.remove(spath)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
if req_path and os.path.exists(req_path):
|
|
os.remove(req_path)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
if os.path.isdir(venv_path_for(name)):
|
|
shutil.rmtree(venv_path_for(name), ignore_errors=True)
|
|
except Exception:
|
|
pass
|
|
raise HTTPException(status_code=500, detail="internal error during upload")
|
|
|
|
resp: Dict[str, Any] = {"name": name, "path": spath}
|
|
if pip_output is not None:
|
|
resp["pip"] = pip_output
|
|
return ScriptUploadResponse(**resp)
|
|
|
|
|
|
@router.get(
|
|
"/{name}",
|
|
response_class=FileResponse,
|
|
responses={
|
|
200: {
|
|
"content": {"application/octet-stream": {"schema": {"type": "string", "format": "binary"}}},
|
|
"description": "Python script file (binary).",
|
|
"headers": {
|
|
"X-Script-Is-Protected-Example": {"schema": {"type": "string"}, "description": "'true' when script is a protected example"},
|
|
"X-Requirements-Is-Protected-Example": {"schema": {"type": "string"}, "description": "'true' when requirements file is a protected example"},
|
|
},
|
|
},
|
|
404: {"description": "Not found"},
|
|
400: {"description": "Invalid name"},
|
|
},
|
|
)
|
|
def download_script(name: str) -> FileResponse:
|
|
"""
|
|
Download a script as a binary file. Clients should request raw bytes (blob).
|
|
Swagger/OpenAPI will document the response as binary.
|
|
"""
|
|
try:
|
|
validate_name(name)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
path = script_path_for(name)
|
|
if not os.path.exists(path):
|
|
logger.warning("Download requested for missing script %s", name)
|
|
raise HTTPException(status_code=404, detail="not found")
|
|
logger.info("Download script %s", name)
|
|
|
|
is_protected = is_example_script(name)
|
|
req_is_protected = os.path.exists(example_requirements_path_for(name))
|
|
|
|
# Return generic octet-stream so clients treat as binary blob.
|
|
return FileResponse(
|
|
path,
|
|
media_type="application/octet-stream",
|
|
filename=f"{name}.py",
|
|
headers={
|
|
"X-Script-Is-Protected-Example": "true" if is_protected else "false",
|
|
"X-Requirements-Is-Protected-Example": "true" if req_is_protected else "false",
|
|
},
|
|
)
|
|
|
|
|
|
@router.get(
|
|
"/{name}/requirements",
|
|
response_class=FileResponse,
|
|
responses={
|
|
200: {
|
|
"content": {"application/octet-stream": {"schema": {"type": "string", "format": "binary"}}},
|
|
"description": "requirements.txt file (binary).",
|
|
"headers": {
|
|
"X-Script-Is-Protected-Example": {"schema": {"type": "string"}, "description": "'true' when script is a protected example"},
|
|
"X-Requirements-Is-Protected-Example": {"schema": {"type": "string"}, "description": "'true' when requirements file is a protected example"},
|
|
},
|
|
},
|
|
404: {"description": "Not found"},
|
|
400: {"description": "Invalid name"},
|
|
},
|
|
)
|
|
def download_requirements(name: str) -> FileResponse:
|
|
"""
|
|
Download the stored requirements file for a script as binary blob.
|
|
"""
|
|
try:
|
|
validate_name(name)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
path = requirements_path_for(name)
|
|
if not os.path.exists(path):
|
|
raise HTTPException(status_code=404, detail="requirements not found")
|
|
logger.info("Download requirements for %s", name)
|
|
is_protected = is_example_script(name)
|
|
req_is_protected = os.path.exists(example_requirements_path_for(name))
|
|
return FileResponse(
|
|
path,
|
|
media_type="application/octet-stream",
|
|
filename=f"{name}-requirements.txt",
|
|
headers={
|
|
"X-Script-Is-Protected-Example": "true" if is_protected else "false",
|
|
"X-Requirements-Is-Protected-Example": "true" if req_is_protected else "false",
|
|
},
|
|
)
|
|
|
|
|
|
@router.put(
|
|
"/{name}/requirements",
|
|
responses={
|
|
200: {"description": "Requirements replaced and pip output returned"},
|
|
400: {"description": "Invalid name or bad request"},
|
|
500: {"description": "pip install failed or storage error"},
|
|
},
|
|
)
|
|
async def upload_requirements_install(name: str, requirements: UploadFile = File(...)) -> Dict[str, Any]:
|
|
"""
|
|
Replace / upload the requirements file for a given script.
|
|
- requirements: multipart file upload (UploadFile) — keeps Swagger UI file input.
|
|
- This endpoint ALWAYS runs pip install -r <file> into the script's venv and returns pip stdout/stderr.
|
|
If pip install fails, a 500 error is returned with the pip failure message.
|
|
"""
|
|
try:
|
|
validate_name(name)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
|
assert_not_example_script(name, "modified")
|
|
|
|
# ensure script exists (we don't allow attaching requirements to non-existing script)
|
|
spath = script_path_for(name)
|
|
if not os.path.exists(spath):
|
|
raise HTTPException(status_code=404, detail="script not found")
|
|
|
|
if not requirements:
|
|
raise HTTPException(status_code=400, detail="requirements file required")
|
|
|
|
req_path = requirements_path_for(name)
|
|
try:
|
|
data = await requirements.read()
|
|
with open(req_path, "wb") as fh:
|
|
fh.write(data)
|
|
logger.info("Saved requirements for %s at %s", name, req_path)
|
|
except Exception as e:
|
|
logger.exception("Failed to write requirements for %s: %s", name, e)
|
|
raise HTTPException(status_code=500, detail="failed to save requirements")
|
|
|
|
# Now ALWAYS install and return pip output (raise 500 on failure)
|
|
try:
|
|
create_venv(name)
|
|
res = pip_install_requirements(name, req_path)
|
|
logger.info("pip install completed for %s via requirements upload", name)
|
|
return {"pip": {"stdout": res.get("stdout", ""), "stderr": res.get("stderr", "")}}
|
|
except Exception as e:
|
|
logger.exception("pip install failed for %s: %s", name, e)
|
|
# keep the requirements file for inspection; return 500 with details
|
|
raise HTTPException(status_code=500, detail=f"pip install failed: {str(e)}")
|
|
|
|
|
|
@router.delete(
|
|
"/{name}/requirements",
|
|
responses={
|
|
200: {"description": "requirements removed and venv cleaned up"},
|
|
404: {"description": "script or requirements not found"},
|
|
500: {"description": "cleanup error"},
|
|
},
|
|
)
|
|
def delete_requirements_and_cleanup(name: str) -> Dict[str, Any]:
|
|
"""
|
|
Delete only the requirements file for a script and attempt to remove the script's venv directory.
|
|
Returns a summary of what was removed and any errors.
|
|
"""
|
|
try:
|
|
validate_name(name)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
|
assert_not_example_script(name, "modified")
|
|
|
|
spath = script_path_for(name)
|
|
if not os.path.exists(spath):
|
|
raise HTTPException(status_code=404, detail="script not found")
|
|
|
|
rpath = requirements_path_for(name)
|
|
vpath = venv_path_for(name)
|
|
|
|
removed = {"requirements_removed": False, "venv_removed": False}
|
|
errors: List[str] = []
|
|
|
|
# remove requirements file
|
|
try:
|
|
if os.path.exists(rpath):
|
|
os.remove(rpath)
|
|
removed["requirements_removed"] = True
|
|
logger.info("Removed requirements file %s", rpath)
|
|
else:
|
|
logger.debug("No requirements file to remove for %s", name)
|
|
except Exception as e:
|
|
logger.exception("Failed removing requirements file %s: %s", rpath, e)
|
|
errors.append(f"remove_requirements {rpath}: {e}")
|
|
|
|
# remove venv directory
|
|
try:
|
|
if os.path.isdir(vpath):
|
|
shutil.rmtree(vpath, ignore_errors=False)
|
|
removed["venv_removed"] = True
|
|
logger.info("Removed venv directory %s", vpath)
|
|
else:
|
|
logger.debug("No venv directory to remove for %s", name)
|
|
except Exception as e:
|
|
logger.exception("Failed removing venv %s: %s", vpath, e)
|
|
errors.append(f"remove_venv {vpath}: {e}")
|
|
|
|
if errors:
|
|
return JSONResponse(status_code=500, content={"removed": removed, "errors": errors})
|
|
return {"removed": removed}
|
|
|
|
|
|
@router.post("/{name}/enable", response_model=OperationResult)
|
|
def enable_script(name: str, req: EnableRequest) -> OperationResult:
|
|
try:
|
|
validate_name(name)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
|
assert_not_example_script(name, "manually enabled")
|
|
|
|
script_path = script_path_for(name)
|
|
if not os.path.exists(script_path):
|
|
raise HTTPException(status_code=404, detail="script not found")
|
|
|
|
qnum = req.qnum
|
|
service_name = req.service_name or make_service_name(name, qnum)
|
|
python_path = venv_python_for(name)
|
|
exec_start = f"{python_path} {script_path} {qnum}"
|
|
if req.extra_args:
|
|
exec_start += " " + req.extra_args
|
|
|
|
try:
|
|
write_unit(service_name, exec_start, description=f"FW script {name} queue {qnum}", enable_at_boot=req.enable_at_boot or False)
|
|
time.sleep(0.05)
|
|
start_unit(service_name)
|
|
except subprocess.CalledProcessError as e:
|
|
logger.exception("Failed to start service %s for %s: %s", service_name, name, e)
|
|
try:
|
|
remove_unit(service_name)
|
|
except Exception:
|
|
pass
|
|
raise HTTPException(status_code=500, detail=f"systemd start failed: {e}")
|
|
except Exception as e:
|
|
logger.exception("Unknown error starting service %s for %s: %s", service_name, name, e)
|
|
try:
|
|
remove_unit(service_name)
|
|
except Exception:
|
|
pass
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
logger.info("Enabled script %s on qnum=%d as service=%s (python=%s)", name, qnum, service_name, python_path)
|
|
return OperationResult(status="ok", name=name, qnum=qnum, service=service_name, python=python_path)
|
|
|
|
|
|
@router.post("/{name}/disable", response_model=OperationResult)
|
|
def disable_script(name: str, qnum: int) -> OperationResult:
|
|
try:
|
|
validate_name(name)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
|
assert_not_example_script(name, "disabled")
|
|
service_name = make_service_name(name, qnum)
|
|
# attempt stop + remove via systemctl-based remove_unit
|
|
try:
|
|
# If unit exists according to systemctl, remove it; otherwise still try stopping
|
|
units = list_fw_units()
|
|
if service_name in units:
|
|
remove_unit(service_name)
|
|
logger.info("Disabled and removed unit %s", service_name)
|
|
else:
|
|
# attempt stop anyway
|
|
subprocess.run(["systemctl", "stop", _systemctl_unit_name(service_name)], check=False)
|
|
logger.info("Tried stopping unit %s (unit file not present)", service_name)
|
|
except Exception as e:
|
|
logger.exception("Failed to disable unit %s: %s", service_name, e)
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
return OperationResult(status="ok", name=name, qnum=qnum, service=service_name)
|
|
|
|
|
|
@router.delete("/{name}", response_model=Dict[str, Any])
|
|
def delete_script(name: str, qnum: Optional[int] = Query(None, description="If given, only remove the unit for this qnum; otherwise remove all units for the script")) -> Dict[str, Any]:
|
|
"""
|
|
Delete a script and its associated resources.
|
|
- If qnum is provided: stop/remove fw-script-<name>-q<qnum>.service (if present).
|
|
- If qnum is not provided: stop/remove all fw-script-<name>-q*.service units found.
|
|
- Remove script file, requirements file, and venv directory.
|
|
Returns JSON summarizing performed actions and any errors.
|
|
"""
|
|
try:
|
|
validate_name(name)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
|
assert_not_example_script(name, "deleted")
|
|
|
|
removed_units: List[str] = []
|
|
failed_units: List[str] = []
|
|
errors: List[str] = []
|
|
|
|
# determine units to handle
|
|
if qnum is not None:
|
|
svc = make_service_name(name, qnum)
|
|
units_to_handle = [svc]
|
|
else:
|
|
all_units = list_fw_units()
|
|
prefix = f"{UNIT_PREFIX}-{name}-q"
|
|
units_to_handle = [u for u in all_units if u.startswith(prefix)]
|
|
|
|
# stop and remove units
|
|
for svc in units_to_handle:
|
|
try:
|
|
try:
|
|
subprocess.run(["systemctl", "stop", _systemctl_unit_name(svc)], check=False)
|
|
except Exception:
|
|
pass
|
|
# remove unit (systemd-based)
|
|
remove_unit(svc)
|
|
removed_units.append(svc)
|
|
except Exception as e:
|
|
logger.exception("Error removing unit %s: %s", svc, e)
|
|
failed_units.append(svc)
|
|
errors.append(f"remove_unit {svc}: {e}")
|
|
|
|
# remove files
|
|
spath = script_path_for(name)
|
|
rpath = requirements_path_for(name)
|
|
vpath = venv_path_for(name)
|
|
file_removed: List[str] = []
|
|
file_failed: List[str] = []
|
|
|
|
try:
|
|
if os.path.exists(spath):
|
|
os.remove(spath)
|
|
file_removed.append(spath)
|
|
logger.info("Removed script file %s", spath)
|
|
except Exception as e:
|
|
logger.exception("Failed removing script file %s: %s", spath, e)
|
|
file_failed.append(spath)
|
|
errors.append(f"remove_script {spath}: {e}")
|
|
|
|
try:
|
|
if os.path.exists(rpath):
|
|
os.remove(rpath)
|
|
file_removed.append(rpath)
|
|
logger.info("Removed requirements file %s", rpath)
|
|
except Exception as e:
|
|
logger.exception("Failed removing requirements file %s: %s", rpath, e)
|
|
file_failed.append(rpath)
|
|
errors.append(f"remove_requirements {rpath}: {e}")
|
|
|
|
try:
|
|
if os.path.isdir(vpath):
|
|
shutil.rmtree(vpath, ignore_errors=False)
|
|
file_removed.append(vpath)
|
|
logger.info("Removed venv directory %s", vpath)
|
|
except Exception as e:
|
|
logger.exception("Failed removing venv %s: %s", vpath, e)
|
|
file_failed.append(vpath)
|
|
errors.append(f"remove_venv {vpath}: {e}")
|
|
|
|
result: Dict[str, Any] = {
|
|
"name": name,
|
|
"units_removed": removed_units,
|
|
"units_failed": failed_units,
|
|
"files_removed": file_removed,
|
|
"files_failed": file_failed,
|
|
"errors": errors,
|
|
}
|
|
logger.info("Delete script %s completed: removed_units=%d files_removed=%d errors=%d", name, len(removed_units), len(file_removed), len(errors))
|
|
return result
|
|
|
|
|
|
# ---------- Lifecycle helper ----------
|
|
def register_lifecycle(app) -> None:
|
|
@app.on_event("startup")
|
|
def _startup_event() -> None:
|
|
logger.info("Startup: syncing and deploying protected example scripts from %s", EXAMPLE_SCRIPT_DIR)
|
|
deploy_example_scripts_on_startup()
|
|
|
|
@app.on_event("shutdown")
|
|
def _shutdown_event() -> None:
|
|
logger.info("Shutdown: stopping/removing manager-created units with prefix %s", UNIT_PREFIX)
|
|
units = list_fw_units()
|
|
for svc in units:
|
|
if svc.startswith(UNIT_PREFIX + "-"):
|
|
try:
|
|
subprocess.run(["systemctl", "stop", _systemctl_unit_name(svc)], check=False)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
remove_unit(svc)
|
|
except Exception:
|
|
logger.exception("Failed to remove unit %s on shutdown", svc)
|
|
logger.info("Shutdown cleanup complete") |