[grade=A] Shipdeck fleet module and manual deploy flow
Adds validated fleet install/lifecycle routes, bridge-backed Git and OCI workflows, persistent service cards, and the Local-tab Shipdeck deployment UI while preserving catalog and External flows. Judge: urn:ump:if6udffdyelsf4qvskkr65ikhuprsjiajzij6h653fhf2zgyynzq
This commit is contained in:
DashCaddy Polish Loop
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env python3
|
||||
"""shipdeck-bridge — token-gated HTTP wrapper around the shipdeck CLI.
|
||||
|
||||
Runs on the DNS2 HOST (not in the container) so that SSH keys, fleet-dns
|
||||
credentials and root-level execution stay out of the DashCaddy web container.
|
||||
The container reaches it via the docker bridge IP (172.17.0.1), the same
|
||||
pattern as the Caddy admin API.
|
||||
|
||||
Endpoints (all require X-Shipdeck-Token matching /etc/shipdeck/bridge-token):
|
||||
GET /api/health -> {ok, version}
|
||||
GET /api/repos -> deployable repos (dirs with a Shipdeckfile)
|
||||
GET /api/services -> journal-derived service inventory
|
||||
GET /api/journal?service=N -> journal rows (newest first)
|
||||
GET /api/status?service=N -> live re-probe output
|
||||
POST /api/deploy {dir} -> run `shipdeck deploy <dir>` (serialized)
|
||||
POST /api/rollback {service} -> run `shipdeck rollback <service>` (serialized)
|
||||
POST /api/install {repo_url, service, subdomain?} -> GitHub clone+deploy+card (DC-131)
|
||||
|
||||
Security model:
|
||||
- Listens on 127.0.0.1:8977 and 172.17.0.1:8977 ONLY (docker bridge + local).
|
||||
- Every request must carry the shared token (0600 file, root-owned).
|
||||
- Deploy dirs are validated: realpath must sit under SHIPDECK_REPOS_ROOT and
|
||||
contain a Shipdeckfile. Service names are strict [a-z0-9-].
|
||||
- The CLI is exec'd via argv lists — never a shell.
|
||||
- Mutations (deploy/rollback) are serialized with a lock; status/journal are
|
||||
concurrent.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
from gh_install import gh_install # DC-131: GitHub -> card installs (init_shared below)
|
||||
from gh_install import list_repos as gitea_list_repos # DC-133
|
||||
from image_install import install_image
|
||||
import image_install as _image
|
||||
|
||||
SHIPDECK_BIN = os.environ.get("SHIPDECK_BIN", "/usr/local/bin/shipdeck")
|
||||
TOKEN_FILE = os.environ.get("SHIPDECK_BRIDGE_TOKEN_FILE", "/etc/shipdeck/bridge-token")
|
||||
REPOS_ROOT = os.environ.get("SHIPDECK_REPOS_ROOT", "/root")
|
||||
LISTEN_HOSTS = ["127.0.0.1", os.environ.get("SHIPDECK_BRIDGE_DOCKER_IP", "172.17.0.1")]
|
||||
PORT = int(os.environ.get("SHIPDECK_BRIDGE_PORT", "8977"))
|
||||
DEPLOY_TIMEOUT = int(os.environ.get("SHIPDECK_DEPLOY_TIMEOUT", "600"))
|
||||
PROBE_TIMEOUT = 90
|
||||
MAX_BODY = 65536
|
||||
|
||||
RE_SERVICE = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$")
|
||||
|
||||
mutation_lock = threading.Lock()
|
||||
|
||||
|
||||
def read_token() -> str:
|
||||
with open(TOKEN_FILE, "r", encoding="utf-8") as f:
|
||||
token = f.read().strip()
|
||||
if not token:
|
||||
raise RuntimeError("SHIPDECK_BRIDGE_TOKEN_FILE is empty; refusing to start unauthenticated")
|
||||
return token
|
||||
|
||||
|
||||
TOKEN = read_token()
|
||||
|
||||
|
||||
def run_shipdeck(args, timeout):
|
||||
"""Exec the shipdeck CLI via argv (no shell). Returns (code, stdout+stderr)."""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[SHIPDECK_BIN] + args,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
env={**os.environ, "SHIPDECK_JOURNAL": os.environ.get("SHIPDECK_JOURNAL", "/var/lib/shipdeck/journal.jsonl")},
|
||||
)
|
||||
return proc.returncode, (proc.stdout or "") + (proc.stderr or "")
|
||||
except subprocess.TimeoutExpired:
|
||||
return 124, f"shipdeck {' '.join(args)} timed out after {timeout}s"
|
||||
except FileNotFoundError:
|
||||
return 127, f"shipdeck binary not found at {SHIPDECK_BIN}"
|
||||
|
||||
|
||||
# DC-131: hand shared state to the GitHub-install module (after run_shipdeck's
|
||||
# def so everything it needs exists; avoids a circular import).
|
||||
import gh_install as _gh
|
||||
_gh.init_shared(RE_SERVICE, REPOS_ROOT, PORT, mutation_lock, run_shipdeck)
|
||||
_image.init_shared(mutation_lock, run_shipdeck)
|
||||
|
||||
|
||||
def parse_journal(raw: str):
|
||||
"""Parse `shipdeck journal [name]` output rows robustly."""
|
||||
rows = []
|
||||
for line in raw.splitlines():
|
||||
m = re.match(
|
||||
r"^(\d{4}-\d{2}-\d{2}T[\d:]+Z)\s+(\S+)\s+(\S+)\s+epoch=(\d+)\s+(\S+)$",
|
||||
line.strip(),
|
||||
)
|
||||
if m:
|
||||
rows.append(
|
||||
{
|
||||
"time": m.group(1),
|
||||
"service": m.group(2),
|
||||
"action": m.group(3),
|
||||
"epoch": int(m.group(4)),
|
||||
"duration": m.group(5),
|
||||
}
|
||||
)
|
||||
continue
|
||||
# rollback rows can have 0.0s duration too; tolerate missing duration
|
||||
m = re.match(r"^(\d{4}-\d{2}-\d{2}T[\d:]+Z)\s+(\S+)\s+(\S+)\s+epoch=(\d+)$", line.strip())
|
||||
if m:
|
||||
rows.append(
|
||||
{
|
||||
"time": m.group(1),
|
||||
"service": m.group(2),
|
||||
"action": m.group(3),
|
||||
"epoch": int(m.group(4)),
|
||||
"duration": None,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def service_inventory():
|
||||
code, out = run_shipdeck(["journal"], PROBE_TIMEOUT)
|
||||
if code != 0:
|
||||
return None, out
|
||||
rows = parse_journal(out)
|
||||
inv = {}
|
||||
for r in rows:
|
||||
s = inv.setdefault(
|
||||
r["service"],
|
||||
{"name": r["service"], "host": None, "last_action": r["action"], "last_time": r["time"], "last_epoch": r["epoch"]},
|
||||
)
|
||||
if s["host"] is None:
|
||||
code2, out2 = run_shipdeck(["status", r["service"]], PROBE_TIMEOUT)
|
||||
m = re.search(r"host (\S+)", out2)
|
||||
if m:
|
||||
s["host"] = m.group(1)
|
||||
return sorted(inv.values(), key=lambda x: x["last_time"], reverse=True), None
|
||||
|
||||
|
||||
def list_repos():
|
||||
"""Depth-1 scan of REPOS_ROOT for dirs containing a Shipdeckfile."""
|
||||
repos = []
|
||||
try:
|
||||
for name in sorted(os.listdir(REPOS_ROOT)):
|
||||
d = os.path.join(REPOS_ROOT, name)
|
||||
if not os.path.isdir(d) or name.startswith("."):
|
||||
continue
|
||||
if os.path.isfile(os.path.join(d, "Shipdeckfile")):
|
||||
repos.append({"dir": d, "name": name})
|
||||
except OSError as e:
|
||||
return None, str(e)
|
||||
return repos, None
|
||||
|
||||
|
||||
def resolve_deploy_dir(d):
|
||||
"""Validate a deploy dir request. Returns (realpath, None) or (None, error)."""
|
||||
if not isinstance(d, str) or not d.strip():
|
||||
return None, "dir is required"
|
||||
real = os.path.realpath(d)
|
||||
root_real = os.path.realpath(REPOS_ROOT)
|
||||
if real != root_real and not real.startswith(root_real + os.sep):
|
||||
return None, "dir must be under " + root_real
|
||||
if not os.path.isfile(os.path.join(real, "Shipdeckfile")):
|
||||
return None, "no Shipdeckfile in " + real
|
||||
return real, None
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "shipdeck-bridge/1.0"
|
||||
|
||||
def log_message(self, fmt, *args): # quiet default access log
|
||||
pass
|
||||
|
||||
def _authed(self) -> bool:
|
||||
return bool(TOKEN) and self.headers.get("X-Shipdeck-Token", "") == TOKEN
|
||||
|
||||
def _send(self, code, payload):
|
||||
body = json.dumps(payload).encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _deny(self):
|
||||
self._send(401, {"ok": False, "error": "invalid or missing X-Shipdeck-Token"})
|
||||
|
||||
# ----- GET -----
|
||||
def do_GET(self):
|
||||
if not self._authed():
|
||||
return self._deny()
|
||||
u = urlparse(self.path)
|
||||
q = parse_qs(u.query)
|
||||
if u.path == "/api/health":
|
||||
code, out = run_shipdeck(["version"], 10)
|
||||
return self._send(200, {"ok": code == 0, "shipdeck": out.strip()})
|
||||
if u.path == "/api/repos":
|
||||
repos, err = list_repos()
|
||||
if err:
|
||||
return self._send(500, {"ok": False, "error": err})
|
||||
return self._send(200, {"ok": True, "repos": repos})
|
||||
if u.path == "/api/services":
|
||||
inv, err = service_inventory()
|
||||
if err:
|
||||
return self._send(500, {"ok": False, "error": err})
|
||||
return self._send(200, {"ok": True, "services": inv})
|
||||
if u.path == "/api/managed":
|
||||
code, out = run_shipdeck(["ls", "--json"], PROBE_TIMEOUT)
|
||||
if code != 0:
|
||||
return self._send(500, {"ok": False, "error": "shipdeck ls failed", "output": out[-4000:]})
|
||||
try:
|
||||
services = json.loads(out)
|
||||
except json.JSONDecodeError:
|
||||
return self._send(500, {"ok": False, "error": "shipdeck ls returned invalid JSON"})
|
||||
return self._send(200, {"ok": True, "services": services})
|
||||
if u.path == "/api/logs":
|
||||
service = (q.get("name") or [""])[0]
|
||||
if not RE_SERVICE.match(service):
|
||||
return self._send(400, {"ok": False, "error": "invalid service name"})
|
||||
code, out = run_shipdeck(["logs", "-n", "300", service], PROBE_TIMEOUT)
|
||||
return self._send(200 if code == 0 else 500, {"ok": code == 0, "output": out[-64000:]})
|
||||
if u.path == "/api/shipdeckfile":
|
||||
service = (q.get("name") or [""])[0]
|
||||
if not RE_SERVICE.match(service):
|
||||
return self._send(400, {"ok": False, "error": "invalid service name"})
|
||||
code, out = run_shipdeck(["shipdeckfile", service], PROBE_TIMEOUT)
|
||||
if code != 0:
|
||||
return self._send(404, {"ok": False, "error": "Shipdeckfile not found"})
|
||||
out = re.sub(r'(?im)^([A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|PASS|KEY)[A-Z0-9_]*)\s*=.*$', r'\1 = "<redacted>"', out)
|
||||
return self._send(200, {"ok": True, "shipdeckfile": out})
|
||||
if u.path == "/api/journal":
|
||||
service = (q.get("service") or [""])[0]
|
||||
args = ["journal"]
|
||||
if service:
|
||||
if not RE_SERVICE.match(service):
|
||||
return self._send(400, {"ok": False, "error": "invalid service name"})
|
||||
args.append(service)
|
||||
code, out = run_shipdeck(args, PROBE_TIMEOUT)
|
||||
return self._send(200 if code == 0 else 500, {"ok": code == 0, "rows": parse_journal(out), "raw": out[-4000:]})
|
||||
if u.path == "/api/status":
|
||||
service = (q.get("service") or [""])[0]
|
||||
if not RE_SERVICE.match(service):
|
||||
return self._send(400, {"ok": False, "error": "invalid service name"})
|
||||
code, out = run_shipdeck(["status", service], PROBE_TIMEOUT)
|
||||
return self._send(200 if code == 0 else 500, {"ok": code == 0, "output": out[-8000:]})
|
||||
return self._send(404, {"ok": False, "error": "not found"})
|
||||
|
||||
# ----- POST -----
|
||||
def do_POST(self):
|
||||
if not self._authed():
|
||||
return self._deny()
|
||||
u = urlparse(self.path)
|
||||
length = int(self.headers.get("Content-Length", "0") or 0)
|
||||
if length > MAX_BODY:
|
||||
return self._send(413, {"ok": False, "error": "body too large"})
|
||||
raw = self.rfile.read(length) if length else b"{}"
|
||||
try:
|
||||
payload = json.loads(raw or b"{}")
|
||||
except json.JSONDecodeError:
|
||||
return self._send(400, {"ok": False, "error": "invalid JSON body"})
|
||||
|
||||
if u.path == "/api/deploy":
|
||||
real, err = resolve_deploy_dir(payload.get("dir"))
|
||||
if err:
|
||||
return self._send(400, {"ok": False, "error": err})
|
||||
with mutation_lock:
|
||||
code, out = run_shipdeck(["deploy", real], DEPLOY_TIMEOUT)
|
||||
return self._send(200 if code == 0 else 500, {"ok": code == 0, "exit": code, "output": out[-16000:]})
|
||||
|
||||
if u.path == "/api/rollback":
|
||||
service = payload.get("service")
|
||||
if not isinstance(service, str) or not RE_SERVICE.match(service):
|
||||
return self._send(400, {"ok": False, "error": "invalid service name"})
|
||||
with mutation_lock:
|
||||
code, out = run_shipdeck(["rollback", service], DEPLOY_TIMEOUT)
|
||||
return self._send(200 if code == 0 else 500, {"ok": code == 0, "exit": code, "output": out[-16000:]})
|
||||
|
||||
if u.path in {"/api/start", "/api/stop", "/api/restart", "/api/rm"}:
|
||||
service = payload.get("name")
|
||||
if not isinstance(service, str) or not RE_SERVICE.match(service):
|
||||
return self._send(400, {"ok": False, "error": "invalid service name"})
|
||||
action = u.path.rsplit("/", 1)[-1]
|
||||
with mutation_lock:
|
||||
code, out = run_shipdeck([action, service], DEPLOY_TIMEOUT)
|
||||
return self._send(200 if code == 0 else 500, {"ok": code == 0, "exit": code, "output": out[-16000:]})
|
||||
|
||||
if u.path == "/api/image/install":
|
||||
if not isinstance(payload, dict):
|
||||
return self._send(400, {"ok": False, "error": "JSON object required"})
|
||||
code, body = install_image(payload)
|
||||
return self._send(code, body)
|
||||
|
||||
if u.path == "/api/gitea/repos":
|
||||
code, body = gitea_list_repos(payload if isinstance(payload, dict) else {})
|
||||
return self._send(code, body)
|
||||
|
||||
if u.path == "/api/install":
|
||||
# DC-131: GitHub URL -> clone -> Shipdeckfile -> deploy -> metadata.
|
||||
# Serialized with the same mutation lock; long timeout (build).
|
||||
if not isinstance(payload, dict):
|
||||
return self._send(400, {"ok": False, "error": "JSON object required"})
|
||||
code, body = gh_install(payload)
|
||||
return self._send(code, body)
|
||||
|
||||
return self._send(404, {"ok": False, "error": "not found"})
|
||||
|
||||
|
||||
def main():
|
||||
servers = []
|
||||
last_err = None
|
||||
for host in LISTEN_HOSTS:
|
||||
try:
|
||||
srv = ThreadingHTTPServer((host, PORT), Handler)
|
||||
srv.daemon_threads = True
|
||||
servers.append(srv)
|
||||
except OSError as e:
|
||||
last_err = e
|
||||
print(f"shipdeck-bridge: FAILED to bind {host}:{PORT}: {e}", file=sys.stderr, flush=True)
|
||||
if not servers:
|
||||
# fail fast: systemd restarts us; a silently-dead daemon is worse
|
||||
print(f"shipdeck-bridge: no listeners could bind on {LISTEN_HOSTS}:{PORT}; exiting", file=sys.stderr, flush=True)
|
||||
sys.exit(1)
|
||||
for srv in servers:
|
||||
threading.Thread(target=srv.serve_forever, daemon=True).start()
|
||||
print(f"shipdeck-bridge listening on {srv.server_address[0]}:{PORT}", flush=True)
|
||||
if last_err is not None:
|
||||
# degraded-but-alive: at least one listener bound; keep serving
|
||||
print("shipdeck-bridge: running in DEGRADED mode (partial bind); check logs", file=sys.stderr, flush=True)
|
||||
try:
|
||||
threading.Event().wait()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user