dasandClaude Sonnet 5.5
60af93df74
build / build (push) Successful in 21m27s
The packaged copies use config.default_path() and pipeline._history_path(): files next to the sources in a checkout, ~/.config/<name> and ~/.local/state/<name> when installed. The copies now differ from the originals only in the neutral defaults. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
417 lines
16 KiB
Python
417 lines
16 KiB
Python
"""Publish steps: preview() builds a Plan (nothing changes on the server), execute() applies it."""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import hashlib
|
|
import subprocess
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
from . import packages, remote
|
|
from .config import App, Config
|
|
from .packages import Kind, Local, Published, version_key
|
|
|
|
|
|
|
|
def _history_path() -> Path:
|
|
"""history.log next to the sources (a checkout), else ~/.local/state/publisher/ (an installed package)."""
|
|
local = Path(__file__).resolve().parent.parent
|
|
if os.access(local, os.W_OK):
|
|
return local / "history.log"
|
|
return Path(os.environ.get("XDG_STATE_HOME") or Path.home() / ".local/state") / "publisher" / "history.log"
|
|
|
|
|
|
HISTORY = _history_path()
|
|
|
|
# Plan item actions; the first five change the server
|
|
NEW, REPLACE, META, NOTES, DELETE, SAME, SKIP = "new", "replace", "meta", "notes", "delete", "same", "skip"
|
|
CHANGING = (NEW, REPLACE, META, NOTES, DELETE)
|
|
|
|
|
|
class PublishError(RuntimeError):
|
|
pass
|
|
|
|
|
|
@dataclass
|
|
class Options:
|
|
build: bool = True
|
|
force: bool = False # publish an older version, overwrite a published file with other content
|
|
prune: bool = True # delete versions beyond keep
|
|
meta: bool = True # upload changed <App>.txt / icon
|
|
notes: str = "" # "what's new" of the published version
|
|
|
|
|
|
@dataclass
|
|
class Item:
|
|
action: str
|
|
target: str # file name in apps_dir
|
|
local: Path | None = None
|
|
kind: Kind | None = None
|
|
version: str = ""
|
|
reason: str = ""
|
|
|
|
@property
|
|
def is_package(self) -> bool:
|
|
return self.kind is not None
|
|
|
|
def describe(self, app: App) -> str:
|
|
"""'<local file, relative to the project> -> <target>' and size/reason, for plan listings."""
|
|
src = ""
|
|
if self.local:
|
|
rel = self.local.relative_to(app.src_path) if self.local.is_relative_to(app.src_path) else self.local
|
|
src = f"{rel} -> " if str(rel) != self.target else ""
|
|
size = f" ({self.local.stat().st_size / 1048576:.1f} MB)" if self.local and self.is_package else ""
|
|
return f"{src}{self.target}{size}" + (f" [{self.reason}]" if self.reason else "")
|
|
|
|
|
|
@dataclass
|
|
class Plan:
|
|
cfg: Config
|
|
app: App
|
|
opts: Options
|
|
items: list[Item] = field(default_factory=list)
|
|
notes: list[str] = field(default_factory=list)
|
|
git_rev: str = ""
|
|
|
|
@property
|
|
def server(self):
|
|
return self.cfg.server
|
|
|
|
@property
|
|
def changes(self) -> list[Item]:
|
|
return [i for i in self.items if i.action in CHANGING]
|
|
|
|
|
|
@dataclass
|
|
class Result:
|
|
ok: bool
|
|
message: str
|
|
|
|
|
|
@dataclass
|
|
class RemoteState:
|
|
# name -> (size, mtime, executable)
|
|
files: dict[str, tuple[int, float, bool]] = field(default_factory=dict)
|
|
hashes: dict[str, str] = field(default_factory=dict)
|
|
|
|
def published(self, app: str | None = None) -> list[Published]:
|
|
"""Packages of an app (all packages for None), grouped case-insensitively like apps.jsp."""
|
|
out = []
|
|
for name, (size, mtime, ex) in self.files.items():
|
|
p = packages.parse_published(name, ex, size, mtime)
|
|
if p and (app is None or p.app.lower() == app.lower()):
|
|
out.append(p)
|
|
return out
|
|
|
|
|
|
_STATE = r'''
|
|
[ -d "$DIR" ] || { echo "no such dir: $DIR" >&2; exit 3; }
|
|
find "$DIR" -maxdepth 1 -type f -printf 'F %s %T@ %m %f\n'
|
|
cd "$DIR" && for n in "${HASH[@]}"; do [ -f "$n" ] && sha256sum -- "$n" | sed 's/^/H /'; done
|
|
true
|
|
'''
|
|
|
|
|
|
def remote_state(cfg: Config, hash_names: list[str] = ()) -> RemoteState:
|
|
out = remote.run(cfg.server, _STATE, DIR=cfg.server.apps_dir, HASH=list(hash_names))
|
|
st = RemoteState()
|
|
for line in out.splitlines():
|
|
if line.startswith("F "):
|
|
_, size, mtime, mode, name = line.split(" ", 4)
|
|
st.files[name] = (int(size), float(mtime), int(mode, 8) & 0o111 != 0)
|
|
elif line.startswith("H "):
|
|
_, sha, name = line.split(None, 2)
|
|
st.hashes[name] = sha
|
|
return st
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
for chunk in iter(lambda: f.read(1 << 20), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def git_rev(src: Path) -> str:
|
|
try:
|
|
rev = subprocess.run(["git", "-C", str(src), "rev-parse", "--short", "HEAD"], capture_output=True,
|
|
text=True, timeout=10).stdout.strip()
|
|
if not rev:
|
|
return ""
|
|
dirty = subprocess.run(["git", "-C", str(src), "status", "--porcelain"], capture_output=True, text=True,
|
|
timeout=10).stdout.strip()
|
|
return rev + ("-dirty" if dirty else "")
|
|
except (OSError, subprocess.SubprocessError):
|
|
return ""
|
|
|
|
|
|
def meta_files(app: App) -> list[tuple[Path, str]]:
|
|
"""(local file, name in apps_dir) of the description and the icon."""
|
|
out = []
|
|
if app.info:
|
|
out.append((app.local(app.info), f"{app.name}.txt"))
|
|
if app.icon:
|
|
icon = app.local(app.icon)
|
|
out.append((icon, f"{app.name}{icon.suffix.lower()}"))
|
|
return out
|
|
|
|
|
|
def notes_name(app: str, version: str) -> str:
|
|
return f"{app}_v{version}.txt"
|
|
|
|
|
|
def prune(published: list[Published], keep: int) -> list[Published]:
|
|
"""Packages beyond the newest `keep` versions of each type."""
|
|
if keep <= 0:
|
|
return []
|
|
out = []
|
|
for kind in {p.kind for p in published}:
|
|
same = sorted((p for p in published if p.kind == kind), key=lambda p: (version_key(p.version), p.mtime),
|
|
reverse=True)
|
|
out += same[keep:]
|
|
return out
|
|
|
|
|
|
def preview(cfg: Config, app: App, opts: Options, log=print) -> Plan:
|
|
src = app.src_path
|
|
if not src.exists():
|
|
raise PublishError(f"local source not found: {src}")
|
|
if opts.build and app.build:
|
|
log(f"== build: {app.build}")
|
|
remote.run_local(app.build, src if src.is_dir() else src.parent, log, app.build_env)
|
|
try:
|
|
builds, notes = packages.resolve(app)
|
|
except packages.PackageError as e:
|
|
raise PublishError(str(e)) from None
|
|
if not builds:
|
|
raise PublishError("nothing to publish: " + "; ".join(notes))
|
|
meta = [(p, n) for p, n in meta_files(app) if opts.meta]
|
|
for p, _ in meta:
|
|
if not p.is_file():
|
|
notes.append(f"no local {p}")
|
|
meta = [(p, n) for p, n in meta if p.is_file()]
|
|
|
|
log(f"== probe {cfg.server.target}:{cfg.server.apps_dir}")
|
|
st = remote_state(cfg, [b.site_name(app.name) for b in builds] + [n for _, n in meta])
|
|
published = st.published(app.name)
|
|
plan = Plan(cfg, app, opts, notes=notes, git_rev=git_rev(src if src.is_dir() else src.parent))
|
|
|
|
for b in builds:
|
|
plan.items.append(_package_item(app, b, st, published, opts))
|
|
added = [i for i in plan.items if i.action in (NEW, REPLACE)]
|
|
|
|
for path, name in meta:
|
|
if st.hashes.get(name) != sha256(path):
|
|
plan.items.append(Item(META, name, path, reason="update" if name in st.files else "new"))
|
|
|
|
if opts.notes.strip():
|
|
for v in sorted({b.version for b in builds}, key=version_key):
|
|
name = notes_name(app.name, v)
|
|
plan.items.append(Item(NOTES, name, version=v, reason="update" if name in st.files else "new"))
|
|
|
|
if opts.prune:
|
|
after = [p for p in published if p.file not in {i.target for i in added}]
|
|
after += [Published(i.target, app.name, i.version, i.kind, mtime=float("inf")) for i in added]
|
|
gone = prune(after, cfg.keep(app))
|
|
for p in gone:
|
|
plan.items.append(Item(DELETE, p.file, kind=p.kind, version=p.version, reason=f"keep {cfg.keep(app)}"))
|
|
left = {version_key(p.version) for p in after if p not in gone}
|
|
for v in sorted({p.version for p in gone}, key=version_key):
|
|
n = notes_name(app.name, v)
|
|
if version_key(v) not in left and n in st.files:
|
|
plan.items.append(Item(DELETE, n, version=v, reason="notes of a deleted version"))
|
|
|
|
if not added:
|
|
plan.notes.append("no new packages" + ("" if opts.force else " (--force publishes older or changed builds)"))
|
|
if plan.git_rev.endswith("-dirty"):
|
|
plan.notes.append("working tree has uncommitted changes")
|
|
return plan
|
|
|
|
|
|
def _package_item(app: App, b: Local, st: RemoteState, published: list[Published], opts: Options) -> Item:
|
|
target = b.site_name(app.name)
|
|
item = Item(NEW, target, b.path, b.kind, b.version)
|
|
if target in st.files:
|
|
if st.hashes.get(target) == sha256(b.path):
|
|
item.action, item.reason = SAME, "already published"
|
|
elif opts.force:
|
|
item.action, item.reason = REPLACE, "same name, other content"
|
|
else:
|
|
item.action, item.reason = SKIP, "published with other content: --force overwrites"
|
|
return item
|
|
newest = max((p for p in published if p.kind == b.kind), key=lambda p: version_key(p.version), default=None)
|
|
if newest and version_key(b.version) <= version_key(newest.version):
|
|
what = "older than" if version_key(b.version) < version_key(newest.version) else "same version as"
|
|
if opts.force:
|
|
item.reason = f"{what} {newest.file}"
|
|
else:
|
|
item.action, item.reason = SKIP, f"{what} {newest.file}"
|
|
return item
|
|
|
|
|
|
_WRITE = r'''
|
|
umask 022
|
|
printf '%s\n' "$TEXT" > "$DIR/.$NAME.tmp" && mv -f "$DIR/.$NAME.tmp" "$DIR/$NAME" && echo "wrote $NAME"
|
|
'''
|
|
|
|
_DELETE = r'''
|
|
cd "$DIR" || exit 1
|
|
for n in "${NAMES[@]}"; do rm -f -- "$n" && echo "deleted $n"; done
|
|
'''
|
|
|
|
_CHECK = r'''
|
|
rc=0
|
|
for i in "${!URLS[@]}"; do
|
|
h=$(curl -sI --max-time 15 "${URLS[$i]}" | tr -d '\r')
|
|
code=$(printf '%s\n' "$h" | head -1 | cut -d' ' -f2)
|
|
len=$(printf '%s\n' "$h" | grep -i '^content-length:' | tail -1 | awk '{print $2}')
|
|
if [ "$code" = 200 ] && [ "$len" = "${SIZES[$i]}" ]; then
|
|
echo "${NAMES[$i]}: HTTP 200, $len bytes"
|
|
else
|
|
echo "ERROR: ${NAMES[$i]}: HTTP ${code:-none}, ${len:-?} bytes (expected ${SIZES[$i]})"; rc=1
|
|
fi
|
|
done
|
|
exit $rc
|
|
'''
|
|
|
|
|
|
def _delete(cfg: Config, names: list[str], log) -> None:
|
|
if any("/" in n or n.startswith(".") or not n for n in names):
|
|
raise PublishError(f"refusing to delete {names}")
|
|
remote.run(cfg.server, _DELETE, log, DIR=cfg.server.apps_dir, NAMES=names)
|
|
|
|
|
|
def check(cfg: Config, files: list[tuple[str, int]], log) -> bool:
|
|
"""The site serves each file with the right size (asked on the server)."""
|
|
if not files or not cfg.server.check_url:
|
|
return True
|
|
try:
|
|
remote.run(cfg.server, _CHECK, log, URLS=[cfg.server.check_url + quote(n) for n, _ in files],
|
|
NAMES=[n for n, _ in files], SIZES=[s for _, s in files])
|
|
except remote.RemoteError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _record(cfg: Config, app: App, action: str, result: str, files: list[str], rev: str, log) -> None:
|
|
ts = datetime.now().isoformat(timespec="seconds")
|
|
line = f"{ts} {app.name} {action} {result} rev={rev or '-'} {','.join(files) or '-'}"
|
|
try:
|
|
remote.run(cfg.server, 'mkdir -p "$STATE" && printf "%s\\n" "$LINE" >> "$STATE/publish.log"',
|
|
STATE=cfg.server.state_dir, LINE=line)
|
|
except remote.RemoteError as e:
|
|
log(f"WARN: publish.log not written: {e}")
|
|
HISTORY.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(HISTORY, "a") as f:
|
|
f.write(line + "\n")
|
|
|
|
|
|
def execute(plan: Plan, log=print) -> Result:
|
|
cfg, app, d = plan.cfg, plan.app, plan.server.apps_dir
|
|
done: list[str] = []
|
|
try:
|
|
# notes first: the page shows them together with the new package
|
|
for i in plan.items:
|
|
if i.action == NOTES:
|
|
remote.run(cfg.server, _WRITE, log, DIR=d, NAME=i.target, TEXT=plan.opts.notes.strip())
|
|
done.append(i.target)
|
|
for i in plan.items:
|
|
if i.action in (NEW, REPLACE, META):
|
|
size = i.local.stat().st_size
|
|
log(f"== upload {i.local.name} -> {i.target} ({size / 1048576:.1f} MB)")
|
|
mode = "755" if i.kind and i.kind.executable else "644"
|
|
remote.upload(cfg.server, str(i.local), f"{d}/{i.target}", mode, log)
|
|
done.append(i.target)
|
|
gone = [i.target for i in plan.items if i.action == DELETE]
|
|
if gone:
|
|
log("== prune")
|
|
_delete(cfg, gone, log)
|
|
done += [f"-{n}" for n in gone]
|
|
uploaded = [(i.target, i.local.stat().st_size) for i in plan.items if i.action in (NEW, REPLACE)]
|
|
ok = True
|
|
if uploaded and cfg.server.check_url:
|
|
log("== check")
|
|
ok = check(cfg, uploaded, log)
|
|
except (remote.RemoteError, PublishError, OSError) as e:
|
|
log(f"ERROR: {e}")
|
|
_record(cfg, app, "publish", "FAILED", done, plan.git_rev, log)
|
|
return Result(False, str(e))
|
|
_record(cfg, app, "publish", "ok" if ok else "UNCHECKED", done, plan.git_rev, log)
|
|
msg = "published" if ok else "published, but the site does not serve it correctly"
|
|
log(f"== {msg}")
|
|
return Result(ok, msg)
|
|
|
|
|
|
def versions(cfg: Config, app: App) -> list[Published]:
|
|
"""Published packages of the app, newest first."""
|
|
pub = remote_state(cfg).published(app.name)
|
|
return sorted(pub, key=lambda p: (version_key(p.version), -packages.KINDS.index(p.kind)
|
|
if p.kind in packages.KINDS else 0), reverse=True)
|
|
|
|
|
|
def remove(cfg: Config, app: App, version: str, log=print) -> Result:
|
|
"""Unpublish one version: all its packages and its notes."""
|
|
st = remote_state(cfg)
|
|
names = [p.file for p in st.published(app.name) if version_key(p.version) == version_key(version)]
|
|
if not names:
|
|
return Result(False, f"{app.name} v{version} is not published")
|
|
n = notes_name(app.name, version)
|
|
if n in st.files:
|
|
names.append(n)
|
|
try:
|
|
_delete(cfg, names, log)
|
|
except (remote.RemoteError, PublishError) as e:
|
|
log(f"ERROR: {e}")
|
|
return Result(False, str(e))
|
|
_record(cfg, app, "remove", "ok", [f"-{x}" for x in names], "", log)
|
|
return Result(True, f"removed {', '.join(names)}")
|
|
|
|
|
|
@dataclass
|
|
class Status:
|
|
state: str # current | newer | unpublished | older | no build | error
|
|
local: str = "" # "apk 1.2.0, AppImage 1.2.0"
|
|
remote: str = ""
|
|
|
|
|
|
def _summary(pairs: list[tuple[Kind, str]]) -> str:
|
|
return ", ".join(f"{k.label} {v}" for k, v in sorted(pairs, key=lambda x: version_key(x[1]), reverse=True))
|
|
|
|
|
|
def status_all(cfg: Config) -> tuple[dict[str, Status], list[str]]:
|
|
"""name -> Status (versions only, contents are compared by preview), and published apps
|
|
that are not in publish.toml."""
|
|
st = remote_state(cfg)
|
|
pub = st.published()
|
|
out = {}
|
|
for a in cfg.apps:
|
|
mine = [p for p in pub if p.app.lower() == a.name.lower()]
|
|
latest: dict[Kind, Published] = {}
|
|
for p in mine:
|
|
if p.kind not in latest or version_key(p.version) > version_key(latest[p.kind].version):
|
|
latest[p.kind] = p
|
|
rem = _summary([(k, p.version) for k, p in latest.items()])
|
|
try:
|
|
builds, _ = packages.resolve(a)
|
|
except packages.PackageError as e:
|
|
out[a.name] = Status("error", str(e), rem)
|
|
continue
|
|
loc = _summary([(b.kind, b.version) for b in builds])
|
|
if not builds:
|
|
state = "no build"
|
|
elif not latest:
|
|
state = "unpublished"
|
|
else:
|
|
cmp = [version_key(b.version) > version_key(latest[b.kind].version) if b.kind in latest else True
|
|
for b in builds]
|
|
older = [b.kind in latest and version_key(b.version) < version_key(latest[b.kind].version)
|
|
for b in builds]
|
|
state = "newer" if any(cmp) else ("older" if any(older) else "current")
|
|
out[a.name] = Status(state, loc, rem)
|
|
known = {a.name.lower() for a in cfg.apps}
|
|
others = sorted({p.app for p in pub if p.app.lower() not in known}, key=str.lower)
|
|
return out, others
|