🐳 Install Podman Server

Podman is an open-source container engine for developing, managing, and running containers and container images. Podman provides a Docker-compatible experience while running containers without requiring a central daemon, making it a lightweight and secure alternative for containerized applications.

Features:

• rootless Podman container execution with automatic boot startup via systemd --user and linger
• one dedicated Linux user per Podman service, each with isolated home/config/cache/data directories
• strict filesystem isolation between service users, with explicit administrative override access for the debian sudoer account
• automatic ownership/group inheritance for newly created service files and directories
• administrative access from the debian sudoer user to all Podman service files and directories

---

📥 Installation

Podman can be installed easily on debian system using apt.

Install Podman

# install podman and its requirements
sudo apt update
sudo apt install -y \
  podman \
  podman-compose \
  uidmap \
  passt \
  aardvark-dns \
  slirp4netns \
  fuse-overlayfs \
  systemd-container \
  openssh-server \
  skopeo \
  jq \
  dbus-x11

# install ACL
sudo apt install -y acl

# install apparmor
sudo apt install apparmor apparmor-utils
sudo systemctl enable --now apparmor

# check that podman is installed
podman --version

Install the helper image_userinfo to your shell (run once)

In rootless Podman, a container writes its data as a specific in-container UID/GID, which maps to a host subuid. For a service to read and write its bind-mounted data directories, those directories must be owned by that matching subuid, so the first thing you need to know is which user the image runs as.

The helper below reads an image's config straight from the registry and prints, in a single table:

• the UID/GID to chown the data dirs to,
• whether to pin User= in the quadlet,
• and whether the image expects PUID/PGID.

image_userinfo

if grep -qF 'image_userinfo()' ~/.bashrc 2>/dev/null; then
    echo "image_userinfo() already in ~/.bashrc — skipping"
else
cat >> ~/.bashrc <<'EOF'
# image_userinfo — for a container image, print (for rootless Podman) the uid[:gid] to
# chown its data dirs to, whether to pin User=, and whether it needs PUID/PGID.
# Reads the image config from the registry. For images whose runtime identity isn't in the
# config (named USER, a gosu/su-exec drop, or a docker-entrypoint image with a built-in
# user like redis), it pulls the image once to read /etc/passwd, then removes it again if
# it wasn't already cached.
# Usage: image_userinfo [IMAGE] [UID]   (prompts if no IMAGE; UID defaults to 1000)
# Requires: skopeo, jq  (and podman for the resolve step)
image_userinfo() {
    local img="${1:-}" want="${2:-1000}"
    if [ -z "$img" ]; then
        read -r -p "Docker image path: " img
        [ -z "$img" ] && { echo "error: no image provided" >&2; return 1; }
    fi
    local t; for t in skopeo jq; do
        command -v "$t" >/dev/null 2>&1 || { echo "error: '$t' not found (sudo apt install $t)" >&2; return 1; }
    done
    local cfg; cfg="$(skopeo inspect --config "docker://${img}" 2>/dev/null)" \
        || { echo "error: cannot read config for '$img'" >&2; return 1; }

    local user entry cmd ep0 cmd0 imgenv vendor src maint base cands
    user="$(jq -r '.config.User // ""'                                        <<<"$cfg")"
    entry="$(jq -r '(.config.Entrypoint // []) | join(" ")'                   <<<"$cfg")"
    cmd="$(jq -r '(.config.Cmd // []) | join(" ")'                            <<<"$cfg")"
    ep0="$(jq -r '(.config.Entrypoint // [])[0] // ""'                        <<<"$cfg")"
    cmd0="$(jq -r '(.config.Cmd // [])[0] // ""'                              <<<"$cfg")"; cmd0="${cmd0##*/}"
    imgenv="$(jq -r '(.config.Env // []) | join(" ")'                         <<<"$cfg")"
    vendor="$(jq -r '.config.Labels["org.opencontainers.image.vendor"] // ""' <<<"$cfg")"
    src="$(jq -r '.config.Labels["org.opencontainers.image.source"]    // ""' <<<"$cfg")"
    maint="$(jq -r '.config.Labels.maintainer // ""'                          <<<"$cfg")"
    base="${img##*/}"; base="${base%%:*}"
    cands="$base ${base%%-*} $cmd0 ${cmd0%%-*}"        # candidate built-in usernames

    local is_s6=false is_lsio=false is_gosu=false is_wrapper=false
    case " $entry " in *" /init "*) is_s6=true ;; esac
    case " $cmd "   in *" /init "*) is_s6=true ;; esac
    case "$(printf '%s %s %s' "$vendor" "$src" "$maint" | tr 'A-Z' 'a-z')" in
        *linuxserver*) is_lsio=true ;;
    esac
    case " $imgenv " in *" GOSU_VERSION="*) is_gosu=true ;; esac
    case "$ep0"      in *docker-entrypoint*) is_wrapper=true ;; esac

    # Resolve a real identity from the image's /etc/passwd (pulls if needed, then removes
    # the image again only if it wasn't already cached). Looks for: a given username, else
    # the entrypoint's gosu/su-exec target, else a built-in user matching the image name.
    __iu_resolve() {   # <img> <user|''> <entrypoint|''> <candidates>  -> "uid[:gid]" ('' if unknown)
        command -v podman >/dev/null 2>&1 || return 0
        local i="$1" u="$2" ep="$3" cl="$4" present=false line uid gid
        podman image exists "$i" 2>/dev/null && present=true
        $present || echo "  …resolving identity from $i (one-time pull)…" >&2
        line="$(podman run --rm --entrypoint sh "$i" -c '
            u="$1"; ep="$2"; cl="$3"
            if [ -z "$u" ]; then
                [ -n "$ep" ] && [ ! -f "$ep" ] && ep="$(command -v "$ep" 2>/dev/null)"
                for f in "$ep" /usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh /entrypoint.sh; do
                    [ -f "$f" ] || continue
                    u="$(grep -hoE "(gosu|su-exec) [A-Za-z0-9_-]+" "$f" 2>/dev/null | awk "{print \$2}" | head -n1)"
                    [ -n "$u" ] && break
                done
            fi
            if [ -z "$u" ]; then
                for c in $cl; do grep -q "^${c}:" /etc/passwd 2>/dev/null && { u="$c"; break; }; done
            fi
            [ -n "$u" ] || exit 0
            grep "^${u}:" /etc/passwd 2>/dev/null || getent passwd "$u" 2>/dev/null
        ' _ "$u" "$ep" "$cl" 2>/dev/null)"
        $present || podman rmi "$i" >/dev/null 2>&1
        uid="$(printf '%s' "$line" | cut -d: -f3 | tr -dc '0-9')"
        gid="$(printf '%s' "$line" | cut -d: -f4 | tr -dc '0-9')"
        [ -z "$uid" ] && return 0
        if [ -n "$gid" ] && [ "$gid" != "$uid" ]; then printf '%s:%s' "$uid" "$gid"; else printf '%s' "$uid"; fi
    }

    local kind quser puid chownto res ru rg
    if $is_s6 || $is_lsio; then
        kind="LinuxServer/s6"; quser="don't set"; puid="PUID/PGID=${want}"; chownto="${want}"
    elif [ -n "$user" ]; then
        kind="declares USER ($user)"; quser="not needed"; puid="no"
        if [[ "$user" =~ ^[0-9]+(:[0-9]+)?$ ]]; then
            chownto="${user}"
        else
            res="$(__iu_resolve "$img" "${user%%:*}" "$ep0" "$cands")"
            chownto="${res:-${user} (verify uid)}"
        fi
    elif $is_gosu || $is_wrapper; then
        res="$(__iu_resolve "$img" "" "$ep0" "$cands")"; puid="no"
        if [ -n "$res" ]; then
            ru="${res%%:*}"; rg="${res#*:}"; [ "$rg" = "$res" ] && rg="$ru"
            $is_gosu && kind="root -> gosu-drop" || kind="root + built-in user"
            quser="User=${ru}:${rg}"; chownto="${res}"
        else
            kind="no USER -> root"; quser="User=${want}:${want}"; chownto="${want}"
        fi
    else
        kind="no USER -> root"; quser="User=${want}:${want}"; puid="no"; chownto="${want}"
    fi

    local -a H=("Image" "Type" "Quadlet User=" "PUID/PGID" "data_dirs to")
    local -a V=("$img" "$kind" "$quser" "$puid" "$chownto")
    local -a W=(); local i len
    for i in 0 1 2 3 4; do W[i]=${#H[i]}; len=${#V[i]}; [ "$len" -gt "${W[i]}" ] && W[i]=$len; done
    __iu_seg() { local n=$(( $1 + 2 )) s=''; while [ "$n" -gt 0 ]; do s+='─'; n=$((n-1)); done; printf '%s' "$s"; }
    printf '┌'; for i in 0 1 2 3 4; do printf '%s' "$(__iu_seg "${W[i]}")"; [ "$i" -lt 4 ] && printf '┬'; done; printf '┐\n'
    printf '│'; for i in 0 1 2 3 4; do printf ' %-*s │' "${W[i]}" "${H[i]}"; done; printf '\n'
    printf '├'; for i in 0 1 2 3 4; do printf '%s' "$(__iu_seg "${W[i]}")"; [ "$i" -lt 4 ] && printf '┼'; done; printf '┤\n'
    printf '│'; for i in 0 1 2 3 4; do printf ' %-*s │' "${W[i]}" "${V[i]}"; done; printf '\n'
    printf '└'; for i in 0 1 2 3 4; do printf '%s' "$(__iu_seg "${W[i]}")"; [ "$i" -lt 4 ] && printf '┴'; done; printf '┘\n'
    unset -f __iu_seg __iu_resolve
}
EOF
echo "image_userinfo() added to ~/.bashrc"
fi
source ~/.bashrc

---

🔐 Rootless Services

Each Podman service runs as its own dedicated Linux user to provide strong isolation between containers, configuration, runtime state, and persistent data.

Example:

• gokapi → user gokapi_svc, group gokapi_admins
• immich → user immich_svc, group immich_admins

The debian administrative user belongs to all *_admins groups, allowing centralized management of every service while keeping services isolated from each other. Service users belong only to their own primary group and do not share access with other services.

Each service uses two separate storage locations:

Path	Purpose
/media/ssd/podman/<service>/	persistent application data, configuration, environment files, and Quadlet definitions
/media/ssd/podman-users/<service>/	rootless Podman user home containing container runtime state, images, volumes, cache, and user configuration

Example layout:

/media/ssd/podman/service/
├── Containerfile
├── postgres.env
├── service.env
├── quadlet/
│   ├── service.network
│   ├── postgres.container
│   └── service.container
└── data/
    ├── service/
    └── postgres-db/

Rootless Podman requires a valid home directory for each service user because Podman stores per-user runtime data under:

/media/ssd/podman-users/<service>/
├── .config/
├── .cache/
└── .local/share/containers/

Container runtime storage is isolated per service user:

~/.local/share/containers/
└── storage/
    ├── overlay/              # container filesystem layers
    ├── overlay-images/       # container image metadata
    ├── overlay-containers/   # container metadata
    └── volumes/              # named Podman volumes

Each service user therefore has its own isolated:

• container images
• containers
• named volumes
• network namespace
• Podman configuration
• runtime state

---

⚙️ Configuration

📁 Base Directories

Prepare the base directories used to store service data and per-user Podman runtime files. The permissions below keep the directories manageable by the debian administrative user while limiting access for other users.

Create podman-users home directory

# create podman-users home directory
# u=rwX → owner gets:
#   - rw for files
#   - rwx for directories
#   - preserves execute bit only for already executable files
# go=x  → group and others get traverse permission on directories only
sudo mkdir -p /media/ssd/podman-users
sudo chown debian:debian /media/ssd/podman-users
sudo chmod u=rwX,go=x /media/ssd/podman-users

Create podman services directory

# create podman services directory
# u=rwX → owner gets:
#   - rw for files
#   - rwx for directories
#   - preserves execute bit only for already executable files
# go=x  → group and others get traverse permission on directories only
sudo mkdir -p /media/ssd/podman
sudo chown debian:debian /media/ssd/podman
sudo chmod u=rwX,go=x /media/ssd/podman

Harden debian user home

# harden debian user home
# u=rwX → owner gets:
#   - rw for files (600)
#   - rwx for directories (700)
#   - preserves execute bit only for already executable files
# go=   → group and others get no permissions
sudo chmod u=rwX,go= /home/debian

Harden /media/ssd top-level directory

# harden the ssd top-level directory
# u=rwX → owner keeps full access
# go=x  → group and others get traverse permission only: every account can
#         still pass through to reach its own subdirectory, but none can
#         list what else lives on the SSD
sudo chmod u=rwX,go=x /media/ssd

Audit /media/ssd top-level permissions

Run this after adding any new directory on the SSD:

# one line per entry: mode, owner:group, path
sudo find /media/ssd -maxdepth 1 -mindepth 1 -exec stat -c '%A %U:%G %n' {} +

Rule of thumb: the last permission triplet ("others") of every entry should show at most --x. An account that needs more on a directory (e.g. www-data on an html directory, the mkdocs container on its bare repo) gets a named ACL entry (setfacl -m u:<user>:...), never world bits — a + after the mode means such entries exist, inspect them with getfacl.

---

👥 User and Group

This step creates a dedicated Linux user for the Podman service and a matching administration group for controlled access. The service runs under its own isolated account, while the debian user is added to the admin group to manage the service files when needed.

Define Service Variables

# define the service name
SERVICE_NAME=XXX

Replace XXX by your service name

Initialize Service Environment

# built-in bash safety and SERVICE_NAME validation
set -euo pipefail; [ -n "${SERVICE_NAME:-}" ] || { echo "SERVICE_NAME is empty"; exit 1; }

# automatically generate variables
SERVICE_HOME="/media/ssd/podman-users/${SERVICE_NAME}"
SERVICE_USER="${SERVICE_NAME}_svc"
SERVICE_ADMIN="${SERVICE_NAME}_admins"

Create Service User and Access Control

# create Linux user for this service
#   --system      → create a dedicated system account (non-human service user)
#   --create-home → automatically create the user's home directory
#   --home-dir    → define the service user's home directory location
#   --shell       → allow local interactive shell access for service administration
# name of the Linux service user to create
sudo useradd \
  --system \
  --create-home \
  --home-dir "${SERVICE_HOME}" \
  --shell /bin/bash \
  "${SERVICE_USER}"

# restrict SSH access for service users
getent group no-ssh >/dev/null || sudo groupadd no-ssh            # create no-ssh group only if missing
id -nG "${SERVICE_USER}" | grep -qw no-ssh ||                     # add service user to no-ssh group only if not already a member
  sudo usermod -aG no-ssh "${SERVICE_USER}"
sudo install -d -m 755 /etc/ssh/sshd_config.d                     # ensure the drop-in directory exists
SSHD_CONFIG="/etc/ssh/sshd_config.d/99-deny-service-users.conf"
sudo tee "${SSHD_CONFIG}" >/dev/null <<EOF                        # enforce SSH deny rule for members of no-ssh
DenyGroups no-ssh
EOF
sudo sshd -t                                                      # test SSH configuration
sudo systemctl reload ssh                                         # reload SSH daemon configuration

# create Linux group for this service and allow debian user to access it
sudo groupadd "${SERVICE_ADMIN}"
sudo usermod -aG "${SERVICE_ADMIN}" debian

Making XDG_RUNTIME_DIR available automatically per user

XDG_RUNTIME_DIR points to the per-user runtime directory (/run/user/) where systemd, D-Bus, and rootless tools like Podman keep their session sockets and state. Setting it in each service user's login profile ensures these tools can find that directory when you switch users with sudo -iu, since that command doesn't always open a full login session that would set it automatically.

# make it persistent: set it automatically on every future login
sudo tee -a "$(getent passwd "${SERVICE_USER}" | cut -d: -f6)/.profile" >/dev/null <<'EOF'

# Provide XDG_RUNTIME_DIR for rootless/user services when missing
if [ -z "${XDG_RUNTIME_DIR}" ]; then
    export XDG_RUNTIME_DIR="/run/user/$(id -u)"
    export DBUS_SESSION_BUS_ADDRESS="unix:path=${XDG_RUNTIME_DIR}/bus"
fi
EOF

Notify: re-login required for group changes

# logout and back for group membership to apply
echo "Log out and back in later for ${SERVICE_ADMIN} group membership to apply in interactive shells."
exit

---

🔁 Persistent Services

Rootless Podman containers run inside the service user's systemd --user session. Without lingering enabled, user services may only start after the user logs in and may stop after logout.

Define Service Variables

# define the service name
SERVICE_NAME=XXX

Replace XXX by your service name

Initialize Service Environment

# built-in bash safety and SERVICE_NAME validation
set -euo pipefail; [ -n "${SERVICE_NAME:-}" ] || { echo "SERVICE_NAME is empty"; exit 1; }

# automatically generate variables
SERVICE_HOME="/media/ssd/podman-users/${SERVICE_NAME}"
SERVICE_DIR="/media/ssd/podman/${SERVICE_NAME}"
SERVICE_USER="${SERVICE_NAME}_svc"
SERVICE_ADMIN="${SERVICE_NAME}_admins"

Enable Persistent User Services with Lingering

# retrieve the service user's numeric UID
SERVICE_UID="$(id -u "${SERVICE_USER}")"

# allow the user's systemd --user manager to run without interactive login
sudo loginctl enable-linger "${SERVICE_USER}"

# immediately start the user's systemd instance
sudo systemctl start "user@${SERVICE_UID}.service"

---

🔍 Detecting a container's runtime UID/GID

Detecting a container's runtime UID/GID

Once installed, run image_userinfo to see how an image runs and what its data directory permissions should be.

image_userinfo                                        # prompts for the image
image_userinfo docker.io/library/postgres:16-alpine   # with docker image argument

It reads the image config from the registry and prints a single table:

debian@odroid:~$ image_userinfo docker.io/library/postgres:16-alpine
  …resolving uid from docker.io/library/postgres:16-alpine (one-time pull)…
+--------------------------------------+-------------------+---------------+-----------+--------------+
| Image                                | Type              | Quadlet User= | PUID/PGID | data_dirs to |
+--------------------------------------+-------------------+---------------+-----------+--------------+
| docker.io/library/postgres:16-alpine | root -> gosu-drop | User=70:70    | no        | 70           |
+--------------------------------------+-------------------+---------------+-----------+--------------+

Reading the table

Column	Meaning
Image	The image reference that was inspected.
Type	How the image sets its user — declares USER (name) (runs as that user), no USER -> root (runs as root), or LinuxServer/s6 (drops to PUID/PGID).
Quadlet User=	What to put in the .container file: a User=uid:gid line to pin, not needed (image already sets it), or don't set (s6 images must stay root).
PUID/PGID	Whether to set PUID/PGID in the service's env file — only LinuxServer/s6 images use them.
data_dirs to	The uid/gid that must own the bind-mounted data directory — i.e. the value to use in your data_dirs list.

---

🧩 The application platform

The own projects share their cross-cutting behavior (auth/sessions/MFA, error envelope, cache policy, security headers, SMTP transport, SSE) through the platform repository, checked out at /media/ssd/podman/platform — also the platform service hosting the golden RefApp.

• Every own project's .build quadlet units carry PodmanArgs=--build-context platform=/media/ssd/podman/platform; their Dockerfiles COPY --from=platform the shared crates and packages.
• The checkout must stay readable by every service user — scoped: traverse on its root, read on crates/, packages/ and PLATFORM_VERSION, never the whole tree (the env files carry secrets). Ownership and modes are normalized before builds, never after — Podman keys COPY layers on tar headers. podmanctl --deploy-run re-applies both on every push.
• Rollout of a platform change: push the platform repo (its hook redeploys RefApp only) → verify with the conformance suite → push each app in turn, its own hook deploying it against the platform commit this host now holds. Rollback = revert the platform commit and push it again, then redeploy whatever already picked the bad revision up.

---

🎛️ Podman Control

podmanctl is a single-file Python tool that gives the debian admin account a console dashboard and lifecycle control over every rootless Podman Quadlet service on the server. Each service runs in its own isolated userspace where a plain podman ps from debian sees nothing; podmanctl bridges that gap with a single command.

Features:

• list the table of all containers from all service users, with health status and systemd unit state
• start / stop / restart / update any service stack from the debian account, in dependency order — start/stop/restart/git-pull act on all services at once when no service is given
• update a service end to end: git pull --rebase its repository → re-own what the pull rewrote → pull the published image or rebuild locally → stop → start → reload nginx if the pull changed the vhost → record the outcome where whoever pushed can read it
• upgrade a service: an update that re-pulls every base and registry image first (a full rebuild), then prunes what they superseded
• deploy the commit a git push just landed: one entry point every service's post-receive hook calls, which acknowledges the push and runs the update detached, serialised fleet-wide
• check against the registries (skopeo) to see which images have a newer version available — local builds are checked through the base images of their Dockerfile FROM lines
• git pull (git pull --rebase) one service data directory under /media/ssd/podman/, or every git repository at once when no service is given
• self-check on every run: compares itself with the copy this site publishes and, when they differ, says so with the command that updates it — it never installs anything itself
• fully dynamic discovery: there is no configuration file to maintain; everything is derived live from the conventions:
  • services: every subdirectory of /media/ssd/podman-users/ that has a matching <name>_svc Linux user
  • units, containers, images: parsed from the *.container and *.build Quadlet files in each service's ~/.config/containers/systemd/, which the debian account can read sudo-less thanks to the <name>_admins group ACLs
  • start/stop order: a topological sort of the After=/Requires= relations between the service's own units
  • live state: podman ps and systemctl --user executed inside each service user's session (sudo -u <user> env HOME=<home> XDG_RUNTIME_DIR=/run/user/<uid> ...)

Installing a new service the usual way (new <name>_svc user + Quadlets) makes it appear in the dashboard automatically.

⚙️ Install the script

Install podmanctl to /usr/local/bin

podmanctl

# download the script published by this documentation site
sudo curl -fsSL https://docs.fum-server.fr/files/podmanctl.py -o /usr/local/bin/podmanctl

# make it executable for everyone, writable by root only
sudo chmod 0755 /usr/local/bin/podmanctl

# check the installation: the installed copy is the one this site publishes
podmanctl --self-check

The script is plain Python 3 standard library — no pip package is needed.

Check the installed copy with podmanctl --self-check

Every podmanctl run first compares the installed file with the copy this site publishes, and says so when they differ, with the command that updates it. It never replaces itself: the file runs as root through sudoers, so installing a new one stays a deliberate step.

podmanctl --self-check
podmanctl: outdated - installed 2ef53f1fa27e, published 911c8f072e7f (https://docs.fum-server.fr/files/podmanctl.py)
podmanctl: update it by hand:  sudo curl -fsSL https://docs.fum-server.fr/files/podmanctl.py -o /usr/local/bin/podmanctl

Exit status 0 up to date, 1 outdated, 2 could not check — the site did not answer, which every other command reports in one dim line and goes on. The ids are git blob ids: git log --oneline --find-object=<id> -- docs/files/podmanctl.py in the docs repository names the commit an installed copy came from. A push deploy carries the same lines back over git push, and deploy.ps1 repeats them after its verdict.

---

🚀 Usage

📊 Dashboard

List services with podmanctl --list

podmanctl
SERVICE      USER             CONTAINER          IMAGE                                    STATUS                   UNIT
calibre      calibre_svc      calibre-server     localhost/calibre:7.21.0                 Up 2 days (healthy)      active
gokapi       gokapi_svc       gokapi-server      docker.io/f0rc3/gokapi:latest            Up 2 days (healthy)      active
immich       immich_svc       immich-postgres    ghcr.io/immich-app/postgres:14-vec…      Up 2 days (healthy)      active
immich       immich_svc       immich-redis       docker.io/redis:6.2-alpine               Up 2 days (healthy)      active
immich       immich_svc       immich-server      localhost/immich:v2.7.5                  Up 2 days (healthy)      active
planka       planka_svc       planka-postgres    docker.io/library/postgres:16-alpine     Up 5 hours (healthy)     active
planka       planka_svc       planka-server      ghcr.io/plankanban/planka:2.0.0-rc.4     Exited (1) 3 min ago     failed
...

Check for available updates with podmanctl --list --check

podmanctl --list --check
SERVICE   USER        CONTAINER        IMAGE                                  STATUS                UNIT    UPDATE
gokapi    gokapi_svc  gokapi-server    docker.io/f0rc3/gokapi:latest          Up 2 days (healthy)   active  update available
immich    immich_svc  immich-postgres  ghcr.io/immich-app/postgres:14-vec…    Up 2 days (healthy)   active  up-to-date
immich    immich_svc  immich-redis     docker.io/redis:6.2-alpine             Up 2 days (healthy)   active  up-to-date
immich    immich_svc  immich-server    localhost/immich:v2.7.5                Up 2 days (healthy)   active  up-to-date
...

Images produced by a .build Quadlet are checked through their Dockerfile FROM base images (resolved from the Quadlet's build args): the bases were pulled into the service user's storage at build time, so if a base moved on the registry the build is reported as update available. Local sources (COPY'd code, git content) are not part of the comparison.

🔁 Start / Stop / Restart

Start/Stop/Restart service with podmanctl --<action> <service>

podmanctl --start immich      # daemon-reload, then start (databases first)
podmanctl --stop immich       # stop the stack (server first, databases last)
podmanctl --restart immich    # stop + start

Start/Stop/Restart ALL services with podmanctl --<action>

Omit the service name to apply the action to every detected service, with a summary table at the end:

podmanctl --restart
Stopping calibre ...
  $ (calibre_svc) systemctl --user stop calibre-server.service
Starting calibre ...
  $ (calibre_svc) systemctl --user daemon-reload
  $ (calibre_svc) systemctl --user start calibre-server.service
...
SERVICE      RESULT
calibre      ✅
gokapi       ✅
immich       ✅
planka       ❌
...

✅ done — ❌ error (the failing unit is visible in the live output above the table).

Crash-loop protection

Every .container unit restarts only on failure, with exponential backoff (RestartSec=10 up to RestartMaxDelaySec=300), and gives up after 6 failed starts within 30 minutes (StartLimitBurst=6 / StartLimitIntervalSec=1800). The unit then shows failed with start-limit-hit and stays down instead of crash-looping forever — any podmanctl --start, --restart or --update clears the counter (systemctl --user reset-failed) and brings it back.

⬆️ Update a service

Update a service with podmanctl --update <service>

--update runs, in order: git pull --rebase the service repository → re-own what the pull rewrote → deploy (see below) → stop the stack → swap the image → start it again. The pull comes first, while the stack is still serving: what it checks out is what decides the rest, and a service that cannot be updated keeps running the build it already has — a failed git pull aborts the update and leaves the service up.

How the image is obtained is decided per service from what podmanctl reads on this host — there is no list of own or cloned projects anywhere:

it reads	what --update does	services today
DEPLOY_REGISTRY set in the service's env file	pulls <registry>/<repo>:<app sha>.<platform sha> for every Image=localhost/<repo>, checks its labels, retags it to the Quadlet's Image=. It never builds	the own projects (and platform)
DEPLOY_REGISTRY unset, .build units present	rebuilds the local images from the .build units, reusing the local base images and their layer cache	immich, calibre, vaultwarden, mkdocs
DEPLOY_REGISTRY unset, no .build unit	nothing to build: restarts the stack on the images it already holds	gokapi, planka, registry
a PLATFORM_VERSION file in the checkout	on a push deploy, also re-opens the scoped read access the other apps' builds need	platform only

The decisive env file is this host's copy, which is gitignored — a dev checkout cannot show which path a service takes. Deleting DEPLOY_REGISTRY from it is how an own project falls back to building here. Nothing compiles on the local path of a cloned app: its .build units layer a patch or a pip install over an upstream image.

The tag pairs two commits because an own project's image embeds the platform crates: the same app commit against a different platform is a different image, and pairing them makes deploying a stale combination impossible. Each localhost/<repo> the Quadlet starts maps to <registry>/<repo>, so an app with sidecars (mediascope's tmdb, tripwise's gmaps and osrm) publishes and pulls all of them under one tag. If nothing is published for the commit pair, the update is refused before anything stops — build and push it from a dev machine (./deploy.ps1) and push again.

Either way an own project's image carries the commits it is built from, so the app reports the code it really runs: APP_REVISION, PLATFORM_REVISION and BUILD_TIME are build args its Dockerfile bakes into the runtime stage's last ENV. deploy.ps1 passes them for a published image; for a local build podmanctl hands them to the runtime .build unit through the service user's systemd environment (systemctl --user set-environment, for the length of the builds) — never through the env file, which the container reads too and where they would override the image's own.

The re-own step matters more than it looks: the pull can run as root (the push-deploy hook's worker is root), and Podman keys a COPY layer on the tar headers of the copied files — uid/gid and mtime included, not just the bytes. Chowning the tree back after the build instead makes identical sources hash differently at the next build, and the following deploy recompiles everything for nothing. --update therefore normalizes ownership to <name>_svc:<name>_admins between the pull and the builds (data/ excluded — it belongs to the containers' mapped subuids). A COPY --chown= in the Dockerfile does not paper over it: Podman hashes the headers on the source side, so the flag changes what lands in the image, not the cache key — host-side ownership has to be stable. (Docker BuildKit keys COPY on content alone and never shows this, so a Dockerfile that caches fine in CI can still rebuild everything here.)

Once the stack is back up, two steps close the update:

• nginx — every vhost in sites-enabled is a symlink into its service checkout (nginx/<fqdn>), so a pull that changed it is live on disk but not loaded yet. --update loads it — any change under nginx/ since the last update that brought the stack up, so a vhost a failed deploy left unloaded is picked up by the next one: nginx -t first, then systemctl reload nginx. A configuration nginx -t refuses fails the update and is not loaded — nginx keeps serving the previous one. Fix the file and push again: the broken one stays on disk, and the next reload by anyone would trip on it too.
• the deploy receipt — the outcome is recorded as a ref in the service's bare repository, where the machine that pushed reads it back (see Push deploys).

Build-based (immich, ...)Latest tag (gokapi, planka, ...)Published image (own projects)

# 1. bump the version in the service env file
sudo nano /media/ssd/podman/immich/immich.env     # IMMICH_VERSION=v2.8.0

# 2. stop, git pull the source, rebuild the images and restart
podmanctl --update immich

# nothing to edit, but --update never re-pulls a tag it already holds:
# --upgrade pulls the new :latest, restarts and prunes the old one
podmanctl --upgrade gokapi

# on a dev machine: build for the server's architecture, publish, then push
./deploy.ps1                                     # build -> registry -> git push

# on the server the push hook does the rest; by hand it is:
podmanctl --update claudenames

The image is built where there is CPU to spare — the ODROID is 4 cores / 4 GB and a full dependency cook there is ~20 min per app. --platform linux/arm64 is a target declaration, so the same command runs native on a Windows-on-ARM machine and cross-compiles on x86: the Dockerfile runs rustc on the build machine's own CPU, and only the runtime stage's few RUN steps are emulated.

A service whose data directory is not a git repository simply skips the git pull step (➖) and updates its images as usual.

Upgrade a service with podmanctl --upgrade <service>

--upgrade is --update with the images refreshed first: it re-pulls every Dockerfile FROM base — which invalidates every layer built on them, so expect a full rebuild — and every registry Image= of the stack, starts it, then prunes the images the new ones superseded. --update alone never re-pulls a tag it already holds: a tag only moves when you move it (a bumped *_VERSION) or when --upgrade pulls it again.

podmanctl --list --check        # UPDATE column: which bases/images moved upstream
podmanctl --upgrade gokapi      # re-pull, rebuild what builds here, restart, prune

Version pins live in each service's env file (GOKAPI_VERSION, IMMICH_VERSION, POSTGRES_VERSION, RUST_VERSION, …), which is gitignored: edit it on this host, then --upgrade. A push never moves one. A published own project builds on a dev machine, so there --upgrade refreshes only the registry images its stack pulls (postgres).

---

🪝 Push deploys

A service whose source lives in a bare repository here deploys by git push. The push fires that repository's post-receive hook, which hands the pushed commit to podmanctl as root — one command, the same for every service:

sudo /usr/local/bin/podmanctl --deploy <service> <short-sha>

It runs in two halves, because a push must not wait for a rebuild:

• --deploy — fast and synchronous, its output streaming back over the push. It prints a short acknowledgement (after the self-check's warning, when this podmanctl is not the published one) and detaches the second half in a new session with its file descriptors off the push connection. git push returns in seconds.
• --deploy-run — the detached worker, logging to /var/log/<service>-deploy.log. It takes /run/lock/fleet-deploy.lock, git pull --rebases the checkout, normalizes ownership, re-opens the platform checkout's scoped read access when the service is the platform, and only then runs --update — which ends by reloading nginx when the push changed the vhost, and by writing the deploy receipt.

The lock is fleet-wide on purpose: every service's deploy takes the same one, so two apps never build at once. This host is 4 cores and 4 GB, and parallel container builds spend longer swapping than the same builds take one after another. A push that lands mid-deploy blocks rather than being dropped — it then deploys whatever the branch points at by the time it gets the lock, so the server always converges on the latest pushed commit.

Pull → re-own → (modes) → build is the order that matters, and it is why the worker pulls itself instead of leaving it to --update: --update re-owns the tree but never touches mode bits, and Podman keys a COPY layer on the copied files' tar headers. A chmod or chown after a build makes identical sources hash differently next time, and the following deploy recompiles untouched code for nothing.

Every deploy that gets the lock ends with a receipt in the service's bare repository — a ref pointing at the commit it deployed:

ref	points at
refs/deploy/running	the last commit an update brought up — writing it also deletes the failure ref
refs/deploy/failed	the last commit an update failed on: a pull that failed, a refused or missing image, a failed build or start, a vhost nginx -t refused

The machine that pushed reads them back over the same authenticated transport as the push — no endpoint of its own:

git ls-remote origin 'refs/deploy/*'

A plain git fetch never copies them (its refspec takes refs/heads/* only), and writing one fires no hook. A push that queued behind another deploy deploys whatever the branch holds by then, so a receipt can name a later commit than the one pushed — it still covers it. This is how a cloned app's ./deploy.ps1 knows its push runs, since the app has no /api/version to ask; an own project asks its /api/version instead. A hand-run --update writes one too, so it clears a failure. The receipt needs the clone's origin rewired to the local bare repository (each page's Updating one-time setup): with an https origin there is nowhere to write it, and the deploy log says so.

One-time — the sudoers rule (one for the whole fleet)

# let the git/web user run ONLY the deploy entry point as root, no password
echo 'www-data ALL=(root) NOPASSWD: /usr/local/bin/podmanctl --deploy *' \
  | sudo tee /etc/sudoers.d/podmanctl-deploy

# sudoers files must be 440 - sudo silently ignores files with wrong permissions
sudo chmod 0440 /etc/sudoers.d/podmanctl-deploy

# verify the syntax is valid before relying on it
sudo visudo -cf /etc/sudoers.d/podmanctl-deploy

One rule covers every service. sudo matches command arguments as a single joined string, so the trailing * spans both <service> and <short-sha> — podmanctl validates them itself: the service name must match [a-z0-9][a-z0-9_-]* and resolve to a real <name>_svc account, and the commit must be pure hex, so neither can turn into a path or an option. No second rule is needed for the worker: --deploy is already root when it launches it.

www-data is the user your HTTPS git backend runs hooks as — replace it if yours differs (e.g. if you push over SSH as debian).

Deploy by hand with podmanctl --deploy <service> <short-sha>

# exactly what a push does, without the push
sudo podmanctl --deploy claudenames a1b2c3d

# follow it
sudo tail -f /var/log/claudenames-deploy.log

This is how a commit that is already pushed gets deployed once its image is published (./deploy.ps1 -NoGitPush): its hook ran before the image existed and declined, and no later push will fire it again. ./deploy.ps1 spots that state itself - production's /api/version names another commit than HEAD - publishes the image and prints this command. A cloned app's ./deploy.ps1 prints it too, when the receipt shows HEAD pushed but not running.

The per-service half of the wiring — the bare repository's post-receive hook and the clone's origin / safe.directory — is on each service's own page, in its Updating section.

⬇️ Pull git repositories

Pull service git repositories with podmanctl --git-pull [service]

podmanctl --git-pull            # every service repository (all at once)
  $ git -C /media/ssd/podman/calibre pull --rebase
  $ git -C /media/ssd/podman/immich pull --rebase
  $ git -C /media/ssd/podman/planka pull --rebase
...
SERVICE      RESULT
calibre      ✅
gokapi       ➖
immich       ✅
planka       ❌
...

podmanctl --git-pull immich     # a single service repository
  $ git -C /media/ssd/podman/immich pull --rebase
SERVICE      RESULT
immich       ✅

Omit the service name to pull every repository; pass a <service> to pull only that one. Repositories are detected as /media/ssd/podman/<name>/.git and pulled with git pull --rebase as the debian account — no sudo involved. ✅ new commits pulled — ➖ nothing done (already up to date, or not a git repository) — ❌ pull failed (the git error is printed below the table).

📜 Logs

Follow a service's journal with podmanctl --logs <service>

podmanctl --logs immich       # sudo journalctl _UID=<immich_svc uid> -n 50 -f

---

🗑️ Removing a Service

Removing a service reverses its setup:

• stops the running stack
• deletes the dedicated Linux user
• delete its admin group and rootless ID mappings
• erases both the per-user Podman home and the service data directory

Define Service Variables

# define the service name
SERVICE_NAME=XXX

Replace XXX by your service name

Initialize Service Environment

# built-in bash safety and SERVICE_NAME validation
set -euo pipefail; [ -n "${SERVICE_NAME:-}" ] || { echo "SERVICE_NAME is empty"; exit 1; }

# automatically generate variables
SERVICE_HOME="/media/ssd/podman-users/${SERVICE_NAME}"
SERVICE_DIR="/media/ssd/podman/${SERVICE_NAME}"
SERVICE_USER="${SERVICE_NAME}_svc"
SERVICE_ADMIN="${SERVICE_NAME}_admins"

Removing a service entirely

# show the absolute path and size of each directory to be deleted
echo "The following directories will be permanently deleted for '${SERVICE_NAME}':"
for dir in "${SERVICE_HOME}" "${SERVICE_DIR}"; do
    if [ -d "${dir}" ]; then
        printf '  %s  (%s)\n' "${dir}" "$(sudo du -sh "${dir}" 2>/dev/null | cut -f1)"
    else
        printf '  %s  (not found)\n' "${dir}"
    fi
done

# ask for confirmation, abort otherwise (read from the terminal, not the pasted input)
read -r -p "Type 'yes' to confirm permanent removal: " CONFIRM </dev/tty
[ "${CONFIRM}" = "yes" ] || { echo "Aborted, nothing was removed."; exit 1; }

# stop the service stack from the debian admin account
podmanctl --stop "${SERVICE_NAME}"

# retrieve the service user's numeric UID (needed to stop its systemd --user instance)
SERVICE_UID="$(id -u "${SERVICE_USER}")"

# disable lingering so the user's systemd --user manager no longer starts at boot
sudo loginctl disable-linger "${SERVICE_USER}"

# stop the service user's systemd --user instance (kills any container still running)
sudo systemctl stop "user@${SERVICE_UID}.service"

# remove the service user (its home directory is deleted explicitly below)
sudo userdel "${SERVICE_USER}"

# remove the per-user Podman home and the service data directory
sudo rm -rf "${SERVICE_HOME}" "${SERVICE_DIR}"

# remove the admin group (debian's membership is dropped with it)
sudo groupdel "${SERVICE_ADMIN}"

# remove the rootless subuid/subgid mappings (in case userdel left them behind)
sudo sed -i "/^${SERVICE_USER}:/d" /etc/subuid /etc/subgid

The shared no-ssh group and the /etc/ssh/sshd_config.d/99-deny-service-users.conf deny rule are left untouched, they are used by every service user.