#!/usr/bin/env python3
"""ODG for Linux.

Installs and updates the Operator Development Group pack for OPERATOR on Linux,
where Steam runs the game through Proton. It does the job ODG Ops does on
Windows: the same releases, the same checksums and the same stamp files, so a
game folder that has seen both stays consistent.

    python3 odg-linux.py            install ODG, or update it
    python3 odg-linux.py help       every command

Kept in step with the Windows launcher (odg-launcher/src): update.rs for the
release rules, melonsetup.rs for the loader, stale.rs and uninstall.rs for what
may be deleted, access.rs for accounts, terms.rs for the terms.
"""

import collections
import concurrent.futures
import errno
import hashlib
import http.client
import json
import os
import re
import shutil
import ssl
import stat
import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import zipfile
import zlib
from pathlib import Path

ODG_LINUX_TOOL = True
# THE LINUX VERSION is the public release this copy went out with: ODG for Linux 2.2.3 shipped
# beside ODG 2.2.3. A Linux-only fix between releases adds a fourth number (2.2.3.1). Every public
# release moves this to the new number, then stage-site.py and a site deploy publish it; copies
# already installed replace themselves on their next run (compare() reads "1" < "2.2.3").
TOOL_VERSION = "3.8.7"

APPID = "1913370"
PUBLIC_API = "https://api.github.com/repos/ArchdukePierre/operator-development-group/releases/latest"
SITE = "https://operatordevgroup.com"
TOOL_KEY = "odg-linux"
DEFAULT_SERVER = "https://account.operatordevgroup.com"
USER_AGENT = "ODGModManager"
GITHUB = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}
LAUNCH_OPTION = 'WINEDLLOVERRIDES="version=n,b" %command%'
GRACE_DAYS = 10
OFF = ".odg-off"
HELD = "version.dll.odg-hold"

STAMP = "Mods/.installed_release"
PARTS_STAMP = "Mods/.installed_parts"
MAP_ROOT = "Mods/MapLoader_Assets"
MAP_STAMP = "Mods/MapLoader_Assets/.installed_mapdata"
SHIPPED_LIST = "Mods/.odg-shipped-files.txt"
OBSOLETE_LIST = "Mods/.odg-obsolete.txt"
PREV_LIST = "Mods/.odg-shipped-files.prev"
FORGOTTEN = "Mods/.installed_release.forgotten"
DEV_HOLD = "Mods/.odg-dev-hold"
# the add-ons' records: <id>.files, <id>.receipt, installed.json (update.rs ADDON_DIR)
ADDON_DIR = "Mods/ODG_Addons"
STAGING = "UserData/ODGModManager/cache"
CUSTOM = "Mods/MapLoader_Assets/Custom"

# What "the mod loader is installed" means: the proxy, the loader, its runtime
# config, and the private .NET 6 it starts. Under Proton that runtime is what
# MelonLoader uses, so nothing has to be installed into the Proton prefix.
LOADER_FILES = [
    "version.dll",
    "MelonLoader/net6/MelonLoader.dll",
    "MelonLoader/net6/MelonLoader.runtimeconfig.json",
    "MelonLoader/Dependencies/dotnet/dotnet.exe",
]

# 3.6 (11-13, the launcher's IN_THE_NUMBERING): co-op sends every gun and gun part as a NUMBER, counted in a
# fixed order across these mods. One of them off renumbers everything after it on that machine, and guns come
# out wrong or invisible for the whole lobby, so they stay on, the same as in the launcher.
IN_THE_NUMBERING = "Co-op counts every gun and part in a fixed order. With this off, guns come out wrong or invisible for the whole lobby, so it stays on."

# The launcher's names for ODG's mods (mods.rs). A reason marks a mod that stays on.
KNOWN = {
    "maploader.mod.dll": ("ODG Map Loader", "Carries the ODG Ops tab and every custom map. Off, the operations are gone."),
    "ammolock.mod.dll": ("ODG AmmoLock", "Keeps ammunition fair for everyone in an ODG lobby, so it stays on."),
    "diagnostics.mod.dll": ("ODG Diagnostics", "Records what happened when something breaks. Off, a crash report says nothing."),
    "hsr.mod.dll": ("M200 Intervention", IN_THE_NUMBERING),
    "m107.mod.dll": ("Barrett M107A1", IN_THE_NUMBERING),
    "vks.mod.dll": ("VKS", None),
    "fireteamroles.mod.dll": ("Fireteam Roles", None),
    "airborneinfil.mod.dll": ("Airborne Infiltration", None),
    "newslots.mod.dll": ("New Slots", None),
    "loadoutpresets.mod.dll": ("Loadout Presets", None),
    "quartermaster.mod.dll": ("Quartermaster", None),
    "thirdperson.mod.dll": ("Third Person", None),
    "operatorcompass.mod.dll": ("Operator Compass", None),
    "spectatorname.mod.dll": ("Spectator Name", None),
    "nofatigue.mod.dll": ("No Fatigue", None),
    "hostpin.mod.dll": ("Host Pin", None),
    "infilmarker.mod.dll": ("Infil Markers", None),
    "smoke.mod.dll": ("ODG Smoke", IN_THE_NUMBERING),
    "m14.mod.dll": ("Mk 14 EBR", IN_THE_NUMBERING),
    "arms.mod.dll": ("ODG Arms", IN_THE_NUMBERING),
    "harbinger.mod.dll": ("ODG Harbinger", IN_THE_NUMBERING),
    "camo.mod.dll": ("ODG Camo", IN_THE_NUMBERING),
    "silencers.mod.dll": ("ODG Silencers", IN_THE_NUMBERING),
    "optics.mod.dll": ("ODG Optics", IN_THE_NUMBERING),
    "thermal.mod.dll": ("ODG Thermal", IN_THE_NUMBERING),
    "rest.mod.dll": ("ODG Rest", None),
    "sling.mod.dll": ("ODG Sling", None),
}

# Retired files earlier packs left behind (cleanup.rs). Exact names in Mods only.
RETIRED_FILES = ["ODGGuard.Mod.dll", "ODGGuard.Mod.dll.odg-off", "manifest.odgsig",
                 "Weather.Mod.dll", "Weather.Mod.dll.odg-off", "Weather.log"]
RETIRED_DIRS = ["Weather_Assets"]

# Files the game or the mods write into the pack's folders (stale.rs). Never leftovers.
ALLOWLISTED = {".installed_release", ".installed_mapdata", ".installed_parts", ".modpack_version",
               "door_placements.json", "ai_skeleton.json", "thumbs.db", "desktop.ini"}

# .installed_parts left behind told the next install the pack's parts were
# already there, and NewSlots came up with no content (uninstall.rs, 10-15).
UNINSTALL_MARKERS = [
    "Mods/.odg-shipped-files.txt", "Mods/.odg-shipped-files.prev", "Mods/.odg-obsolete.txt",
    "Mods/.installed_release", "Mods/.installed_release.forgotten", "Mods/.installed_parts",
    "Mods/.modpack_version", "Mods/MapLoader_Assets/.installed_mapdata",
]

CUSTOM_README = (
    "Put your own maps in here. One folder per map.\n"
    "\n"
    "A map is the folder the build tool wrote, copied in whole:\n"
    "it has manifest.bin and gameplay.bin at the top, and meshes and textures\n"
    "folders next to them. If manifest.bin is not there it is not a map yet.\n"
    "\n"
    "Do not put maps next to this folder. Anything up there that is not part of\n"
    "the pack gets removed on the next update. In here is safe.\n"
)

# THE TERMS, word for word as the Windows launcher shows them (terms.rs). The
# version is the contract: change a word there, change it here, and move both.
TERMS_VERSION = "2026-11-06"
TERMS_URL = "https://archduke-interactive.com/terms"
TERMS = [
    ("WE ARE NOT THE GAME",
     "Archduke Interactive is a group of players. We are not connected to the people who make or publish OPERATOR, and nothing here is official. If the game itself breaks, that is theirs. If something we made breaks, tell us and we will fix it."),
    ("WHAT IT DOES TO YOUR GAME FOLDER",
     "It installs the mod pack into your OPERATOR folder, and it renames mod files in place to turn them on and off. It installs the mod loader if you do not have one. It removes files that an older pack of ours left behind, and it asks first. When you move between a test build and the public one, it removes what only the other build had. SETTINGS > UNINSTALL takes all of it off again, the launcher included. It never touches your saves, and it never deletes anything it did not put there. Your own mods and your own maps are left alone, always."),
    ("WHAT LEAVES YOUR PC",
     "For the free release: nothing. It downloads from GitHub and tells us nothing else. For a subscriber: your launcher key and the launcher version, to our server, so it can answer whether your subscription is live and which build your tier is on. No telemetry, no game files, no saves, no hardware. A crash report leaves only when you press SEND on the REPORT page, and only from an account signed in with Steam. It carries the logs the game and the mods wrote, your mod settings, the names of the files in your game folder, a description of your PC (processor, graphics card and driver, memory, motherboard), what Windows recorded about the game crashing, your account name, the Steam account you signed in with, which Steam account your PC is on, your Discord name if you add it, and the crash dump if you leave it in. Your Windows user name and PC name are blanked out first. A message you send on a report goes the same way. SAVE THE ZIP INSTEAD shows you exactly what would go. While you are signed in with Steam, the launcher also tells our server which Steam account owns OPERATOR on your PC, so a launcher passed to somebody else shows up."),
    ("IF YOU SIGN IN WITH STEAM",
     "You sign in on Steam's own page, and we never see your Steam password. Steam tells us the number of the account you signed in with. From that account's public profile we read its name, and whether it is a limited account or has a VAC ban on record, and we keep those with your account. Sending a crash report needs it."),
    ("IF YOU SIGN IN WITH PATREON",
     "Patreon tells us your member id, the name on your account, whether you are an active patron, what you are entitled to, and whether your last payment went through. We keep those so the launcher knows a subscription is live and paid. We never see your card, and we do not ask Patreon for your address. If you would rather send us none of that, use an invite code instead; it tells us only the name and tier on the code."),
    ("WHAT WE KEEP ABOUT YOU",
     "One row: a name, your tier, whether access is live, a one-way hash of your key, your Patreon member id if you used that, the pledge amount, when your launcher last checked in, its version, which version of these terms you accepted, and your GitHub username if you asked for the developer repository, and the Steam account you signed in with. No email address. No payment details. A crash report you send is kept on our server and in a private GitHub repository only ODG's developers can open: the zip for 90 days, or 30 if it holds a crash dump, and the ticket, your note and the messages on it until you ask, or until we clear out closed reports. Ask and we will delete the row or any report."),
    ("THE DEVELOPER REPOSITORY",
     "Two tiers reach the private repository. That is a real GitHub invite, so it needs your GitHub username, and a person confirms it by hand before anything is sent. Do not pass what is in there to people who did not pay for it. That is the one thing that ends access on the spot."),
    ("ACCESS CAN END",
     "Your subscription ends when you cancel, when a payment fails, or when we end it for sharing access or for conduct that spoils the channels. When it ends this launcher stops starting the game and stops treating the pack as installed. It then offers you three things: resubscribe, remove the pack, or take the free launcher and carry on with the public release. Nothing is deleted from your PC unless you press delete yourself."),
    ("SUPPORTING US",
     "Supporting us on Patreon buys a voice and our thanks. Connect it in the launcher and your tier shows in the corner. If what supporters get ever changed, it would change on the website before it changed here."),
    ("NO WARRANTY",
     "This is unofficial software made by volunteers and it comes as it is. We test it and we use it ourselves, and we still cannot promise it will not break a save, a mod or an afternoon. Back up anything you would be sorry to lose. Using it is your decision."),
    ("SHARING, AND WHAT IS NOT OURS TO GIVE",
     "Our own work is yours to pass on. Do not sell it, do not put it behind a paywall, and do not present it as yours. Some of what the pack carries came from elsewhere and is not ours to licence, so sharing the pack is not permission to lift the pieces out of it."),
]


# ---------------------------------------------------------------------------
# output
# ---------------------------------------------------------------------------

TTY = sys.stdout.isatty()


def say(msg=""):
    print(msg, flush=True)


def mb(n):
    n = max(int(n or 0), 0)
    if n >= 1048576:
        return "{:,.0f} MB".format(n / 1048576)
    return "{:,.0f} KB".format(max(n / 1024, 1))


class Problem(Exception):
    """Something went wrong, in words meant for the player."""


class NoAnswer(Exception):
    """The server could not be reached, or what came back was not an answer."""


class Refused(Exception):
    """The server answered with an HTTP error."""

    def __init__(self, status, body=""):
        Exception.__init__(self, "HTTP {}".format(status))
        self.status = status
        self.body = body


class Bar:
    """One progress line, redrawn in place on a terminal."""

    def __init__(self, label, total):
        self.label = label
        self.total = max(int(total or 0), 0)
        self.last = 0.0
        self.shown = -1

    def update(self, got):
        if TTY:
            now = time.monotonic()
            if now - self.last < 0.2 and got < self.total:
                return
            self.last = now
            if self.total:
                line = "  {}: {} of {} ({}%)".format(self.label, mb(got), mb(self.total),
                                                    min(100, 100 * got // self.total))
            else:
                line = "  {}: {}".format(self.label, mb(got))
            sys.stdout.write("\r" + line.ljust(72))
            sys.stdout.flush()
        elif self.total:
            step = min(100, 100 * got // self.total) // 25 * 25
            if step != self.shown:
                self.shown = step
                say("  {}: {}%".format(self.label, step))

    def done(self):
        if TTY:
            sys.stdout.write("\n")
            sys.stdout.flush()


# ---------------------------------------------------------------------------
# small helpers
# ---------------------------------------------------------------------------

def now_unix():
    return int(time.time())


def parts_of(rel):
    return [p for p in str(rel).replace("\\", "/").split("/") if p not in ("", ".")]


def norm(rel):
    s = str(rel).strip().replace("\\", "/")
    while s.startswith("./"):
        s = s[2:]
    return s.lstrip("/")


def read_list(text):
    out = []
    for line in text.splitlines():
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        line = norm(line)
        if line:
            out.append(line)
    return out


def safe_rel(rel):
    """Under Mods, no '..', no drive letter, either separator (stale.rs)."""
    lower = rel.lower()
    trimmed = rel.rstrip("/\\")
    return (lower.startswith("mods/") and lower != "mods/" and ":" not in rel
            and all(seg not in ("", ".", "..") for seg in re.split(r"[/\\]", trimmed)))


# ---- add-ons (2026-10-28, ADDONS_3_0.md; the launcher's update.rs) ----------
# What is paid never ships in the free pack. Each paid part is an ADD-ON in a
# private release only the account server reads, and this tool asks the server
# for exactly the ones the account owns. Each comes with a receipt the server
# signs, naming the account, kept beside it.

def addon_word(s):
    return isinstance(s, str) and re.fullmatch(r"[a-z0-9-]{1,40}", s) is not None


def parse_offers(doc):
    """The server's list, read. Anything malformed is left out."""
    out = []
    for x in (doc.get("addons") if isinstance(doc, dict) else None) or []:
        if not isinstance(x, dict) or not addon_word(x.get("id")):
            continue
        s = {k: (x.get(k) if isinstance(x.get(k), str) else "") for k in ("title", "digest", "updated_at", "receipt", "receipt_sig")}
        size = x.get("size")
        s.update(id=x["id"], size=size if isinstance(size, int) and size > 0 else 0)
        out.append(s)
    return out


def parse_owned(doc):
    """What the account owns, from the same answer: the only list an add-on is
    ever taken off the disk by. None when the answer carries none, and then
    nothing is removed. It differs from the list sent when the add-ons are
    switched off or a file is between uploads, and then the copy stays."""
    lst = doc.get("owned") if isinstance(doc, dict) else None
    if not isinstance(lst, list):
        return None
    return {x for x in lst if addon_word(x)}


def addons_to_remove(state, owned):
    if owned is None:
        return []
    return sorted(k for k in state if k not in owned)


def addon_entry_ok(name):
    """An add-on's zip may only put files under Mods, never climb out, and never
    write over the add-on records. The bare Mods/ folder entry is fine."""
    n = str(name).replace("\\", "/")
    low = n.lower()
    if low == "mods/":
        return True
    rec = ADDON_DIR.lower()
    return safe_rel(n) and not low.startswith(rec + "/") and low != rec


def addon_files(disk):
    """Every file an add-on put down (its .files list), switched-off DLLs too:
    no pack list judges them (stale.rs)."""
    out = set()
    d = disk.find(ADDON_DIR)
    if d is None or not d.is_dir():
        return out
    try:
        names = os.listdir(str(d))
    except OSError:
        return out
    for n in names:
        if not n.lower().endswith(".files"):
            continue
        for line in disk.read("{}/{}".format(ADDON_DIR, n)).splitlines():
            l = norm(line).lower()
            if not l:
                continue
            out.add(l)
            if l.endswith(".dll"):
                out.add(l + OFF)
    return out


# ODG ZOMBIES AND ODG CINECAM ARE NEVER LEFTOVERS (stale.rs FREE_MODS, 11-07). Every public
# update to v3.2.1 installed them and then deleted them: the public pack's obsolete list still
# carried a 3.1-era block naming them. They are free and in both packs now, so no list (obsolete
# or shipped) and no move between builds takes them: a line naming them is ignored, and delete
# keeps them whatever it is handed. Uninstall is the one way they go.
FREE_MODS = ("mods/zombies.mod.dll", "mods/zombies_content", "mods/cinecam.mod.dll", "mods/cinecam_data")


def free_mod_path(rel):
    """`rel` (any case, either slash) is one of the free mods, its switched-off DLL,
    something inside its folder, or a folder that holds one of them (Mods itself)."""
    l = str(rel).strip().replace("\\", "/").rstrip("/").lower()
    for root in FREE_MODS:
        if l == root or l == root + OFF or l.startswith(root + "/") or root.startswith(l + "/"):
            return True
    return False


def zip_names(path):
    try:
        with zipfile.ZipFile(str(path)) as z:
            return [i.filename.replace("\\", "/") for i in z.infolist()]
    except (zipfile.BadZipFile, OSError) as e:
        raise Problem("it is not a readable zip ({})".format(e))


def is_custom(rel_lower):
    return "/maploader_assets/custom/" in rel_lower or rel_lower.endswith("/maploader_assets/custom")


def remove_quietly(path):
    try:
        os.remove(str(path))
    except OSError:
        pass


def sha256_file(path):
    h = hashlib.sha256()
    with open(str(path), "rb") as f:
        while True:
            chunk = f.read(1 << 20)
            if not chunk:
                break
            h.update(chunk)
    return h.hexdigest()


def write_atomic(path, text):
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    tmp = path.with_name(path.name + ".tmp")
    with open(str(tmp), "w", encoding="utf-8", newline="\n") as f:
        f.write(text)
    os.replace(str(tmp), str(path))


def ask(question, default=False):
    """Yes or no. With no terminal to ask, the default stands."""
    if not sys.stdin or not sys.stdin.isatty():
        return default
    hint = "[Y/n]" if default else "[y/N]"
    try:
        answer = input("{} {} ".format(question, hint)).strip().lower()
    except EOFError:
        return default
    if not answer:
        return default
    return answer in ("y", "yes")


# ---------------------------------------------------------------------------
# versions (version.rs)
# ---------------------------------------------------------------------------

def parse_version(s):
    s = str(s or "").strip().lstrip("vV")
    if not s:
        return None
    m = re.search(r"[-+_ ]", s)
    if m:
        nums, label = s[:m.start()], s[m.start() + 1:].strip().lower()
    else:
        nums, label = s, ""
    parts = []
    for piece in nums.split("."):
        piece = piece.strip()
        if not piece:
            continue
        if not piece.isdigit():
            return None
        parts.append(int(piece))
    if not parts:
        return None
    return parts, label


def compare(mine, theirs):
    """'behind', 'same', 'ahead', or 'unclear' when either does not parse."""
    a, b = parse_version(mine), parse_version(theirs)
    if a is None or b is None:
        return "unclear"
    n = max(len(a[0]), len(b[0]))
    for i in range(n):
        x = a[0][i] if i < len(a[0]) else 0
        y = b[0][i] if i < len(b[0]) else 0
        if x != y:
            return "behind" if x < y else "ahead"
    la, lb = a[1], b[1]
    if la == lb:
        return "same"
    if not la:
        return "ahead"
    if not lb:
        return "behind"
    return "behind" if la < lb else "ahead"


def lineage(tag):
    """'v1.0.1' is the public line, 'v109' the developer one (update.rs)."""
    t = re.split(r"[-+]", str(tag or "").strip())[0]
    if not t:
        return ""
    return "public" if "." in t else "dev"


# ---------------------------------------------------------------------------
# http
# ---------------------------------------------------------------------------

RETRY_STATUS = (408, 429, 500, 502, 503, 504)


def _local_override(var):
    """Tests point at a server on this PC. Never anywhere else (update.rs, access.rs)."""
    v = os.environ.get(var, "").strip()
    # The port must end the host part: "http://127.0.0.1:1@other.host" reads as a
    # user name in front of another server, and the member key would go there.
    if re.match(r"http://(127\.0\.0\.1|localhost):[0-9]{1,5}(/[^\s]*)?$", v):
        return v.rstrip("/")
    return None


def public_api():
    return _local_override("ODG_TEST_PUBLIC_API") or PUBLIC_API


def server():
    return _local_override("ODG_SERVER") or DEFAULT_SERVER


def site():
    return _local_override("ODG_TEST_SITE") or SITE


def _request(url, key=None, headers=None, data=None, method=None):
    req = urllib.request.Request(url, data=data, method=method)
    req.add_header("User-Agent", USER_AGENT)
    for k, v in (headers or {}).items():
        if k.lower() == "x-odg-launcher":
            # for our server only: never carried onto a 302 to GitHub's storage
            req.add_unredirected_header(k, v)
        else:
            req.add_header(k, v)
    if key:
        # Never carried onto a redirect. The account server answers a download
        # with a 302 to GitHub's storage, and the member key stops at our server.
        req.add_unredirected_header("Authorization", "Bearer " + key)
    return req


def http_open(url, key=None, headers=None, data=None, method=None, timeout=30, attempts=5):
    last = None
    for i in range(attempts):
        try:
            return urllib.request.urlopen(_request(url, key, headers, data, method), timeout=timeout)
        except urllib.error.HTTPError as e:
            body = ""
            try:
                body = e.read().decode("utf-8", "replace")
            except Exception:
                pass
            if e.code in RETRY_STATUS and i + 1 < attempts:
                last = e
                time.sleep(2)
                continue
            raise Refused(e.code, body)
        except (urllib.error.URLError, OSError, http.client.HTTPException) as e:
            last = e
            if i + 1 < attempts:
                time.sleep(2)
                continue
            raise NoAnswer(str(getattr(e, "reason", e)))
    raise NoAnswer(str(last))


def http_json(url, key=None, headers=None, timeout=60, attempts=5):
    r = http_open(url, key, headers, timeout=timeout, attempts=attempts)
    try:
        raw = r.read()
    except (OSError, http.client.HTTPException) as e:
        raise NoAnswer(str(e))
    finally:
        r.close()
    try:
        return json.loads(raw.decode("utf-8"))
    except ValueError:
        raise NoAnswer("the answer was not a release listing")


def account_headers():
    return {"Accept": "application/json", "X-ODG-Launcher": "linux " + TOOL_VERSION}


def account_call(path, key=None, body=None, extra=None, attempts=1):
    """A call to the account server. Its answers carry an 'error' field rather
    than relying on the status code, so any JSON body is an answer."""
    headers = account_headers()
    headers.update(extra or {})
    data = None
    if body is not None:
        data = json.dumps(body).encode("utf-8")
        headers["Content-Type"] = "application/json"
    try:
        r = http_open(server() + path, key, headers, data=data,
                      method="POST" if data is not None else "GET", timeout=20, attempts=attempts)
        try:
            raw = r.read()
        finally:
            r.close()
    except Refused as e:
        raw = (e.body or "").encode("utf-8")
        if not raw.strip():
            raise NoAnswer("the account server answered {}".format(e.status))
    except (OSError, http.client.HTTPException) as e:
        raise NoAnswer(str(e))
    try:
        doc = json.loads(raw.decode("utf-8"))
    except ValueError:
        raise NoAnswer("the account server's answer could not be read")
    if not isinstance(doc, dict):
        raise NoAnswer("the account server's answer could not be read")
    return doc


def check_space(folder, need):
    """Refuse only when failure is certain: the archive alone will not fit (preflight.rs)."""
    if not need:
        return
    try:
        free = shutil.disk_usage(str(folder)).free
    except OSError:
        return
    margin = 64 * 1024 * 1024
    if free < need + margin:
        raise Problem("not enough disk space: {} free, {} needed".format(mb(free), mb(need + margin)))


def download(url, dest, size, label, key=None, extra=None):
    """Stream url to dest. A dropped connection picks up where it stopped."""
    dest = Path(dest)
    remove_quietly(dest)
    bar = Bar(label, size)
    got = 0
    tries = 0
    while True:
        tries += 1
        headers = dict(extra or {})
        headers["Accept"] = "application/octet-stream"
        if got:
            headers["Range"] = "bytes={}-".format(got)
        try:
            r = http_open(url, key, headers, timeout=60, attempts=1)
        except Refused as e:
            if e.status not in RETRY_STATUS or tries >= 5:
                bar.done()
                raise Problem("the download was refused (HTTP {})".format(e.status))
            time.sleep(2)
            continue
        except NoAnswer as e:
            if tries >= 5:
                bar.done()
                raise Problem("the download failed: {}".format(e))
            time.sleep(2)
            continue
        try:
            if got and getattr(r, "status", 200) != 206:
                got = 0
            with open(str(dest), "ab" if got else "wb") as f:
                while True:
                    chunk = r.read(1 << 20)
                    if not chunk:
                        break
                    f.write(chunk)
                    got += len(chunk)
                    bar.update(got)
        except OSError as e:
            if getattr(e, "errno", None) == errno.ENOSPC:
                bar.done()
                remove_quietly(dest)
                raise Problem("the disk filled up during the download")
            if tries >= 5:
                bar.done()
                raise Problem("the download failed: {}".format(e))
            time.sleep(2)
            continue
        except http.client.HTTPException as e:
            if tries >= 5:
                bar.done()
                raise Problem("the download failed: {}".format(e))
            time.sleep(2)
            continue
        finally:
            r.close()
        if size and got < size:
            if tries >= 5:
                bar.done()
                raise Problem("the download stopped early ({} of {})".format(mb(got), mb(size)))
            continue
        break
    bar.done()
    if got == 0:
        raise Problem("downloaded nothing")


def verify(path, size, digest):
    """The size and sha256 GitHub publishes with every asset (sha256.rs)."""
    got = os.path.getsize(str(path))
    if size and got != size:
        raise Problem("the download is {} bytes and the release says {}".format(got, size))
    digest = str(digest or "").strip()
    if digest.lower().startswith("sha256:"):
        if sha256_file(path).lower() != digest[7:].strip().lower():
            raise Problem("the download does not match the release's checksum")


# ---------------------------------------------------------------------------
# the game folder, without regard to case
# ---------------------------------------------------------------------------

class Disk:
    """The game folder, matched the way Windows and Proton match it.

    Proton finds Mods\\MapLoader_Assets whatever case it is written in. A Linux
    disk does not, so every path here is matched against what is on disk, and a
    new folder takes the pack's spelling only when no spelling of it exists
    yet. That keeps one Mods folder, never a Mods and a mods.
    """

    def __init__(self, root):
        self.root = Path(root)
        self._ls = {}

    def _names(self, d):
        k = str(d)
        m = self._ls.get(k)
        if m is None:
            m = {}
            try:
                for n in os.listdir(k):
                    m.setdefault(n.lower(), []).append(n)
            except OSError:
                pass
            self._ls[k] = m
        return m

    def forget(self, d=None):
        if d is None:
            self._ls.clear()
        else:
            self._ls.pop(str(d), None)

    def _step(self, cur, part):
        hits = self._names(cur).get(part.lower())
        if not hits:
            return None
        return cur / (part if part in hits else sorted(hits)[0])

    def find(self, rel):
        cur = self.root
        for part in parts_of(rel):
            nxt = self._step(cur, part)
            if nxt is None:
                return None
            cur = nxt
        return cur

    def note(self, path):
        names = self._names(path.parent)
        spellings = names.setdefault(path.name.lower(), [])
        if path.name not in spellings:
            spellings.append(path.name)

    def _folder(self, cur, part):
        """cur/part as a folder, made if needed.

        Some Linux unzip tools write a pack's folder entries as empty files, so
        a game folder that had the pack copied in by hand can hold an empty
        MapLoader_Assets FILE. An empty file where the pack needs a folder is
        replaced. A spelling that is already a folder wins over one that is not.
        """
        spellings = sorted(self._names(cur).get(part.lower(), []), key=lambda n: (n != part, n))
        for name in spellings:
            if (cur / name).is_dir():
                return cur / name
        for name in spellings:
            q = cur / name
            try:
                st = os.lstat(str(q))
            except OSError:
                continue
            if stat.S_ISREG(st.st_mode) and st.st_size == 0:
                os.remove(str(q))
                self.forget(cur)
                say("Replaced an empty file named {} with a folder.".format(self.rel(q)))
                spellings = []
                break
            raise Problem("{} is a file where the pack needs a folder. Move it out of the way and run this again."
                          .format(self.rel(q)))
        q = cur / part
        try:
            q.mkdir()
        except FileExistsError:
            pass
        except OSError as e:
            raise Problem("could not make {}: {}".format(self.rel(q), e))
        if not q.is_dir():
            raise Problem("could not make {}".format(self.rel(q)))
        self.note(q)
        return q

    def place(self, rel):
        """Where rel is, or will be. Folders on the way are made as needed."""
        cur = self.root
        ps = parts_of(rel)
        for i, part in enumerate(ps):
            if i < len(ps) - 1:
                cur = self._folder(cur, part)
            else:
                cur = self._step(cur, part) or cur / part
        return cur

    def make_dir(self, rel):
        ps = parts_of(rel)
        parent = self.place("/".join(ps[:-1]) + "/x").parent if len(ps) > 1 else self.root
        return self._folder(parent, ps[-1])

    def rel(self, path):
        try:
            return Path(path).relative_to(self.root).as_posix()
        except ValueError:
            return str(path)

    def isfile(self, rel):
        p = self.find(rel)
        return p is not None and p.is_file()

    def read(self, rel, default=""):
        p = self.find(rel)
        if p is None:
            return default
        try:
            return p.read_text(encoding="utf-8", errors="replace")
        except OSError:
            return default

    def write(self, rel, text):
        p = self.place(rel)
        write_atomic(p, text)
        self.note(p)

    def remove(self, rel):
        p = self.find(rel)
        if p is not None:
            remove_quietly(p)
            self.forget(p.parent)


def extract(zip_path, disk, label):
    """Unpack over the game folder. The archive root holds Mods/... and the
    rest, so every file lands where it belongs."""
    try:
        z = zipfile.ZipFile(str(zip_path))
    except (zipfile.BadZipFile, OSError) as e:
        raise Problem("{} is not a readable zip ({})".format(label, e))
    with z:
        infos = z.infolist()
        bar = Bar("unpacking " + label, sum(i.file_size for i in infos))
        done = 0
        try:
            for info in infos:
                ps = parts_of(info.filename)
                if not ps or ".." in ps or ":" in ps[0]:
                    continue
                rel = "/".join(ps)
                if info.filename.endswith("/"):
                    disk.make_dir(rel)
                    continue
                dest = disk.place(rel)
                if dest.is_dir():
                    raise Problem("{}: a folder is where a file goes".format(rel))
                with z.open(info) as src:
                    try:
                        f = open(str(dest), "wb")
                    except PermissionError:
                        os.chmod(str(dest), stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
                        f = open(str(dest), "wb")
                    with f:
                        shutil.copyfileobj(src, f, 1 << 20)
                disk.note(dest)
                done += info.file_size
                bar.update(done)
        except zipfile.BadZipFile as e:
            bar.done()
            raise Problem("{} is damaged ({})".format(label, e))
        except OSError as e:
            bar.done()
            if getattr(e, "errno", None) == errno.ENOSPC:
                raise Problem("the disk filled up while unpacking {}. Free some space and run this again".format(label))
            raise Problem("could not write a file from {}: {}".format(label, e))
        bar.done()


# ---------------------------------------------------------------------------
# steam: where the game is
# ---------------------------------------------------------------------------

class _Open:
    pass


class _Close:
    pass


OPEN, CLOSE = _Open(), _Close()


def vdf_tokens(text):
    out = []
    i, n = 0, len(text)
    esc = {"n": "\n", "t": "\t", "\\": "\\", '"': '"'}
    while i < n:
        c = text[i]
        if c in " \t\r\n":
            i += 1
        elif c == "/" and text.startswith("//", i):
            j = text.find("\n", i)
            i = n if j < 0 else j + 1
        elif c == "{":
            out.append(OPEN)
            i += 1
        elif c == "}":
            out.append(CLOSE)
            i += 1
        elif c == '"':
            i += 1
            buf = []
            while i < n and text[i] != '"':
                if text[i] == "\\" and i + 1 < n:
                    buf.append(esc.get(text[i + 1], "\\" + text[i + 1]))
                    i += 2
                    continue
                buf.append(text[i])
                i += 1
            i += 1
            out.append("".join(buf))
        else:
            j = i
            while j < n and text[j] not in ' \t\r\n{}"':
                j += 1
            out.append(text[i:j])
            i = j
    return out


def vdf_parse(text):
    """Steam's text KeyValues as nested dicts. Keys are lower-cased; Steam has
    written both cases of the same key over the years."""
    toks = vdf_tokens(text)
    pos = [0]

    def block():
        d = {}
        while pos[0] < len(toks):
            t = toks[pos[0]]
            if t is CLOSE:
                pos[0] += 1
                return d
            if t is OPEN:
                pos[0] += 1
                block()
                continue
            pos[0] += 1
            if pos[0] >= len(toks):
                break
            v = toks[pos[0]]
            if v is OPEN:
                pos[0] += 1
                val = block()
            elif v is CLOSE:
                break
            else:
                val = v
                pos[0] += 1
            d.setdefault(t.lower(), val)
        return d

    return block()


def vdf_file(path):
    try:
        return vdf_parse(Path(path).read_text(encoding="utf-8", errors="replace"))
    except OSError:
        return {}


def dig(doc, *keys):
    for k in keys:
        if not isinstance(doc, dict):
            return None
        doc = doc.get(k.lower())
    return doc


def ci_child(folder, name):
    """folder/name whatever its case on disk, or None."""
    if folder is None:
        return None
    folder = Path(folder)
    p = folder / name
    if os.path.lexists(str(p)):
        return p
    try:
        for n in os.listdir(str(folder)):
            if n.lower() == name.lower():
                return folder / n
    except OSError:
        pass
    return None


def steam_roots():
    home = Path.home()
    cands = []
    if os.environ.get("ODG_STEAM_DIR"):
        cands.append(Path(os.environ["ODG_STEAM_DIR"]))
    cands += [
        home / ".steam" / "steam",
        home / ".steam" / "root",
        home / ".local" / "share" / "Steam",
        home / ".var" / "app" / "com.valvesoftware.Steam" / ".local" / "share" / "Steam",
        home / ".var" / "app" / "com.valvesoftware.Steam" / ".steam" / "steam",
        home / "snap" / "steam" / "common" / ".local" / "share" / "Steam",
        home / ".steam" / "debian-installation",
    ]
    seen, out = set(), []
    for c in cands:
        if ci_child(c, "steamapps") is None:
            continue
        try:
            key = str(c.resolve())
        except OSError:
            continue
        if key not in seen:
            seen.add(key)
            out.append(c)
    return out


def library_dirs(root):
    libs = [Path(root)]
    sa = ci_child(root, "steamapps")
    for vdf in (sa / "libraryfolders.vdf" if sa else None, Path(root) / "config" / "libraryfolders.vdf"):
        if vdf is None or not vdf.is_file():
            continue
        lf = vdf_file(vdf).get("libraryfolders")
        if not isinstance(lf, dict):
            continue
        for k, v in lf.items():
            p = v.get("path") if isinstance(v, dict) else (v if k.isdigit() else None)
            if isinstance(p, str) and p:
                libs.append(Path(p))
    seen, out = set(), []
    for lib in libs:
        try:
            key = str(lib.resolve())
        except OSError:
            continue
        if key not in seen:
            seen.add(key)
            out.append(lib)
    return out


class Install:
    def __init__(self, game, steamapps=None, roots=None):
        self.game = Path(game)
        self.steamapps = steamapps
        self.roots = roots or []
        self.prefix = self._prefix()

    def _prefix(self):
        places = []
        if self.steamapps is not None:
            places.append(self.steamapps)
        for r in self.roots:
            places.append(ci_child(r, "steamapps"))
        for sa in places:
            pfx = ci_child(ci_child(ci_child(sa, "compatdata"), APPID), "pfx")
            if pfx is not None and pfx.is_dir():
                return pfx
        return None

    def manifest(self):
        if self.steamapps is not None:
            m = self.steamapps / "appmanifest_{}.acf".format(APPID)
            if m.is_file():
                return m
        for r in self.roots:
            for lib in library_dirs(r):
                m = ci_child(lib, "steamapps")
                m = m / "appmanifest_{}.acf".format(APPID) if m else None
                if m is not None and m.is_file():
                    return m
        return None

    def roaming(self):
        """%APPDATA% as the game sees it inside Proton."""
        if self.prefix is None:
            return None
        return self.prefix / "drive_c" / "users" / "steamuser" / "AppData" / "Roaming"


def is_game_dir(p):
    return p is not None and Path(p).is_dir() and ci_child(p, "OPERATOR.exe") is not None


def find_installs():
    found, seen = [], set()
    roots = steam_roots()
    for root in roots:
        for lib in library_dirs(root):
            sa = ci_child(lib, "steamapps")
            if sa is None:
                continue
            installdir = "OPERATOR"
            man = sa / "appmanifest_{}.acf".format(APPID)
            if man.is_file():
                d = dig(vdf_file(man), "AppState", "installdir")
                if isinstance(d, str) and d:
                    installdir = d
            game = ci_child(ci_child(sa, "common"), installdir)
            if not is_game_dir(game):
                continue
            try:
                key = str(game.resolve())
            except OSError:
                continue
            if key not in seen:
                seen.add(key)
                found.append(Install(game, sa, roots))
    return found


def resolve_install(opts, cfg):
    given = opts.get("game")
    remembered = cfg.get("game")
    if not given and remembered and is_game_dir(Path(remembered)):
        given = remembered
    if given:
        game = Path(os.path.expanduser(given))
        if not is_game_dir(game):
            raise Problem("{} is not the OPERATOR folder: OPERATOR.exe is not in it.".format(game))
        game = game.resolve()
        sa = game.parent.parent if game.parent.name.lower() == "common" else None
        return Install(game, sa, steam_roots())
    found = find_installs()
    if not found:
        raise Problem(
            "OPERATOR was not found. Install it in Steam and start it once, then run this again.\n"
            "If it is installed somewhere unusual, give the folder with --game:\n"
            "  python3 odg-linux.py --game \"/path/to/steamapps/common/OPERATOR\"\n"
            "In Steam: right-click OPERATOR > Manage > Browse local files shows that folder.")
    if len(found) > 1:
        say("OPERATOR is installed in more than one place. Using the first:")
        for f in found:
            say("  " + str(f.game))
        say("Pick another with --game.")
    return found[0]


def game_running():
    """Is OPERATOR running? Under Proton the game's processes carry OPERATOR.exe
    in their command line."""
    if depot_test_mode() and os.environ.get("ODG_TEST_GAME_RUNNING"):
        # tests only: "1" is running; a full path is running while that file is there
        flag = os.environ["ODG_TEST_GAME_RUNNING"]
        if flag == "1":
            return True
        if os.path.isabs(flag):
            return os.path.exists(flag)
    proc = Path("/proc")
    if not proc.is_dir():
        return False
    me = str(os.getpid())
    for d in proc.iterdir():
        if not d.name.isdigit() or d.name == me:
            continue
        try:
            cmd = (d / "cmdline").read_bytes()
        except OSError:
            continue
        if b"operator.exe" in cmd.lower():
            return True
    return False


# Steam accounts (steam.rs). Only the number and the display name are read.
STEAM_BASE = 76561197960265728


def valid_steam_id(s):
    s = str(s or "")
    return len(s) == 17 and s.startswith("7656119") and s.isdigit()


def operator_owner(inst):
    """The Steam account the installed copy belongs to (appmanifest LastOwner)."""
    m = inst.manifest()
    if m is None:
        return None
    owner = dig(vdf_file(m), "AppState", "LastOwner")
    return owner if valid_steam_id(owner) else None


def steam_users(roots):
    users = {}
    for r in roots:
        doc = vdf_file(Path(r) / "config" / "loginusers.vdf")
        for sid, u in (doc.get("users") or {}).items():
            if isinstance(u, dict) and valid_steam_id(sid):
                users[sid] = {
                    "name": u.get("personaname", "") or "",
                    "recent": str(u.get("mostrecent", "0")).strip() == "1",
                    "ts": int(u.get("timestamp", "0") or 0) if str(u.get("timestamp", "0")).isdigit() else 0,
                }
    return users


def steam_pc_account(roots):
    """The Steam account signed in on this PC now, else the last one to be."""
    users = steam_users(roots)
    home = Path.home()
    regs = [home / ".steam" / "registry.vdf",
            home / ".var" / "app" / "com.valvesoftware.Steam" / ".steam" / "registry.vdf"]
    regs += [Path(r) / "registry.vdf" for r in roots]
    for reg in regs:
        active = dig(vdf_file(reg), "Registry", "HKCU", "Software", "Valve", "Steam", "ActiveProcess", "ActiveUser")
        try:
            n = int(str(active or "0"), 0)
        except ValueError:
            n = 0
        if n:
            sid = str(STEAM_BASE + n)
            if valid_steam_id(sid):
                return {"id": sid, "name": users.get(sid, {}).get("name", "")}
    if users:
        sid = max(users, key=lambda k: (users[k]["recent"], users[k]["ts"]))
        return {"id": sid, "name": users[sid]["name"]}
    return None


# ---------------------------------------------------------------------------
# config and account
# ---------------------------------------------------------------------------

def config_dir():
    if os.environ.get("ODG_CONFIG_DIR"):
        return Path(os.environ["ODG_CONFIG_DIR"])
    base = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config")
    return Path(base) / "odg"


def load_json(path):
    try:
        with open(str(path), encoding="utf-8") as f:
            doc = json.load(f)
        return doc if isinstance(doc, dict) else {}
    except (OSError, ValueError):
        return {}


def load_config():
    return load_json(config_dir() / "config.json")


def save_config(cfg):
    write_atomic(config_dir() / "config.json", json.dumps(cfg, indent=2) + "\n")


TIERS = ["free", "release", "beta", "alpha", "bug-hunter", "dev-partner", "developer"]
TIER_STAMP = {"free": "FREE", "release": "RELEASE", "beta": "BETA", "alpha": "ALPHA",
              "bug-hunter": "BUG HUNTER", "dev-partner": "DEV PARTNER", "developer": "DEVELOPER"}


def tier_of(s):
    t = str(s or "").strip().lower().replace(" ", "-").replace("_", "-")
    return {"bughunter": "bug-hunter", "devpartner": "dev-partner", "dev": "developer"}.get(
        t, t if t in TIERS else "free")


def status_of(s):
    t = str(s or "").strip().lower()
    if t == "active":
        return "active"
    if t in ("cancelled", "canceled", "revoked", "expired"):
        return "cancelled"
    if t == "none":
        return "none"
    return "unknown"


def blank_account():
    return {"key": "", "name": "", "tier": "free", "status": "none", "checked": 0, "message": "",
            "dev_repo": False, "dev_state": "none", "github": "", "steam_id": "", "steam_name": "",
            "patreon_linked": False, "owns": [], "gates": [], "ticket": "", "ticket_sig": ""}


def words(doc, key):
    """A list of words ("owns", "gates"), as access.rs words(); anything else is an empty list."""
    v = doc.get(key) if isinstance(doc, dict) else None
    if not isinstance(v, list):
        return []
    return [s.strip().lower() for s in v if isinstance(s, str) and s.strip()]


def ticket_expires(line):
    """When a signed ticket runs out, unix seconds; 0 when it does not say."""
    for p in str(line or "").split("|"):
        if p.startswith("expires="):
            try:
                return int(p[len("expires="):].strip())
            except ValueError:
                return 0
    return 0


def keep_ticket(doc, a, cached, now):
    """access.rs keep_ticket: an answer WITHOUT a ticket is the server failing to sign, never a
    decision, so the last ticket stays while it lasts. A ticket the server did send always wins."""
    if "ticket" not in doc and not a["ticket"] and cached.get("ticket") and ticket_expires(cached["ticket"]) > now:
        a = dict(a)
        a["ticket"] = cached["ticket"]
        a["ticket_sig"] = str(cached.get("ticket_sig") or "")
    return a


def clean_account(doc):
    a = blank_account()
    if not isinstance(doc, dict):
        return a
    a["key"] = str(doc.get("key") or "").strip()
    a["name"] = str(doc.get("name") or "")
    a["tier"] = tier_of(doc.get("tier"))
    a["status"] = status_of(doc.get("status")) if "status" in doc else "none"
    a["checked"] = doc.get("checked") if isinstance(doc.get("checked"), int) else 0
    a["message"] = str(doc.get("message") or "")
    a["dev_repo"] = bool(doc.get("dev_repo"))
    a["dev_state"] = str(doc.get("dev_state") or "none")
    a["github"] = str(doc.get("github") or "")
    a["steam_id"] = str(doc.get("steam_id") or "")
    a["steam_name"] = str(doc.get("steam_name") or "")
    a["patreon_linked"] = bool(doc.get("patreon_linked"))
    a["owns"] = words(doc, "owns")
    a["gates"] = words(doc, "gates")
    a["ticket"] = str(doc.get("ticket") or "")
    a["ticket_sig"] = str(doc.get("ticket_sig") or "")
    return a


def from_answer(doc, key, fallback_name, now):
    steam = doc.get("steam") if isinstance(doc.get("steam"), dict) else {}
    name = str(doc.get("name") or "") or fallback_name
    return {"key": key or "", "name": name, "tier": tier_of(doc.get("tier")),
            "status": status_of(doc.get("status")), "checked": now, "message": str(doc.get("message") or ""),
            "dev_repo": bool(doc.get("dev_repo")), "dev_state": str(doc.get("dev_state") or "none"),
            "github": str(doc.get("github") or ""), "steam_id": str(steam.get("id") or ""),
            "steam_name": str(steam.get("name") or ""), "patreon_linked": bool(doc.get("patreon")),
            # what the ODG Store says this account owns, and the server's signed line saying so: the game
            # believes only the ticket, so without these nothing bought ever opened on Linux (10-28)
            "owns": words(doc, "owns"), "gates": words(doc, "gates"),
            "ticket": str(doc.get("ticket") or ""), "ticket_sig": str(doc.get("ticket_sig") or "")}


def prefix_account_path(inst):
    roaming = inst.roaming() if inst is not None else None
    return roaming / "OperatorMods" / "account.json" if roaming is not None else None


def load_account(inst=None):
    path = config_dir() / "account.json"
    if path.is_file():
        return clean_account(load_json(path))
    # A sign-in made by the Windows launcher, carried over in the prefix.
    pa = prefix_account_path(inst)
    if pa is not None and pa.is_file():
        return clean_account(load_json(pa))
    return blank_account()


def save_account(a, inst=None):
    """The game reads this file too: NewSlots, the Lounge and the Watch gates take
    tier, status and steam_id from %APPDATA%\\OperatorMods\\account.json, which
    under Proton is inside the prefix."""
    text = json.dumps(a, indent=2)
    write_atomic(config_dir() / "account.json", text)
    pa = prefix_account_path(inst)
    if pa is not None:
        try:
            write_atomic(pa, text)
        except OSError as e:
            say("Could not write the sign-in into the game's Proton folder: {}".format(e))


def is_member(a):
    return bool(a["key"]) and a["status"] == "active" and a["tier"] != "free"


def stamp_of(a):
    if a["status"] == "active":
        return TIER_STAMP.get(a["tier"], "FREE")
    return "FREE"


def refresh_account(a, inst):
    """Ask the server about this key (access.rs refresh). Unreachable is never a
    cancellation; an active answer is honoured for GRACE_DAYS without a check."""
    if not a["key"]:
        a = dict(a)
        a["status"] = "none"
        return a
    owner = operator_owner(inst) if inst is not None else None
    extra = {"X-ODG-Steam-Owner": owner} if owner else {}
    now = now_unix()
    try:
        doc = account_call("/api/launcher/status", key=a["key"], extra=extra, attempts=2)
        if not str(doc.get("status") or "").strip():
            raise NoAnswer(str(doc.get("error") or "the account server gave no status"))
        return keep_ticket(doc, from_answer(doc, a["key"], a["name"], now), a, now)
    except NoAnswer as e:
        b = dict(a)
        b["message"] = str(e)
        if b["status"] == "active" and now - int(b["checked"] or 0) > GRACE_DAYS * 86400:
            b["status"] = "unknown"
        return b


def record_consent(key):
    try:
        account_call("/api/launcher/consent", key=key, body={"version": TERMS_VERSION})
    except NoAnswer:
        pass


# ---------------------------------------------------------------------------
# the mod loader (melonsetup.rs)
# ---------------------------------------------------------------------------

def loader_state(disk):
    """('ready'|'absent'|'incomplete'|'held', missing)."""
    missing = [r for r in LOADER_FILES if not disk.isfile(r)]
    if missing == ["version.dll"] and disk.isfile(HELD):
        return "held", []
    if not missing:
        return "ready", []
    if len(missing) == len(LOADER_FILES):
        return "absent", missing
    return "incomplete", missing


def is_melonloader(path):
    try:
        with open(str(path), "rb") as f:
            return b"MelonLoader" in f.read()
    except OSError:
        return False


# ---------------------------------------------------------------------------
# mods on and off (applier.rs)
# ---------------------------------------------------------------------------

def display_name(file):
    k = KNOWN.get(file.lower())
    if k:
        return k[0]
    stem = file[:-4] if file.lower().endswith(".dll") else file
    return stem[:-4] if stem.lower().endswith(".mod") else stem


def requirement(file):
    k = KNOWN.get(file.lower())
    return k[1] if k else None


def mod_files(disk):
    """[(file, on_name, off_name)] for every DLL in Mods, either state."""
    mods = disk.find("Mods")
    if mods is None or not mods.is_dir():
        return []
    found = {}
    for n in sorted(os.listdir(str(mods))):
        low = n.lower()
        if low.endswith(".dll" + OFF):
            canon = n[:-len(OFF)]
            found.setdefault(canon.lower(), [canon, None, None])[2] = n
        elif low.endswith(".dll"):
            found.setdefault(low, [n, None, None])[1] = n
    out = [tuple(v) for v in found.values()]
    out.sort(key=lambda m: (requirement(m[0]) is None, KNOWN.get(m[0].lower()) is None, display_name(m[0]).lower()))
    return out


def switched_off(disk):
    return {f.lower() for f, on, off in mod_files(disk) if off and not on}


def set_mod(disk, file, on_name, off_name, enable):
    mods = disk.find("Mods")
    if enable:
        if on_name:
            return False
        os.replace(str(mods / off_name), str(mods / file))
    else:
        if not on_name:
            return False
        os.replace(str(mods / on_name), str(mods / (on_name + OFF)))
    disk.forget(mods)
    return True


def reapply_off(disk, off_before):
    """An update unpacks a fresh X.dll beside a switched-off X.dll.odg-off. Mods
    that were off stay off, on the new copy."""
    for file, on, off in mod_files(disk):
        if file.lower() in off_before and on and not requirement(file):
            set_mod(disk, file, on, off, False)


def required_on(disk):
    """Load-bearing mods are held on, as the launcher holds them."""
    mods = disk.find("Mods")
    for file, on, off in mod_files(disk):
        if not requirement(file) or not off:
            continue
        if on:
            remove_quietly(mods / off)
        else:
            os.replace(str(mods / off), str(mods / file))
            say("{} is required and was switched back on.".format(display_name(file)))
        disk.forget(mods)


# ---------------------------------------------------------------------------
# leftovers (stale.rs) and deletes
# ---------------------------------------------------------------------------

def allowlisted(name_lower):
    return (name_lower in ALLOWLISTED or name_lower.endswith(".log") or name_lower.endswith(OFF)
            or name_lower.endswith(".tmp") or ".bak" in name_lower)


def walk_files(disk, folder, out):
    try:
        entries = list(os.scandir(str(folder)))
    except OSError:
        return
    for e in entries:
        try:
            if e.is_symlink():
                continue
            if e.is_dir(follow_symlinks=False):
                walk_files(disk, e.path, out)
            elif e.is_file(follow_symlinks=False):
                out.append((disk.rel(e.path), e.stat(follow_symlinks=False).st_size))
        except OSError:
            continue


def dir_bytes(folder):
    total = 0
    for dirpath, _dirs, files in os.walk(str(folder)):
        for f in files:
            try:
                total += os.lstat(os.path.join(dirpath, f)).st_size
            except OSError:
                pass
    return total


def stale_scan(disk):
    """Files earlier packs left that the current one does not ship, inside the
    folders the pack owns, plus whatever the obsolete list names."""
    out, seen = [], set()
    mods = disk.find("Mods")
    if mods is None or not mods.is_dir():
        return out
    # an add-on's files are its owner's: no pack list judges them, not even an
    # obsolete list naming a file a pack used to ship (stale.rs)
    addon = addon_files(disk)
    shipped = read_list(disk.read(SHIPPED_LIST))
    if shipped:
        shipped_lower = {s.lower() for s in shipped}
        owned, owned_assets = set(), set()
        for s in shipped_lower:
            if not s.startswith("mods/"):
                continue
            rest = s[5:]
            if "/" not in rest:
                continue
            top, more = rest.split("/", 1)
            owned.add(top)
            if top == "maploader_assets" and "/" in more:
                owned_assets.add(more.split("/", 1)[0])
        owned_assets.discard("custom")
        files = []
        for e in os.scandir(str(mods)):
            try:
                if e.is_symlink() or not e.is_dir(follow_symlinks=False):
                    continue
            except OSError:
                continue
            if e.name.lower() in owned:
                walk_files(disk, e.path, files)
        for rel, size in files:
            lower = rel.lower()
            if lower in shipped_lower or lower in addon or allowlisted(lower.rsplit("/", 1)[-1]):
                continue
            if lower.startswith("mods/maploader_assets/"):
                rest = lower[len("mods/maploader_assets/"):]
                if "/" in rest:
                    sub = rest.split("/", 1)[0]
                    if sub == "custom" or sub not in owned_assets:
                        continue
            if lower not in seen:
                seen.add(lower)
                out.append((rel, size, False))
    for rel in read_list(disk.read(OBSOLETE_LIST)):
        is_dir = rel.endswith("/")
        clean = rel.rstrip("/")
        if not safe_rel(clean):
            continue
        p = disk.find(clean)
        if p is None:
            continue
        try:
            st = os.lstat(str(p))
        except OSError:
            continue
        if stat.S_ISLNK(st.st_mode) or is_dir != stat.S_ISDIR(st.st_mode):
            continue
        lower = clean.lower()
        # Zombies and Cinecam are free and in both packs: an obsolete line naming them is a
        # mistake in the list (the 3.1-era block that emptied v3.2.1), never an order
        if free_mod_path(lower):
            continue
        # a folder an add-on put files in is the add-on's too
        if lower in addon or (is_dir and any(f.startswith(lower + "/") for f in addon)):
            continue
        if lower not in seen:
            seen.add(lower)
            out.append((disk.rel(p), dir_bytes(p) if is_dir else st.st_size, is_dir))
    out.sort(key=lambda e: e[0].lower())
    return out


def through_link(path, stop):
    d = Path(path)
    stop = Path(stop)
    while d != stop and stop in d.parents:
        if os.path.islink(str(d)):
            return True
        d = d.parent
    return False


def delete_entries(disk, entries):
    """Remove exactly these, each re-checked: under Mods, no '..', not a link."""
    removed, failed = [], []
    mods = disk.find("Mods")
    for rel, _size, _is_dir in entries:
        r = rel.rstrip("/")
        if mods is None or not safe_rel(r):
            failed.append((rel, "refused: outside Mods"))
            continue
        # the free mods, whoever made the list (see FREE_MODS): kept, neither removed nor failed
        if free_mod_path(r):
            continue
        p = disk.find(r)
        if p is None:
            removed.append(rel)
            continue
        if mods not in p.parents:
            failed.append((rel, "refused: outside Mods"))
            continue
        try:
            st = os.lstat(str(p))
        except OSError:
            removed.append(rel)
            continue
        if stat.S_ISLNK(st.st_mode) or through_link(p.parent, mods):
            failed.append((rel, "refused: a link"))
            continue
        try:
            if stat.S_ISDIR(st.st_mode):
                shutil.rmtree(str(p))
            else:
                os.remove(str(p))
            removed.append(rel)
        except OSError as e:
            failed.append((rel, str(e)))
        disk.forget(p.parent)
    return removed, failed


# ---------------------------------------------------------------------------
# install and update (update.rs)
# ---------------------------------------------------------------------------

def asset_named(assets, name):
    for a in assets:
        if str(a.get("name") or "").lower() == name.lower():
            return a
    return None


def asset_size(a):
    s = a.get("size")
    return s if isinstance(s, int) and s > 0 else 0


class Updater:
    def __init__(self, inst):
        self.inst = inst
        self.disk = Disk(inst.game)
        self.failed = []
        self.changed = False
        self.src = "public"
        self.key = None
        self.tag = ""
        self.hold = None
        self.running = False
        self.depot = Depot(self)

    def fail(self, msg):
        self.failed.append(msg)
        say(msg)

    def staging(self):
        try:
            cache = self.disk.make_dir(STAGING)
        except Problem:
            import tempfile
            return Path(tempfile.gettempdir())
        day = 24 * 3600
        try:
            for e in os.scandir(str(cache)):
                n = e.name.lower()
                if n.startswith("odg_") and n.endswith(".zip") and time.time() - e.stat().st_mtime > day:
                    remove_quietly(e.path)
        except OSError:
            pass
        return cache

    def asset_url(self, a):
        if self.src == "broker":
            if not isinstance(a.get("id"), int):
                return None
            return "{}/api/launcher/dev/asset?id={}".format(server(), a["id"])
        u = a.get("browser_download_url")
        return u if isinstance(u, str) and u else None

    def fetch(self, a, dest, label):
        url = self.asset_url(a)
        if not url:
            raise Problem("the release gives no address for it")
        size = asset_size(a)
        check_space(Path(dest).parent, size)
        if self.src == "broker":
            # /dev/asset hears which tool asks, as /api/launcher/status and /dev/sign do
            download(url, dest, size, label, key=self.key,
                     extra={"X-ODG-Launcher": account_headers()["X-ODG-Launcher"]})
        else:
            download(url, dest, size, label)
        verify(dest, size, a.get("digest"))

    # ---- which release ----------------------------------------------------

    def listing(self, acct, public_doc):
        """The release this PC should be on, or None when a member's pack stays
        because the account server did not answer."""
        have = self.disk.read(STAMP).strip()
        if is_member(acct):
            self.src, self.key = "broker", acct["key"]
            say("Checking your channel")
            doc = None
            try:
                doc = http_json(server() + "/api/launcher/dev/release", key=self.key,
                                headers=account_headers(), timeout=30, attempts=2)
            except Refused as e:
                if e.status in (401, 403):
                    say("No channel build on this account, so this is the public release.")
                    self.src = "public"
            except NoAnswer:
                pass
            if self.src == "broker":
                tag = doc.get("tag_name") if isinstance(doc, dict) else None
                if not isinstance(tag, str):
                    if have:
                        say("The account server did not answer, so {} stays.".format(have))
                        return None
                    say("The account server could not be reached, so this is the public release.")
                    self.src = "public"
                elif not tag:
                    say("Nothing is pushed to your channel, so this is the public release.")
                    self.src = "public"
                else:
                    brief = str(doc.get("brief") or "").strip()
                    say("Your channel ({}) is on {}.{}".format(doc.get("channel") or "?", tag,
                                                              " " + brief if brief else ""))
                    return doc
        self.src, self.key = "public", None
        return public_doc if public_doc is not None else public_listing()

    # ---- the run -----------------------------------------------------------

    def run(self, acct, public_doc):
        disk = self.disk
        doc = self.listing(acct, public_doc)
        if doc is None:
            return
        self.tag = tag = str(doc.get("tag_name") or "")
        # 2.2: a store prerelease or the installer is never a build; what is installed stays
        if tag.startswith("depot-") or tag == "installer":
            self.fail("The release {} is not a build, so the pack stays as it is.".format(tag))
            return
        assets = [a for a in (doc.get("assets") or []) if isinstance(a, dict)]
        exact = self.src == "broker" and bool(doc.get("exact"))
        self.hold = hold = "held by .odg-dev-hold" if disk.find(DEV_HOLD) is not None else None

        # ---- a depot release: one file list, and only what changed comes down ----
        if asset_named(assets, MANIFEST_ASSET) is not None:
            if hold:
                say("Pack {} kept ({}, the release is {}).".format(disk.read(STAMP).strip() or "(none)", hold, tag))
            else:
                self.depot.run(doc, assets, exact, acct)
            # the add-ons are not in the pack's file list; they follow it as they follow mods.zip
            self.sync_addons(acct)
            return

        self.running = running = hold is None and game_running()
        staging = self.staging()

        # ---- mods.zip ----
        have = disk.read(STAMP).strip()
        forgotten = disk.read(FORGOTTEN).strip()
        kept_newer = False
        pack = asset_named(assets, "mods.zip")
        if pack is None:
            self.fail("This release carries no mod pack.")
        else:
            ahead = (not exact) and lineage(have) == lineage(tag) and compare(have, tag) == "ahead"
            if hold or ahead:
                kept_newer = ahead
                say("Pack {} kept ({}, the release is {}).".format(have, hold or "ahead of the release", tag))
            elif have == tag and tag:
                if not running:
                    self.finish_leftovers(tag)
                say("The pack is up to date ({}).".format(tag))
            elif running:
                self.fail("OPERATOR is running, so the update to {} waits until it is closed.".format(tag))
            else:
                self.install_pack(pack, tag, have, forgotten, staging)

        # ---- mods_2.zip, mods_3.zip ...: the rest of the pack ----
        # Only over THIS release's pack: kept-newer, held, running or a failed
        # mods.zip all leave the parts alone, so they never land on another build.
        if hold is None and not running and tag and disk.read(STAMP).strip() == tag:
            self.install_parts(assets, tag, staging)

        # ---- map data, each map once ----
        maps = [a for a in assets
                if str(a.get("name") or "").lower().startswith("mapdata_")
                and str(a.get("name") or "").lower().endswith(".zip")]
        shipped_maps = []
        history = None
        for idx, a in enumerate(maps, 1):
            name = str(a.get("name"))
            lower = name.lower()
            stem = lower[len("mapdata_"):-len(".zip")]
            shipped_maps.append(stem)
            lines = [l.strip() for l in disk.read(MAP_STAMP).splitlines() if l.strip()]
            on_disk = self.map_on_disk(stem)
            digest = str(a.get("digest") or "")
            digest = digest if digest.startswith("sha256:") else ""
            updated_at = str(a.get("updated_at") or "")
            want = digest or updated_at or str(a.get("size") or "")
            stamped = None
            for l in lines:
                n, _, marker = l.partition("\t")
                if n.strip().lower() == lower:
                    stamped = marker.strip()
                    break
            if stamped is not None:
                current = bool(stamped) and (stamped == want or (updated_at and stamped == updated_at))
                if not current and on_disk and digest and stamped and not stamped.startswith("sha256:"):
                    if history is None:
                        history = self.digest_history()
                    current = any(n == lower and at == stamped and d == digest for n, at, d in history)
                if current and on_disk:
                    if stamped != want and hold is None:
                        self.record_map(lines, name, want)
                    continue
            if hold:
                say("Map data {} kept ({}).".format(idx, hold))
                continue
            if running:
                say("Map data {} waits until OPERATOR is closed.".format(idx))
                continue
            say("Map data {} of {}: {}, one time only".format(idx, len(maps), mb(asset_size(a))))
            tmp = staging / "odg_mapdata_{}.zip".format(os.getpid())
            try:
                self.fetch(a, tmp, "map data")
                extract(tmp, disk, "map data")
            except Problem as e:
                self.fail("Map data {} skipped: {}".format(idx, e))
                continue
            finally:
                remove_quietly(tmp)
            disk.forget()
            lines = [l.strip() for l in disk.read(MAP_STAMP).splitlines() if l.strip()]
            self.record_map(lines, name, want)
            self.changed = True

        # ---- the mod loader ----
        state, _missing = loader_state(disk)
        if hold is None and not running and state in ("absent", "incomplete"):
            a = asset_named(assets, "melonloader.zip")
            if a is None:
                self.fail("The mod loader is missing and this release does not carry one.")
            else:
                say("Installing the mod loader ({})".format(mb(asset_size(a))))
                tmp = staging / "odg_melonloader_{}.zip".format(os.getpid())
                try:
                    self.fetch(a, tmp, "mod loader")
                    extract(tmp, disk, "the mod loader")
                    disk.forget()
                    state, missing = loader_state(disk)
                    if state != "ready":
                        raise Problem("still not complete after unpacking: " + ", ".join(missing))
                    say("Mod loader installed.")
                    self.changed = True
                except Problem as e:
                    self.fail("The mod loader could not be installed: {}".format(e))
                finally:
                    remove_quietly(tmp)

        # ---- add-ons: exactly what this account owns (ADDONS_3_0.md) ----
        # After the pack, so a pack switch that took back an older pack's copy of
        # an add-on's files is followed by the owner's own copy coming down.
        self.sync_addons(acct)

        # ---- maps the release no longer ships ----
        if hold:
            say("Maps kept ({}).".format(hold))
        elif not kept_newer and not running:
            self.prune_maps(shipped_maps)

    # ---- add-ons -----------------------------------------------------------

    def addon_state(self):
        """Which add-ons are on disk, and the digest each came as."""
        try:
            doc = json.loads(self.disk.read(ADDON_DIR + "/installed.json") or "{}")
        except ValueError:
            return {}
        if not isinstance(doc, dict):
            return {}
        return {k: v for k, v in doc.items() if addon_word(k) and isinstance(v, str)}

    def addon_list(self, aid):
        return [norm(l) for l in self.disk.read("{}/{}.files".format(ADDON_DIR, aid)).splitlines() if norm(l)]

    def addon_present(self, aid):
        """Every file the add-on put down is there (a DLL may be switched off)."""
        lst = self.addon_list(aid)
        if not lst:
            return False
        for rel in lst:
            if not (self.disk.isfile(rel) or (rel.lower().endswith(".dll") and self.disk.isfile(rel + OFF))):
                return False
        return True

    def remove_addon(self, aid):
        """Takes an add-on back off the disk: every file it put down, but never
        one the free pack ships and never anything not truly inside Mods (a
        link cannot carry the delete out). Folders it leaves empty go too."""
        disk = self.disk
        mods = disk.find("Mods")
        real_mods = os.path.realpath(str(mods)) if mods is not None else None
        shipped = {s.lower() for s in self.shipped()}
        gone, emptied = 0, set()
        for rel in self.addon_list(aid):
            if not addon_entry_ok(rel) or rel.endswith("/") or rel.lower() in shipped:
                continue
            for n in [rel] + ([rel + OFF] if rel.lower().endswith(".dll") else []):
                p = disk.find(n)
                if p is None or real_mods is None or not os.path.realpath(str(p)).startswith(real_mods + os.sep):
                    continue
                try:
                    st = os.lstat(str(p))
                except OSError:
                    continue
                if stat.S_ISREG(st.st_mode):
                    remove_quietly(p)
                    gone += 1
                    emptied.add(p.parent)
        # the zip's own folder entries too: an empty one holds no file to lead the
        # clean-up to it (10-28: Zombies' shop/presets/F2000 kept the whole folder)
        for rel in [norm(l) for l in disk.read("{}/{}.dirs".format(ADDON_DIR, aid)).splitlines() if norm(l)]:
            if not addon_entry_ok(rel):
                continue
            p = disk.find(rel.rstrip("/"))
            if p is None or os.path.islink(str(p)) or not p.is_dir():
                continue
            emptied.add(p)
        ups = set()
        for d in emptied:
            while mods is not None and d != mods and mods in d.parents:
                ups.add(d)
                d = d.parent
        # deepest first: a folder only goes once it is empty
        for d in sorted(ups, key=lambda q: len(q.parts), reverse=True):
            try:
                os.rmdir(str(d))
            except OSError:
                pass
        disk.forget()
        for ext in ("files", "dirs", "receipt"):
            disk.remove("{}/{}.{}".format(ADDON_DIR, aid, ext))
        return gone

    def sync_addons(self, acct):
        """Brings the add-ons on disk in line with what the account owns:
        downloads what is missing or changed (checked against its digest) and
        takes back what the account no longer owns. Nothing is removed unless
        the server answered with what the account owns. Never on a held folder."""
        key = (acct or {}).get("key") or ""
        if not key:
            return
        if self.hold:
            say("Add-ons kept ({}).".format(self.hold))
            return
        try:
            doc = http_json(server() + "/api/launcher/addons", key=key, headers=account_headers(), timeout=30, attempts=2)
        except (NoAnswer, Refused) as e:
            say("Add-ons: no answer from the account server, so nothing changes ({}).".format(e))
            return
        if not isinstance(doc, dict) or not isinstance(doc.get("addons"), list):
            say("Add-ons: the account server's answer had no list, so nothing changes.")
            return
        disk = self.disk
        offers, owned = parse_offers(doc), parse_owned(doc)
        state = self.addon_state()
        dirty = False
        staging = None
        for o in offers:
            aid, title = o["id"], o["title"] or o["id"]
            # the receipt is renewed on every answer, so it names who holds it now
            if o["receipt"]:
                disk.write("{}/{}.receipt".format(ADDON_DIR, aid), "{}\n{}\n".format(o["receipt"], o["receipt_sig"]))
            if o["digest"] and state.get(aid) == o["digest"] and self.addon_present(aid):
                continue
            if self.running:
                self.fail("OPERATOR is running, so {} waits until it is closed.".format(title))
                continue
            staging = staging or self.staging()
            tmp = staging / "odg_addon_{}_{}.zip".format(aid, os.getpid())
            say("Downloading {} ({})".format(title, mb(o["size"])))
            try:
                check_space(staging, o["size"])
                download("{}/api/launcher/addons/asset?id={}".format(server(), aid), tmp, o["size"], title, key=key)
                verify(tmp, o["size"], o["digest"])
                names = zip_names(tmp)
                if not names or any(not addon_entry_ok(n) for n in names):
                    raise Problem("it holds a file outside Mods, so it was not installed")
                # a newer copy of one already here: what the old one put down goes first
                if aid in state:
                    self.remove_addon(aid)
                extract(tmp, disk, title)
            except Problem as e:
                self.fail("{} was not installed: {}".format(title, e))
                continue
            finally:
                remove_quietly(tmp)
            disk.forget()
            disk.write("{}/{}.files".format(ADDON_DIR, aid), "\n".join(n for n in names if not n.endswith("/")) + "\n")
            # its folders, so a refund takes the empty ones too (never Mods itself)
            disk.write("{}/{}.dirs".format(ADDON_DIR, aid), "\n".join(n for n in names if n.endswith("/") and n.lower() != "mods/") + "\n")
            state[aid] = o["digest"]
            dirty = self.changed = True
            disk.write(ADDON_DIR + "/installed.json", json.dumps(state, sort_keys=True))
            say("{} installed.".format(title))
        # what the account no longer owns (a refund), on this real answer only
        for aid in addons_to_remove(state, owned):
            if self.running:
                break
            n = self.remove_addon(aid)
            state.pop(aid, None)
            dirty = self.changed = True
            say("{} taken off: the account no longer owns it ({} file{}).".format(aid, n, "" if n == 1 else "s"))
        if dirty:
            disk.write(ADDON_DIR + "/installed.json", json.dumps(state, sort_keys=True))

    # ---- the pack ----------------------------------------------------------

    def install_pack(self, a, tag, have, forgotten, staging):
        disk = self.disk
        was = have or forgotten
        leaving = bool(was) and (lineage(was) != lineage(tag) or compare(was, tag) == "ahead")
        before = self.remember_old_pack(tag, was, leaving)
        off_before = switched_off(disk)
        say("Downloading the pack {} ({})".format(tag, mb(asset_size(a))))
        tmp = staging / "odg_mods_{}.zip".format(os.getpid())
        try:
            self.fetch(a, tmp, "pack")
            # 6.19: the depot's record of this folder stops being true once the zip is
            # unpacked; set aside, the next depot release adopts the folder again
            depot_mark_legacy(self.inst.game)
            # A depot switch still pending would hold every later run (STOPPED) and could
            # one day lay the depot build back over this pack: it goes, as the launcher's
            # leave_depot drops it (20.4).
            if depot_switch_pending(self.inst.game):
                JournalFile(depot_dir(self.inst.game)).clear()
            extract(tmp, disk, "the pack")
        except Problem as e:
            self.fail("Pack update skipped: {}".format(e))
            return
        finally:
            remove_quietly(tmp)
        disk.forget()
        if leaving:
            self.take_back_leftovers(before, was, tag)
        else:
            disk.remove(PREV_LIST)
        reapply_off(disk, off_before)
        # Written last: until it is, the next run installs this build again.
        disk.write(STAMP, tag)
        disk.remove(FORGOTTEN)
        self.changed = True
        say("Pack {} installed.".format(tag))

    def install_parts(self, assets, tag, staging):
        """GitHub refuses any release file over 2 GiB, so a big pack ships as
        mods.zip plus mods_2.zip, mods_3.zip ..., each unpacked over the game
        folder like mods.zip. NewSlots' content pack rides in them cut into
        content.odgpak.001, .002 ... (update.rs).

        Their own stamp, one line per part, name<TAB>digest: a tool from before
        parts stamped the tag and never fetched them, and this one still finds
        them unrecorded. An unchanged part (same digest) is never pulled again."""
        disk = self.disk
        parts = []
        for a in assets:
            name = str(a.get("name") or "")
            m = re.fullmatch(r"mods_(\d+)\.zip", name.lower())
            if m:
                parts.append((int(m.group(1)), a, name))
        parts.sort(key=lambda p: p[0])
        have = [l.strip() for l in disk.read(PARTS_STAMP).splitlines() if l.strip()]
        # ASK THE DISK, NOT THE RECEIPT. The receipt outlived a wipe of the pack
        # on a member's PC (10-15): the reinstall matched every part against it,
        # skipped them all, and NewSlots found no content. The parts carry only
        # the pieces of the cut files, and the shipped list names every piece,
        # so one missing piece voids the receipt.
        disk.forget()
        missing = self.missing_pieces()
        if have and missing:
            say("Pack parts are missing ({}), so they are fetched again.".format(missing[0]))
            have = []
        for _n, a, name in parts:
            digest = str(a.get("digest") or "").strip()
            size = a.get("size")
            marker = digest or "{}/{}".format(tag, "" if size is None else size)
            line = "{}\t{}".format(name, marker)
            if line in have:
                continue
            if not self.asset_url(a):
                continue
            say("Downloading the pack {}, {} ({})".format(tag, name, mb(asset_size(a))))
            tmp = staging / "odg_part_{}.zip".format(os.getpid())
            try:
                self.fetch(a, tmp, "pack")
                extract(tmp, disk, "the pack")
            except Problem as e:
                self.fail("Pack update skipped ({}): {}".format(name, e))
                break
            finally:
                remove_quietly(tmp)
            disk.forget()
            have = [l for l in have if l.partition("\t")[0].lower() != name.lower()]
            have.append(line)
            disk.write(PARTS_STAMP, "\n".join(have) + "\n")
            self.changed = True
            say("Pack {}, {} installed.".format(tag, name))

    def missing_pieces(self):
        """The pieces of the cut pack files (content.odgpak.001, .002 ...) the
        shipped list names and the disk does not have. No shipped list (an old
        or a local pack) is nothing missing: the receipt is all there is then."""
        out = []
        for rel in self.shipped():
            ext = rel.rsplit(".", 1)[-1] if "." in rel else ""
            if len(ext) != 3 or not ext.isdigit() or not ext.isascii():
                continue
            if ".." in rel.split("/"):
                continue
            if not self.disk.isfile(rel):
                out.append(rel)
        return out

    def shipped(self):
        return read_list(self.disk.read(SHIPPED_LIST))

    def read_prev(self):
        text = self.disk.read(PREV_LIST)
        lines = text.splitlines()
        if not lines or not lines[0].strip().startswith("# for "):
            return None
        head = lines[0].strip()[len("# for "):].split("\t")
        target = head[0].strip() if head else ""
        was = head[1].strip() if len(head) > 1 else ""
        leaving = len(head) > 2 and head[2].strip() == "1"
        return target, was, leaving, read_list("\n".join(lines[1:]))

    def remember_old_pack(self, target, was, leaving):
        now = self.shipped()
        prev = self.read_prev()
        if prev and prev[0] == target:
            return prev[3]
        if prev:
            lst = list(prev[3])
            low = {x.lower() for x in lst}
            for l in now:
                if l.lower() not in low:
                    lst.append(l)
                    low.add(l.lower())
        else:
            lst = now
        self.disk.write(PREV_LIST, "# for {}\t{}\t{}\n{}\n".format(target, was, 1 if leaving else 0, "\n".join(lst)))
        return lst

    def switch_leftovers(self, old, new_lower):
        out, seen = [], set()
        # an owner's add-on copy is never a pack's leftover
        addon = addon_files(self.disk)
        for rel in old:
            lower = rel.lower()
            # never the free mods (FREE_MODS): both packs carry them
            if (not lower.startswith("mods/") or "/maploader_assets/custom/" in lower or lower in new_lower
                    or lower in addon or free_mod_path(lower)):
                continue
            names = [rel] + ([rel + OFF] if lower.endswith(".dll") else [])
            for n in names:
                if n.lower() in seen:
                    continue
                seen.add(n.lower())
                p = self.disk.find(n)
                if p is None:
                    continue
                try:
                    st = os.lstat(str(p))
                except OSError:
                    continue
                if stat.S_ISREG(st.st_mode):
                    out.append((self.disk.rel(p), st.st_size, False))
        return out

    def take_back_leftovers(self, before, was, tag):
        new_lower = {s.lower() for s in self.shipped()}
        if not new_lower:
            self.disk.remove(PREV_LIST)
            return
        gone = self.switch_leftovers(before, new_lower)
        failed = []
        if gone:
            removed, failed = delete_entries(self.disk, gone)
            if lineage(was) != lineage(tag):
                which = "test" if lineage(was) == "dev" else "public"
            else:
                which = "newer"
            say("Removed {} file{} only the {} build had.".format(len(removed), "" if len(removed) == 1 else "s", which))
        if not failed:
            self.disk.remove(PREV_LIST)

    def finish_leftovers(self, tag):
        prev = self.read_prev()
        if prev is None:
            return
        target, was, leaving, lst = prev
        if target == tag and leaving:
            self.take_back_leftovers(lst, was, tag)
        else:
            self.disk.remove(PREV_LIST)

    # ---- maps --------------------------------------------------------------

    def map_on_disk(self, stem):
        root = self.disk.find(MAP_ROOT)
        if root is None:
            return False
        for actual in self.disk._names(root).get(stem, []):
            if self.disk.isfile("{}/{}/manifest.bin".format(MAP_ROOT, actual)):
                return True
        return False

    def record_map(self, lines, name, marker):
        kept = [l for l in lines if l.partition("\t")[0].strip().lower() != name.lower()]
        kept.append("{}\t{}".format(name, marker))
        self.disk.write(MAP_STAMP, "\n".join(kept) + "\n")

    def digest_history(self):
        """Every asset's (name, upload time, digest) in recent public releases, to
        recognise a map stamped before stamps carried digests."""
        if self.src == "broker":
            return []
        base = public_api()
        if not base.endswith("/latest"):
            return []
        try:
            doc = http_json(base[:-len("/latest")] + "?per_page=40", headers=GITHUB, attempts=2)
        except (NoAnswer, Refused):
            return []
        out = []
        for rel in doc if isinstance(doc, list) else []:
            for a in rel.get("assets") or [] if isinstance(rel, dict) else []:
                if not isinstance(a, dict):
                    continue
                n = str(a.get("name") or "").lower()
                at = str(a.get("updated_at") or "")
                d = str(a.get("digest") or "")
                if n and at and d.startswith("sha256:"):
                    out.append((n, at, d))
        return out

    def prune_maps(self, shipped):
        """Only a map this tool or the launcher installed, never the player's own,
        and never anything when the release listed no maps at all."""
        if not shipped:
            return
        disk = self.disk
        root = disk.find(MAP_ROOT)
        if root is None:
            return
        installed = set()
        for l in disk.read(MAP_STAMP).splitlines():
            n = l.split()[0].lower() if l.split() else ""
            if n.startswith("mapdata_") and n.endswith(".zip") and len(n) > len("mapdata_.zip"):
                installed.add(n[len("mapdata_"):-len(".zip")])
        removed = []
        for e in list(os.scandir(str(root))):
            try:
                if e.is_symlink() or not e.is_dir(follow_symlinks=False):
                    continue
            except OSError:
                continue
            low = e.name.lower()
            if low == "custom" or low not in installed or low in shipped:
                continue
            if ci_child(e.path, "manifest.bin") is None:
                continue
            try:
                shutil.rmtree(e.path)
                removed.append(low)
                say("Removed a map the release no longer ships ({}).".format(e.name))
            except OSError as err:
                self.fail("Could not remove {}: {}".format(e.name, err))
        disk.forget(root)
        if removed:
            lines = [l.strip() for l in disk.read(MAP_STAMP).splitlines() if l.strip()]
            keep = []
            for l in lines:
                n = l.split()[0].lower()
                stem = n[len("mapdata_"):] if n.startswith("mapdata_") else n
                stem = stem[:-len(".zip")] if stem.endswith(".zip") else stem
                if stem not in removed:
                    keep.append(l)
            disk.write(MAP_STAMP, "\n".join(keep) + "\n")

    # ---- tidy up -----------------------------------------------------------

    def tidy(self, ask_leftovers):
        disk = self.disk
        mods = disk.find("Mods")
        if mods is None:
            return
        for name in RETIRED_FILES:
            p = ci_child(mods, name)
            if p is not None and os.path.isfile(str(p)) and not os.path.islink(str(p)):
                remove_quietly(p)
                say("Removed a retired file ({}).".format(name))
        for name in RETIRED_DIRS:
            p = ci_child(mods, name)
            if p is not None and os.path.isdir(str(p)) and not os.path.islink(str(p)):
                shutil.rmtree(str(p), ignore_errors=True)
                say("Removed a retired folder ({}).".format(name))
        disk.forget()
        if disk.find(MAP_ROOT) is not None:
            if disk.find(CUSTOM) is None:
                disk.make_dir(CUSTOM)
            if disk.find(CUSTOM + "/README.txt") is None:
                disk.write(CUSTOM + "/README.txt", CUSTOM_README)
        required_on(disk)
        if ask_leftovers and self.hold is None and not self.running:
            stale = stale_scan(disk)
            if stale:
                total = sum(s for _r, s, _d in stale)
                say("")
                say("{} file{} an older pack left behind ({}), for example {}.".format(
                    len(stale), "" if len(stale) == 1 else "s", mb(total), stale[0][0][len("Mods/"):]))
                if OPTS.get("yes") or ask("Remove them?", default=False):
                    removed, failed = delete_entries(disk, stale)
                    say("Removed {}.{}".format(len(removed), " {} could not be removed.".format(len(failed)) if failed else ""))
                else:
                    say("Left as they are.")


def public_listing():
    try:
        doc = http_json(public_api(), headers=GITHUB)
    except Refused as e:
        raise Problem("GitHub refused the release listing (HTTP {}). Try again in a few minutes.".format(e.status))
    except NoAnswer as e:
        raise Problem("Could not reach GitHub: {}. Check your internet connection.".format(e))
    if not isinstance(doc, dict) or "tag_name" not in doc:
        raise Problem("GitHub's answer was not a release listing. Try again in a few minutes.")
    return doc


# ---------------------------------------------------------------------------
# the depot (SPEC.md section 11): releases that carry odg-manifest.json.gz
# ---------------------------------------------------------------------------
#
# WHY. A release used to carry the whole pack as zips, so a one-DLL update cost
# 2.1 GB. A depot release carries one file list instead (the manifest): every
# file of the install, the SHA-256 of its bytes, and where those bytes sit in
# the store packs, write-once prereleases named depot-0001, depot-0002 ... This
# tool compares that list with what is on disk, keeps what is already right,
# downloads only the objects it lacks as single byte ranges, checks every byte,
# stages complete files beside the game, and only then renames them into Mods
# under a journal. A crash at any point leaves either the old install or a
# switch the next run finishes.
#
# Everything under UserData/ODGModManager/depot (state.json, stage.json,
# journal.json and its progress file journal.next, lock) has exactly the format
# the Windows launcher writes (SPEC section 4), so a game folder both of them
# update stays consistent and a journal either one leaves is finished by the
# other. The rules below are the Rust crate's (odg-depot), held to it by the
# shared test vectors and the dual-use test (launcher repo, tools/depot-tests).

DEPOT_FORMAT = 1
CLIENT_KIND = "odg-linux-py"
DEPOT_TOOL = "odg-linux.py {} {}".format(CLIENT_KIND, TOOL_VERSION)
MANIFEST_ASSET = "odg-manifest.json.gz"
PART_SIZE = 67108864
PIECE_BASE = "Mods/NewSlots_Content/content.odgpak"
PUBLIC_REPO = "ArchdukePierre/operator-development-group"
# The developer repo is private and this file is a public download, so its name
# is never kept here as text: a dev manifest's repo is checked by its SHA-256
# (the crate keeps the name XOR-coded for the same reason, SPEC 20.4).
DEV_REPO_SHA256 = "190490539fe6596324c531f3eac3e9e5dbcbc471675406ed640ad8017ed19437"


def lineage_repo_ok(lineage, repo):
    if lineage == "public":
        return repo == PUBLIC_REPO
    return hashlib.sha256(repo.encode("utf-8")).hexdigest() == DEV_REPO_SHA256


GITHUB_WEB = "https://github.com"
CLIENT_KINDS = ("windows-dev", "windows-public", "linux-dev", "linux-public", "odg-linux-py")
# 3.3 and 3.5: what a manifest may hold at most.
MAX_MANIFEST_JSON = 64 << 20
MAX_FILES, MAX_OBJECTS, MAX_PACKS, MAX_STORES = 100000, 200000, 10000, 1000
MAX_CLEN = 68157440
MAX_PACK_SIZE = 1 << 31
MAX_PATH_LEN = 200
# The clothing pack (HashFiles.PackId): NewSlots reads pieces .001 to .999.
PAK_MAGIC = b"ODGPAK\x01\x00"
MAX_PIECE = 999
MAX_INDEX = 64 << 20
MAPS_DIR = "Mods/MapLoader_Assets/"
LOADER_NAME = "melonloader.zip"
EMPTY_SHA = hashlib.sha256(b"").hexdigest()
ZERO_SHA = "0" * 64
# Rust's str::trim: Unicode White_Space, which (unlike Python's str.strip) leaves U+001C to U+001F.
RUST_WS = ("\t\n\x0b\x0c\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a"
           "\u2028\u2029\u202f\u205f\u3000")
HEX64 = re.compile(r"[0-9a-f]{64}")
# 3.5 rule 8: where a manifest may never write, and the names the launcher owns
# or its cleanup deletes at every start (paths.rs).
NEVER_UNDER = ("Mods/MapLoader_Assets/Custom/", "Mods/ODG_Addons/", "Mods/_dlss5_backup/")
LAUNCHER_OWNED = ("Mods/.installed_release", "Mods/.installed_release.forgotten", "Mods/.installed_parts",
                  "Mods/.odg-shipped-files.prev", "Mods/.odg-dev-hold", "Mods/MapLoader_Assets/.installed_mapdata")
CLEANUP_NAMES = ("Mods/ODGGuard.Mod.dll", "Mods/ODGGuard.Mod.dll.odg-off", "Mods/manifest.odgsig",
                 "Mods/Weather.Mod.dll", "Mods/Weather.Mod.dll.odg-off", "Mods/Weather.log",
                 "Mods/Metrics.Mod.dll", "Mods/Metrics.Mod.dll.odg-off")
CLEANUP_DIRS = ("Mods/Weather_Assets/",)
RESERVED_NAMES = frozenset(["CON", "PRN", "AUX", "NUL"] + ["COM%d" % i for i in range(1, 10)]
                           + ["LPT%d" % i for i in range(1, 10)])
# The three generated lists are placed last, .modpack_version the very last (6.12).
LIST_FILES = ("Mods/.odg-obsolete.txt", "Mods/.odg-shipped-files.txt", "Mods/.modpack_version")
# 6.10: neighbouring objects share one range across a gap of 256 KiB, up to 32 MiB a range.
RANGE_GAP = 262144
RANGE_CAP = 33554432
SIGN_BATCH = 64
DEPOT_WORKERS = 4
RUNNING_RATE = 1048576
# A download at full speed looks for OPERATOR this often, and drops to one thread at
# RUNNING_RATE once it starts (within about two seconds).
GAME_POLL_SECS = 1.0
# what _get answers when its thread stops because OPERATOR started
_HANDED_BACK = -2
# A signed address is good for 60 minutes; public ones are used for 40, dev ones
# are signed again 10 minutes before the expiry the server gives.
URL_REUSE_SECS = 40 * 60
RESIGN_BEFORE_SECS = 10 * 60
RANGE_TRIES = 5
RANGE_PAUSES = (2, 5, 15, 30)
LOCK_STALE_SECS = 2 * 3600
LOCK_SAME_HOST_SECS = 24 * 3600   # this machine's lock: by its pid, an age only against a reused pid (F9)
LOCK_UNREADABLE_SECS = 60
# 4.1: FAT-family volumes store local time, so a DST change moves every mtime one hour.
DST_SHIFT_NS = 3600 * 10 ** 9
DST_SLACK_NS = 2 * 10 ** 9
STAGE_MAX_AGE = 14 * 86400
JOURNAL_FLUSH_OPS = 64
JOURNAL_FLUSH_SECS = 0.5
# The switch retries a rename an antivirus or another program holds, 10 times over about 5 s.
SWITCH_TRIES = 10
SWITCH_PAUSE = 0.5


class DepotError(Exception):
    """A file list that cannot be used. `code` is the shared vectors' code:
    gzip, json, too_big or "rule N" (rule 1 means a newer client is needed)."""

    def __init__(self, code, what):
        Exception.__init__(self, "{}: {}".format(code, what))
        self.code = code
        self.what = what


class SealError(Exception):
    """The clothing pack's seal id could not be read: not_found, invalid_data, eof or io."""

    def __init__(self, code, what):
        Exception.__init__(self, what)
        self.code = code


class PlanError(Exception):
    """No plan could be made (plan.rs PlanError): missing_hash or unresolvable."""

    def __init__(self, code, what):
        Exception.__init__(self, "{}: {}".format(code, what))
        self.code = code


def depot_dir(game):
    return Path(game) / "UserData" / "ODGModManager" / "depot"


def depot_test_mode():
    """A stand-in for github.com on this PC is set. Only tests do that, so every
    test hook below is dead in a player's copy."""
    return _local_override("ODG_TEST_GITHUB_WEB") is not None


def github_web():
    """github.com, or fakehub on this PC (A4: localhost only, like the launcher's ODG_TEST_GITHUB_WEB)."""
    return _local_override("ODG_TEST_GITHUB_WEB") or GITHUB_WEB


def _test_pause(seconds):
    if depot_test_mode() and os.environ.get("ODG_TEST_DEPOT_PAUSES") == "0":
        return 0
    return seconds


# ---- paths (paths.rs) --------------------------------------------------------

_ASCII_LOWER = {c: c + 32 for c in range(65, 91)}


def akey(p):
    """The key a path is matched by: ASCII lowercase, as the launcher's paths::key."""
    return p.translate(_ASCII_LOWER)


def pkey(p):
    """Sort key of the manifest's order: lowercase, byte by byte, ties by the exact string."""
    b = p.encode("utf-8", "surrogateescape")
    return b.lower(), b


def is_top_dll(p):
    """A top-level Mods/X.dll, the files the launcher switches on and off by renaming."""
    if not p.startswith("Mods/"):
        return False
    name = p[5:]
    return "/" not in name and len(name) > 4 and akey(name).endswith(".dll")


def _under(k, d):
    d = d.lower()
    return k.startswith(d) or k == d.rstrip("/")


def depot_path_problem(p):
    """3.5 rule 8 for one path: None when it may be shipped, else why not."""
    for c in p:
        if not " " <= c <= "~":
            return "{!r}: not printable ASCII".format(p)
    for c in '\\:*?"<>|':
        if c in p:
            return "{}: the character {} is not allowed".format(p, c)
    if not p.startswith("Mods/"):
        return "{}: does not start with Mods/".format(p)
    if len(p) > MAX_PATH_LEN:
        return "{}: longer than {} characters".format(p, MAX_PATH_LEN)
    for seg in p.split("/"):
        if not seg:
            return "{}: an empty name between slashes".format(p)
        if seg in (".", ".."):
            return "{}: {} is not a name".format(p, seg)
        if seg.endswith(" ") or seg.endswith("."):
            return "{}: a name ends in a space or a dot".format(p)
        # "nul.txt" and "CON.tar.gz" are reserved too: Windows looks at the name
        # before the first dot, without the spaces at its end.
        if seg.split(".")[0].rstrip(" ").upper() in RESERVED_NAMES:
            return "{}: {} is a name Windows reserves".format(p, seg)
    k = p.lower()
    for d in NEVER_UNDER:
        if _under(k, d):
            return "{}: nothing may be shipped under {}".format(p, d)
    if any(n.lower() == k for n in LAUNCHER_OWNED):
        return "{}: the launcher owns that file".format(p)
    if any(n.lower() == k for n in CLEANUP_NAMES) or any(_under(k, d) for d in CLEANUP_DIRS):
        return "{}: the launcher's cleanup deletes that at every start".format(p)
    if k.endswith(OFF):
        return "{}: the {} ending belongs to switched-off mods".format(p, OFF)
    return None


def _order_problem(paths):
    """Strictly increasing by lowercase path: sorted, and unique ignoring case."""
    prev = None
    for p in paths:
        if prev is not None:
            a, b = pkey(prev)[0], pkey(p)[0]
            if a == b:
                return "{}: listed twice, ignoring case ({})".format(p, prev)
            if a > b:
                return "{}: out of order (after {}); paths are sorted ignoring case".format(p, prev)
        prev = p
    return None


def _clash_problem(paths):
    """No file path is a folder of another (Mods/a and Mods/a/b), ignoring case."""
    keys = set(p.lower() for p in paths)
    for p in paths:
        k = p.lower()
        i = k.find("/")
        while i >= 0:
            if k[:i] in keys:
                return "{}: {} is a file, so it cannot also be a folder".format(p, p[:i])
            i = k.find("/", i + 1)
    return None


# ---- strict JSON: what the Rust client's serde_json reads ---------------------
#
# WHY STRICT. Every client must accept and refuse the same manifests and state
# files, or a folder updated by the launcher and by this tool would be planned
# two ways. serde refuses a wrong type, a missing field, a known field given
# twice, a float where an integer goes and a string with half a surrogate pair;
# Python's json takes all of those, so they are checked here by hand.

class _Json(Exception):
    """Not readable as the type the Rust client reads."""


class _JObj(dict):
    """A JSON object that remembers the keys it saw twice."""

    def __init__(self):
        dict.__init__(self)
        self.twice = set()


def _jpairs(pairs):
    d = _JObj()
    for k, v in pairs:
        if k in d:
            d.twice.add(k)
        d[k] = v
    return d


def _jconst(name):
    raise ValueError("{} is not JSON".format(name))


def json_strict(raw):
    """JSON bytes as serde_json reads them: UTF-8, no BOM, no NaN or Infinity.
    (serde skips unknown fields without a depth limit, and so does this.)"""
    try:
        return json.loads(raw.decode("utf-8"), object_pairs_hook=_jpairs, parse_constant=_jconst)
    except (ValueError, RecursionError) as e:
        raise _Json(str(e))


def _juint(bits):
    top = 1 << bits

    def read(v):
        if type(v) is not int or not 0 <= v < top:
            raise _Json("{!r} is not an unsigned {}-bit integer".format(v, bits))
        return v
    return read


_u8, _u32, _u64 = _juint(8), _juint(32), _juint(64)


def _jstr(v):
    if type(v) is not str:
        raise _Json("{!r} is not a string".format(v))
    try:
        v.encode("utf-8")
    except UnicodeEncodeError:
        raise _Json("a string holds half of a surrogate pair")
    return v


def _jbool(v):
    if type(v) is not bool:
        raise _Json("{!r} is not true or false".format(v))
    return v


def _jsha(v):
    if not HEX64.fullmatch(_jstr(v)):
        raise _Json("{!r} is not 64 lowercase hex digits".format(v))
    return v


def _jopt(read):
    return lambda v: None if v is None else read(v)


def _jlist(read):
    def r(v):
        if not isinstance(v, list):
            raise _Json("{!r} is not a list".format(v))
        return [read(x) for x in v]
    return r


def _jmap(read):
    def r(v):
        if not isinstance(v, dict):
            raise _Json("{!r} is not an object".format(v))
        return dict((_jstr(k), read(x)) for k, x in v.items())
    return r


def _jenum(*names):
    def r(v):
        # serde also reads a unit variant written as {"name": null}
        if isinstance(v, dict) and len(v) == 1 and list(v.values())[0] is None:
            v = list(v.keys())[0]
        if type(v) is not str or v not in names:
            raise _Json("{!r} is not one of {}".format(v, ", ".join(names)))
        return v
    return r


_REQ = object()     # a required field
_OPT = object()     # an Option without #[serde(default)]: may be missing from an object, not from an array


def _jstruct(v, fields, what):
    """A struct as serde's derived Deserialize reads it: an object (unknown keys
    ignored, a known key twice refused) or an array of the fields in order.
    `fields` is [(name, reader, default)]; default is _REQ, _OPT, a value, or a
    callable making one."""
    out = {}
    if isinstance(v, list):
        if len(v) > len(fields):
            raise _Json("{}: {} elements, at most {}".format(what, len(v), len(fields)))
        for i, (name, read, dflt) in enumerate(fields):
            if i < len(v):
                out[name] = read(v[i])
            elif dflt is _REQ or dflt is _OPT:
                raise _Json("{}: {} elements, too few".format(what, len(v)))
            else:
                out[name] = dflt() if callable(dflt) else dflt
        return out
    if not isinstance(v, dict):
        raise _Json("{}: {!r} is not an object".format(what, v))
    twice = getattr(v, "twice", ())
    for name, read, dflt in fields:
        if name in twice:
            raise _Json("{}: {} is given twice".format(what, name))
        if name in v:
            out[name] = read(v[name])
        elif dflt is _REQ:
            raise _Json("{}: {} is missing".format(what, name))
        elif dflt is _OPT:
            out[name] = None
        else:
            out[name] = dflt() if callable(dflt) else dflt
    return out


def _jrec(fields, what):
    return lambda v: _jstruct(v, fields, what)


def _jobject_row(v):
    """An object row: [pack, offset, clen, rawlen, codec, sha256], nothing more."""
    if not isinstance(v, list) or len(v) != 6:
        raise _Json("an object is [pack, offset, clen, rawlen, codec, sha256], not {!r}".format(v))
    codec = _u8(v[4])
    if codec not in (0, 1):
        raise _Json("unknown codec {} (0 stored, 1 raw deflate)".format(codec))
    return (_u32(v[0]), _u64(v[1]), _u64(v[2]), _u64(v[3]), codec, _jsha(v[5]))


# The manifest (3.4), field by field in the crate's order.
_F_STORE = [("id", _u32, _REQ), ("repo", _jstr, _REQ), ("tag", _jstr, _REQ),
            ("access", _jenum("public", "broker", "broker-release"), _REQ)]
_F_PACK = [("id", _u32, _REQ), ("store", _u32, _REQ), ("name", _jstr, _REQ), ("asset_id", _u64, _REQ),
           ("size", _u64, _REQ), ("sha256", _jsha, _REQ)]
_F_FILE = [("p", _jstr, _REQ), ("s", _u64, _REQ), ("o", _jlist(_u32), _REQ), ("h", _jopt(_jsha), None),
           ("x", _u8, 0)]
_F_PIECE = [("s", _u64, _REQ), ("h", _jsha, _REQ)]
_F_KNOWN = [("seal", _jstr, _REQ), ("pieces", _jlist(_jrec(_F_PIECE, "piece")), _REQ)]
_F_PAK = [("base", _jstr, _REQ), ("pieces", _u32, _REQ), ("seal", _jstr, _REQ), ("bytes", _u64, _REQ),
          ("known", _jlist(_jrec(_F_KNOWN, "known seal")), _REQ)]
_F_MAP = [("name", _jstr, _REQ), ("dir", _jstr, _REQ), ("zip", _jstr, _REQ), ("legacy", _jstr, _REQ)]
_F_LOADER = [("name", _jstr, _REQ), ("s", _u64, _REQ), ("o", _jlist(_u32), _REQ), ("h", _jopt(_jsha), None)]
_F_LINFO = [("version", _jstr, _REQ), ("size", _u64, _REQ), ("sha256", _jsha, _REQ)]
_F_LAUNCHERS = [("windows", _jrec(_F_LINFO, "launcher"), _REQ), ("linux", _jrec(_F_LINFO, "launcher"), _REQ)]
_F_OVERFLOW = [("part", _jstr, _REQ), ("digests", _jlist(_jstr), _REQ), ("dirs", _jlist(_jstr), _REQ)]
_F_ADOPT = [("tag", _jstr, _REQ), ("overflow", _jopt(_jrec(_F_OVERFLOW, "overflow")), None),
            ("differs", _jlist(_jstr), _REQ)]
_F_STATS = [("files", _u64, _REQ), ("bytes", _u64, _REQ), ("objects", _u64, _REQ), ("fresh", _u64, _REQ),
            ("prev_tag", _jstr, _REQ), ("from_prev", _u64, _REQ)]
_F_MANIFEST = [
    ("format", _u32, _REQ), ("min_client", _jmap(_jstr), _REQ), ("lineage", _jenum("dev", "public"), _REQ),
    ("repo", _jstr, _REQ), ("tag", _jstr, _REQ), ("pack_version", _jstr, _REQ), ("created", _jstr, _REQ),
    ("tool", _jstr, _REQ), ("stores", _jlist(_jrec(_F_STORE, "store")), _REQ),
    ("packs", _jlist(_jrec(_F_PACK, "pack")), _REQ), ("objects", _jlist(_jobject_row), _REQ),
    ("files", _jlist(_jrec(_F_FILE, "file")), _REQ), ("pak", _jrec(_F_PAK, "pak"), _REQ),
    ("maps", _jlist(_jrec(_F_MAP, "map")), _REQ), ("loader", _jrec(_F_LOADER, "loader"), _REQ),
    ("launchers", _jrec(_F_LAUNCHERS, "launchers"), _REQ), ("build", _jmap(_jstr), _REQ),
    ("adopt", _jlist(_jrec(_F_ADOPT, "adopt")), list), ("stats", _jrec(_F_STATS, "stats"), _REQ)]
# What every format keeps, so a client can say which version it needs for a
# format it cannot otherwise read.
_F_HEADER = [("format", _u32, _REQ), ("min_client", _jmap(_jstr), dict)]


class Manifest(object):
    """A parsed manifest (3.4) with the lookups the plan needs. `sha` is the
    SHA-256 of the .gz bytes it came from."""

    def __init__(self, d, sha=""):
        self.d = d
        self.sha = sha
        self.format = d["format"]
        self.min_client = d["min_client"]
        self.lineage = d["lineage"]
        self.repo = d["repo"]
        self.tag = d["tag"]
        self.stores = d["stores"]
        self.packs = d["packs"]
        self.objects = d["objects"]
        self.files = d["files"]
        self.pak = d["pak"]
        self.maps = d["maps"]
        self.loader = d["loader"]
        self.adopt = d["adopt"]
        self.by_key = {}
        for f in self.files:
            self.by_key.setdefault(akey(f["p"]), f)

    def fsha(self, f):
        """The SHA-256 of a whole file (or of the loader): its one object's, else
        h, else the empty hash for an empty file. All zeros, which no file has,
        on a manifest that failed validation."""
        o = f["o"]
        if len(o) == 1:
            return self.objects[o[0]][5] if o[0] < len(self.objects) else ZERO_SHA
        if f.get("h") is not None:
            return f["h"]
        return EMPTY_SHA if not o else ZERO_SHA

    def file(self, path):
        return self.by_key.get(akey(path))

    def piece_paths(self):
        return ["{}.{:03d}".format(self.pak["base"], i) for i in range(1, self.pak["pieces"] + 1)]

    def store_of(self, pack_id):
        return self.stores[self.packs[pack_id]["store"]]


def depot_gunzip(gz, cap=MAX_MANIFEST_JSON):
    """Gunzip at most `cap` bytes; more is too_big (a hostile list can inflate to gigabytes)."""
    if len(gz) < 18:
        raise DepotError("gzip", "too short to be gzip")
    d = zlib.decompressobj(16 + zlib.MAX_WBITS)
    try:
        out = d.decompress(gz, cap + 1)
    except zlib.error as e:
        raise DepotError("gzip", str(e))
    if len(out) > cap:
        raise DepotError("too_big", "the file list unpacks to more than 64 MiB")
    if not d.eof:
        raise DepotError("gzip", "the gzip stream is cut short")
    return out


def depot_parse_json(raw, sha=""):
    """A manifest from its JSON. A format other than 1 is rule 1 even when the
    rest does not fit format 1, so "needs a newer client" stays reachable (6.4)."""
    try:
        doc = json_strict(raw)
    except _Json as e:
        raise DepotError("json", str(e))
    try:
        d = _jstruct(doc, _F_MANIFEST, "the file list")
    except _Json as e:
        try:
            head = _jstruct(doc, _F_HEADER, "the file list")
        except _Json:
            raise DepotError("json", str(e))
        if head["format"] != DEPOT_FORMAT:
            raise DepotError("rule 1", "format {}; this tool reads format {}".format(head["format"], DEPOT_FORMAT))
        raise DepotError("json", str(e))
    if d["format"] != DEPOT_FORMAT:
        raise DepotError("rule 1", "format {}; this tool reads format {}".format(d["format"], DEPOT_FORMAT))
    return Manifest(d, sha)


def depot_parse_gz(gz):
    return depot_parse_json(depot_gunzip(gz), hashlib.sha256(gz).hexdigest())


def depot_peek_gz(gz):
    """(format, min_client) of a manifest of any format, for "needs a newer client"."""
    try:
        head = _jstruct(json_strict(depot_gunzip(gz)), _F_HEADER, "the file list")
    except _Json as e:
        raise DepotError("json", str(e))
    return head["format"], head["min_client"]


def _digits(s):
    return bool(s) and all("0" <= c <= "9" for c in s)


def dev_tag_shape(t):
    return t.startswith("v") and _digits(t[1:])


def public_tag_shape(t):
    if not t.startswith("v"):
        return False
    parts = t[1:].split(".")
    return len(parts) >= 2 and all(_digits(p) for p in parts)


def tag_shape_ok(lin, t):
    return dev_tag_shape(t) if lin == "dev" else public_tag_shape(t)


def store_tag_shape(t):
    return t.startswith("depot-") and len(t) >= 10 and _digits(t[6:])


def pack_name_shape(n):
    return (len(n) == 33 and n.startswith("odgs-") and n.endswith(".zip")
            and all(c in "0123456789abcdef" for c in n[5:29]))


def _alnum(c):
    return "0" <= c <= "9" or "a" <= c <= "z" or "A" <= c <= "Z"


def map_name_shape(n):
    return bool(n) and all(_alnum(c) or c == "_" for c in n)


def repo_shape(r):
    if "/" not in r:
        return False
    o, n = r.split("/", 1)
    return all(bool(s) and all(_alnum(c) or c in "._-" for c in s) for s in (o, n))


def _rule(n, what):
    return DepotError("rule {}".format(n), what)


def _object_problem(m, o, s, h):
    """The object rules shared by files (rule 7) and the loader (rule 11)."""
    total = 0
    for k, n in enumerate(o):
        if n >= len(m.objects):
            return "object {} does not exist".format(n)
        raw = m.objects[n][3]
        if len(o) > 1:
            if k + 1 < len(o) and raw != PART_SIZE:
                return "part {} is {} bytes; every part but the last is {}".format(k, raw, PART_SIZE)
            if k + 1 == len(o) and raw == 0:
                return "the last part is empty"
        total += raw
        if total >= 1 << 64:
            return "the parts' sizes overflow"
    if total != s:
        return "its objects hold {} bytes and the file is {}".format(total, s)
    if s == 0 and o:
        return "an empty file has no object"
    if len(o) == 1:
        return "h is given for a one-object file" if h is not None else None
    if h is None:
        return "h is missing for a file of {} objects".format(len(o))
    if not o and h != EMPTY_SHA:
        return "an empty file's h is not the empty hash"
    return None


def depot_validate(m, release_tag, source, listing_ids, test=False):
    """3.5, every rule in order; the first that fails is raised as DepotError("rule N").
    `source` is public, broker or token-dev. `test` accepts repos other than the
    two real ones (fakehub, the rehearsal repos) under the same structure rules."""
    # rule 1: the format, and the kill switch reaches every kind of client
    if m.format != DEPOT_FORMAT:
        raise _rule(1, "format {}".format(m.format))
    for k in CLIENT_KINDS:
        if not m.min_client.get(k, "").strip(RUST_WS):
            raise _rule(1, "min_client has no entry for {}".format(k))
    # rule 2: lineage, repo, tag
    if test:
        if not repo_shape(m.repo):
            raise _rule(2, "{} is not a repo".format(m.repo))
    elif not lineage_repo_ok(m.lineage, m.repo):
        raise _rule(2, "a {} manifest names the repo {}".format(m.lineage, m.repo))
    if m.tag != release_tag:
        raise _rule(2, "the file list is for {}, the release is {}".format(m.tag, release_tag))
    if not tag_shape_ok(m.lineage, m.tag):
        raise _rule(2, "{} is not a {} tag".format(m.tag, m.lineage))
    if m.d["pack_version"] != m.tag:
        raise _rule(2, "pack_version {} is not the tag {}".format(m.d["pack_version"], m.tag))
    # rule 3: the source agrees with the lineage
    if (source, m.lineage) not in (("public", "public"), ("broker", "dev"), ("token-dev", "dev")):
        raise _rule(3, "a {} manifest from the {} source".format(m.lineage, source))
    # rule 4: stores
    if len(m.stores) > MAX_STORES:
        raise _rule(4, "{} stores".format(len(m.stores)))
    first_public = None
    for i, s in enumerate(m.stores):
        if s["id"] != i:
            raise _rule(4, "store {} has id {}".format(i, s["id"]))
        acc = s["access"]
        if m.lineage == "public":
            if acc != "public":
                raise _rule(4, "a public manifest names the {} store {}".format(acc, s["tag"]))
            ok = s["repo"] == m.repo
        elif acc == "public":
            if test:
                if first_public is None:
                    first_public = s["repo"]
                ok = s["repo"] != m.repo and repo_shape(s["repo"]) and s["repo"] == first_public
            else:
                ok = s["repo"] == PUBLIC_REPO
        else:
            ok = s["repo"] == m.repo
        if not ok:
            raise _rule(4, "the {} store {} is in the repo {}".format(acc, s["tag"], s["repo"]))
        if not (store_tag_shape(s["tag"]) if acc in ("public", "broker") else s["tag"] == m.tag):
            raise _rule(4, "a {} store tagged {}".format(acc, s["tag"]))
    # rule 5: packs
    if len(m.packs) > MAX_PACKS:
        raise _rule(5, "{} packs".format(len(m.packs)))
    ids = set(listing_ids)
    for i, p in enumerate(m.packs):
        if p["id"] != i:
            raise _rule(5, "pack {} has id {}".format(i, p["id"]))
        if p["store"] >= len(m.stores):
            raise _rule(5, "pack {} names store {}, which does not exist".format(p["name"], p["store"]))
        if not pack_name_shape(p["name"]):
            raise _rule(5, "{} is not a pack name".format(p["name"]))
        if not 0 < p["size"] < MAX_PACK_SIZE:
            raise _rule(5, "pack {} is {} bytes".format(p["name"], p["size"]))
        if m.stores[p["store"]]["access"] == "broker-release" and p["asset_id"] not in ids:
            raise _rule(5, "pack {} (asset {}) is not on the release".format(p["name"], p["asset_id"]))
    # rule 6: objects
    if len(m.objects) > MAX_OBJECTS:
        raise _rule(6, "{} objects".format(len(m.objects)))
    for i, (pack, off, clen, raw, codec, _sha) in enumerate(m.objects):
        if pack >= len(m.packs):
            raise _rule(6, "object {} names pack {}, which does not exist".format(i, pack))
        if off + clen > m.packs[pack]["size"]:
            raise _rule(6, "object {} runs past the end of {}".format(i, m.packs[pack]["name"]))
        if codec == 0 and clen != raw:
            raise _rule(6, "object {} is stored with clen {} and rawlen {}".format(i, clen, raw))
        if raw > PART_SIZE:
            raise _rule(6, "object {} is {} bytes; a part is at most {}".format(i, raw, PART_SIZE))
        if clen > MAX_CLEN:
            raise _rule(6, "object {} is {} bytes encoded".format(i, clen))
    # rule 7: files
    if len(m.files) > MAX_FILES:
        raise _rule(7, "{} files".format(len(m.files)))
    for f in m.files:
        why = _object_problem(m, f["o"], f["s"], f["h"])
        if why:
            raise _rule(7, "{}: {}".format(f["p"], why))
        if f["x"] > 1:
            raise _rule(7, "{}: x is {}".format(f["p"], f["x"]))
    # rule 8: paths
    paths = [f["p"] for f in m.files]
    for p in paths:
        why = depot_path_problem(p)
        if why:
            raise _rule(8, why)
    why = _order_problem(paths) or _clash_problem(paths)
    if why:
        raise _rule(8, why)
    # rule 9: the clothing pack's pieces
    pak = m.pak
    if pak["base"] != PIECE_BASE:
        raise _rule(9, "the pak base is {}".format(pak["base"]))
    if pak["pieces"] > MAX_PIECE:
        raise _rule(9, "{} pieces; NewSlots reads at most {}".format(pak["pieces"], MAX_PIECE))
    total = 0
    for p in m.piece_paths():
        f = m.by_key.get(akey(p))
        if f is None or f["p"] != p:
            raise _rule(9, "{} is missing".format(p))
        total = min(total + f["s"], (1 << 64) - 1)
    base = akey(pak["base"]) + "."
    for f in m.files:
        k = akey(f["p"])
        if k.startswith(base) and len(k) == len(base) + 3 and _digits(k[len(base):]):
            if int(k[len(base):]) > pak["pieces"]:
                raise _rule(9, "{} is above the {} pieces".format(f["p"], pak["pieces"]))
    if total != pak["bytes"]:
        raise _rule(9, "the pieces hold {} bytes and pak.bytes is {}".format(total, pak["bytes"]))
    if not HEX64.fullmatch(pak["seal"]):
        raise _rule(9, "pak.seal is not a seal id")
    for k in pak["known"]:
        if not HEX64.fullmatch(k["seal"]):
            raise _rule(9, "known seal {} is not a seal id".format(k["seal"]))
    # rule 10: maps
    for mp in m.maps:
        if not map_name_shape(mp["name"]):
            raise _rule(10, "{} is not a map name".format(mp["name"]))
        if mp["dir"] != MAPS_DIR + mp["name"]:
            raise _rule(10, "map {} lives in {}".format(mp["name"], mp["dir"]))
        if mp["zip"] != "mapdata_{}.zip".format(mp["name"].lower()):
            raise _rule(10, "map {} names the zip {}".format(mp["name"], mp["zip"]))
        if akey(mp["dir"] + "/manifest.bin") not in m.by_key:
            raise _rule(10, "map {} ships no manifest.bin".format(mp["name"]))
        lg = mp["legacy"]
        if not (lg.startswith("sha256:") and HEX64.fullmatch(lg[7:])):
            raise _rule(10, "map {}: {} is not a sha256 digest".format(mp["name"], lg))
    # rule 11: the loader
    ld = m.loader
    if ld["name"] != LOADER_NAME:
        raise _rule(11, "the loader is named {}".format(ld["name"]))
    why = _object_problem(m, ld["o"], ld["s"], ld["h"])
    if why:
        raise _rule(11, "{}: {}".format(LOADER_NAME, why))
    # rule 12: adoption hints
    for a in m.adopt:
        if not tag_shape_ok(m.lineage, a["tag"]):
            raise _rule(12, "adopt names {}, not a {} tag".format(a["tag"], m.lineage))
        for p in a["differs"]:
            why = depot_path_problem(p)
            if why:
                raise _rule(12, "adopt {}: {}".format(a["tag"], why))
        why = _order_problem(a["differs"])
        if why:
            raise _rule(12, "adopt {}: {}".format(a["tag"], why))


def depot_may_apply(m, kind, own):
    """6.4: None when this client may apply `m`, else the version it needs ("" when
    the list names none). An unreadable minimum is refused: this entry is also
    the remote kill switch, and a client that cannot tell must not apply."""
    need = m.min_client.get(kind, "")
    if m.format != DEPOT_FORMAT or not need.strip(RUST_WS):
        return need
    return None if compare(own, need) in ("same", "ahead") else need


# ---- the clothing pack's seal id (HashFiles.PackId, 8.7) ------------------------

def _seal_io(e, p):
    code = "not_found" if getattr(e, "errno", None) == errno.ENOENT else "io"
    return SealError(code, "{}: {}".format(p, e))


def _read_joined(files, lens, off, count):
    out = bytearray()
    start = 0
    for f, ln in zip(files, lens):
        if len(out) < count and off + len(out) < start + ln:
            try:
                with open(str(f), "rb") as fh:
                    fh.seek(off + len(out) - start)
                    while len(out) < count:
                        chunk = fh.read(count - len(out))
                        if not chunk:
                            break
                        out += chunk
            except OSError as e:
                raise _seal_io(e, f)
        start += ln
        if len(out) >= count:
            break
    if len(out) < count:
        raise SealError("eof", "the pack is shorter than its header says")
    return bytes(out)


def seal_id(pieces):
    """SHA-256("odgpak-id/1|" + total length + "|" || the 12 header bytes || the
    sealed index), over the pieces joined in order. Exactly HashFiles.PackId:
    the same refusals, and zero-length (retired) pieces read as nothing."""
    if not pieces:
        raise SealError("not_found", "no pack")
    lens = []
    for p in pieces:
        try:
            st = os.stat(str(p))
        except OSError as e:
            raise _seal_io(e, p)
        if not stat.S_ISREG(st.st_mode):
            raise SealError("not_found", "{} is not a file".format(p))
        lens.append(st.st_size)
    total = sum(lens)
    head = _read_joined(pieces, lens, 0, 12)
    if head[:8] != PAK_MAGIC:
        raise SealError("invalid_data", "not a sealed ODG pack")
    n = int.from_bytes(head[8:12], "little", signed=True)
    if n <= 28 or n > MAX_INDEX or 12 + n > total:
        raise SealError("invalid_data", "bad index length {}".format(n))
    blob = _read_joined(pieces, lens, 12, n)
    h = hashlib.sha256("odgpak-id/1|{}|".format(total).encode("ascii"))
    h.update(head)
    h.update(blob)
    return h.hexdigest()


def _ticks(p):
    """Last-write time in .NET's 100 ns ticks, which is what NewSlots compares."""
    try:
        return os.stat(str(p)).st_mtime_ns // 100
    except OSError:
        return None


def active_source(base):
    """The files NewSlots reads the pack from (Pack.PickSource): the pieces .001
    up to the first missing number, unless a whole content.odgpak is strictly
    newer than .001; else the whole file; else nothing."""
    base = Path(base)
    first = Path(str(base) + ".001")
    if first.is_file():
        tw, tp = _ticks(base) if base.is_file() else None, _ticks(first)
        if not (tw is not None and tp is not None and tw > tp):
            out = []
            for i in range(1, MAX_PIECE + 1):
                p = Path("{}.{:03d}".format(base, i))
                if not p.is_file():
                    break
                out.append(p)
            return out
    return [base] if base.is_file() else []


# ---- client state on disk (section 4) -------------------------------------------

_F_FILEREC = [("h", _jsha, _REQ), ("s", _u64, _REQ), ("t", _u64, _REQ), ("n", _jopt(_jstr), None),
              ("b", _jopt(_jsha), None)]
_F_STATE = [("format", _u32, _REQ), ("lineage", _jenum("dev", "public"), _REQ), ("repo", _jstr, _REQ),
            ("tag", _jstr, _REQ), ("manifest", _jstr, _REQ), ("written_by", _jstr, _REQ), ("written", _jstr, _REQ),
            ("adopted_from", _jopt(_jstr), _OPT), ("forgotten", _jbool, _REQ), ("maps", _jlist(_jstr), _REQ),
            ("files", _jmap(_jrec(_F_FILEREC, "file record")), _REQ)]
_F_STAGED = [("s", _u64, _REQ), ("done", _jbool, False), ("parts", _jlist(_u32), list)]
_F_STAGEIX = [("format", _u32, _REQ), ("target", _jstr, _REQ),
              ("files", _jmap(_jrec(_F_STAGED, "staged file")), _REQ)]
_F_LOCK = [("pid", _u32, _REQ), ("tool", _jstr, _REQ), ("host", _jstr, _REQ), ("started", _u64, _REQ)]


def _jdump(doc):
    """Compact UTF-8 JSON, keys in the order given: what serde_json writes."""
    return json.dumps(doc, separators=(",", ":"), ensure_ascii=False).encode("utf-8")


def _fsync_dir(d):
    """On Linux a rename is only durable once its folder is flushed. Windows has no
    such call, and there the open fails and nothing is lost."""
    try:
        fd = os.open(str(d), os.O_RDONLY)
    except OSError:
        return
    try:
        os.fsync(fd)
    except OSError:
        pass
    finally:
        os.close(fd)


def depot_write(path, data):
    """Temp file, flush, rename, flush the folder: a crash leaves the old file or the new one."""
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    tmp = path.with_name(path.name + ".tmp")
    with open(str(tmp), "wb") as f:
        f.write(data)
        f.flush()
        os.fsync(f.fileno())
    os.replace(str(tmp), str(path))
    _fsync_dir(path.parent)


def depot_state_from(doc):
    """A state.json the way the launcher's state::load reads it; None (adoption)
    when it does not parse or is another format."""
    try:
        s = _jstruct(doc, _F_STATE, "state.json")
    except _Json:
        return None
    return s if s["format"] == DEPOT_FORMAT else None


def depot_load_state(game, name="state.json"):
    try:
        with open(str(depot_dir(game) / name), "rb") as f:
            return depot_state_from(json_strict(f.read()))
    except (OSError, _Json):
        return None


def depot_state_bytes(s):
    files = {}
    for k in sorted(s["files"]):
        r = s["files"][k]
        rec = {"h": r["h"], "s": r["s"], "t": r["t"]}
        if r.get("n") is not None:
            rec["n"] = r["n"]
        if r.get("b") is not None:
            rec["b"] = r["b"]
        files[k] = rec
    return _jdump({"format": DEPOT_FORMAT, "lineage": s["lineage"], "repo": s["repo"], "tag": s["tag"],
                   "manifest": s["manifest"], "written_by": s["written_by"], "written": s["written"],
                   "adopted_from": s.get("adopted_from"), "forgotten": bool(s["forgotten"]),
                   "maps": list(s["maps"]), "files": files})


def depot_save_state(game, s):
    depot_write(depot_dir(game) / "state.json", depot_state_bytes(s))


def depot_forget(game):
    """A subscription ended (the launcher's access::forget_installs): the next
    install treats itself as leaving the old build and takes its files back."""
    s = depot_load_state(game)
    if s is not None and not s["forgotten"]:
        s["forgotten"] = True
        try:
            depot_save_state(game, s)
        except OSError:
            pass


def depot_mark_legacy(game):
    """A legacy release is about to be unpacked over a depot install (6.19): the
    state no longer describes the folder, so it becomes state.legacy.json and the
    next depot release adopts the folder again."""
    d = depot_dir(game)
    if (d / "state.json").is_file():
        try:
            os.replace(str(d / "state.json"), str(d / "state.legacy.json"))
        except OSError:
            pass


def depot_remove_all(game):
    d = depot_dir(game)
    if d.is_dir() and not d.is_symlink():
        shutil.rmtree(str(d), ignore_errors=True)


def depot_switch_pending(game):
    return (depot_dir(game) / "journal.json").is_file()


def _stage_index_from(doc):
    try:
        ix = _jstruct(doc, _F_STAGEIX, "stage.json")
        for k in ix["files"]:
            _jsha(k)
    except _Json:
        return None
    return ix if ix["format"] == DEPOT_FORMAT else None


def _stage_index_bytes(ix):
    files = {}
    for k in sorted(ix["files"]):
        e = ix["files"][k]
        rec = {"s": e["s"]}
        if e.get("done"):
            rec["done"] = True
        elif e.get("parts"):
            rec["parts"] = sorted(set(e["parts"]))
        files[k] = rec
    return _jdump({"format": DEPOT_FORMAT, "target": ix["target"], "files": files})


# ---- the stage folder (4, 20.5) -----------------------------------------------------
#
# A staged object is depot/stage/<h2>/<sha256>, <h2> being the first two lowercase
# hex digits of its hash; its <sha256>.part and <sha256>.tmp sit beside it. The
# loader is assembled at the top as loader.zip, indexed under its hash. Why: on
# exFAT (his D: drive, most USB sticks and SD cards) every create, lookup and
# rename in a folder reads the whole folder, and one flat folder of 10,522 files
# cost a fresh install 285 to 380 s of kernel time (15.11, 11-17). 256 folders of
# about 40 files each cost almost nothing. The launcher and the odg-depot client
# use exactly these paths, so a stage folder or a journal one tool leaves, the
# other resumes and finishes.

STAGE_LOADER = "loader.zip"
_SHARD_NAME = re.compile(r"[0-9a-f]{2}")
_RANGE_NAME = re.compile(r"r[0-9]{6}(?:\.c[0-9]+)?\.bin")
# FILE_ATTRIBUTE_REPARSE_POINT: a junction, which a listing on Windows reports as a folder
_REPARSE = 0x400


def stage_where(sha, suffix=""):
    """(shard folder, name) of a staged object, or of its .part or .tmp (20.5)."""
    return sha[:2], sha + suffix


class _Made(object):
    """A name this run created in the stage, as a listing would have shown it."""
    __slots__ = ("size", "mtime", "dir")

    def __init__(self, size=0, folder=False):
        self.size, self.mtime, self.dir = size, time.time(), folder


class StageView(object):
    """depot/stage as this run sees it. Each folder is read by one listing, at most
    once a run, and kept up to date with what this run writes, renames and removes
    there; the readiness and reuse checks ask it, never the disk one file at a time
    (I3 open issue 3: a lookup per job after the download held a fresh install for
    32 s on exFAT). Nothing else writes here meanwhile: the folder lock is held.
    Windows answers sizes and times from the listing; Linux needs a stat per file
    asked about, which in a shard folder of a few dozen names is cheap."""

    def __init__(self, root):
        self.root = Path(root)
        self.lock = threading.RLock()
        self.folders = {}       # "" (the top) or a shard -> {name: DirEntry, or _Made}
        self.on_disk = {}       # folder -> whether it is there
        self.listings = 0       # folders read, for the tests

    def path(self, sub, name):
        return self.root / sub / name if sub else self.root / name

    def entries(self, sub):
        """The folder's names, from its one listing (read the first time it is asked for)."""
        with self.lock:
            d = self.folders.get(sub)
            if d is None:
                d = {}
                top = self.folders.get("") if sub else None
                if top is not None and sub not in top:
                    self.on_disk[sub] = False       # the top's listing says it is not there
                else:
                    self.listings += 1
                    try:
                        with os.scandir(str(self.root / sub if sub else self.root)) as it:
                            for e in it:
                                d[e.name] = e
                        self.on_disk[sub] = True
                    except OSError:
                        self.on_disk[sub] = False
                self.folders[sub] = d
            return d

    def kind(self, sub, name):
        """file, dir, link (a symlink or a junction: never followed nor removed) or None."""
        with self.lock:
            e = self.entries(sub).get(name)
            if e is None:
                return None
            if isinstance(e, _Made):
                return "dir" if e.dir else "file"
            try:
                if e.is_symlink():
                    return "link"
                if e.is_file(follow_symlinks=False):
                    return "file"
                if e.is_dir(follow_symlinks=False):
                    if os.name == "nt" and getattr(e.stat(follow_symlinks=False), "st_file_attributes", 0) & _REPARSE:
                        return "link"
                    return "dir"
            except OSError:
                pass
            return "other"

    def size(self, sub, name):
        """The size of a regular file in the folder, or None."""
        with self.lock:
            e = self.entries(sub).get(name)
            if e is None:
                return None
            if isinstance(e, _Made):
                return None if e.dir else e.size
            try:
                if e.is_symlink() or not e.is_file(follow_symlinks=False):
                    return None
                return e.stat(follow_symlinks=False).st_size
            except OSError:
                return None

    def age(self, sub, name, now):
        """Seconds since the entry was last written (0 when that cannot be told)."""
        with self.lock:
            e = self.entries(sub).get(name)
            if e is None:
                return 0
            if isinstance(e, _Made):
                return now - e.mtime
            try:
                return now - e.stat(follow_symlinks=False).st_mtime
            except OSError:
                return 0

    def ensure(self, sub):
        """The folder, created unless this run has seen it there."""
        with self.lock:
            if self.on_disk.get(sub):
                return
            try:
                (self.root / sub if sub else self.root).mkdir(parents=True)
                self.folders.setdefault(sub, {})        # made just now: nothing in it to read
            except FileExistsError:
                pass
            self.on_disk[sub] = True
            if sub:
                self.on_disk[""] = True
                top = self.folders.get("")
                if top is not None and sub not in top:
                    top[sub] = _Made(folder=True)

    def added(self, sub, name, size):
        with self.lock:
            self.entries(sub)[name] = _Made(size)
            self.on_disk[sub] = True

    def removed(self, sub, name):
        with self.lock:
            d = self.folders.get(sub)       # a folder is not read only to forget a name in it
            if d is not None:
                d.pop(name, None)

    def delete(self, sub, name):
        """Remove a stage entry: a file, or a folder with what is in it. A link or
        junction is never followed nor removed (20.2's rule for rmdir)."""
        k = self.kind(sub, name)
        p = self.path(sub, name)
        if k == "dir":
            _remove_tree(p)
            gone = not os.path.lexists(str(p))
        elif k == "file" or k == "other":
            try:
                os.remove(str(p))
                gone = True
            except OSError:
                gone = False
        else:
            gone = k is None
        if gone:
            self.removed(sub, name)
            if not sub and k == "dir":
                # a folder whose name differs from a shard's only in case is that shard on a
                # disk that ignores case: what this run knew of it is read again when asked
                with self.lock:
                    for s in (name, name.lower()):
                        self.folders.pop(s, None)
                        self.on_disk.pop(s, None)

    def drop_empty_shards(self):
        """20.5: after a successful switch the shard folders this run knows to be empty
        go (os.rmdir refuses any that still holds something)."""
        with self.lock:
            for sub in sorted(self.folders):
                if not sub or self.folders[sub] or not self.on_disk.get(sub):
                    continue
                try:
                    os.rmdir(str(self.root / sub))
                except OSError:
                    continue
                self.on_disk[sub] = False
                self.removed("", sub)


def _remove_tree(p):
    """A folder and everything in it, never following nor removing a link or junction."""
    if _is_link(p):
        return
    try:
        with os.scandir(str(p)) as it:
            entries = list(it)
    except OSError:
        return
    for e in entries:
        q = Path(e.path)
        if _is_link(q):
            continue
        try:
            isdir = e.is_dir(follow_symlinks=False)
        except OSError:
            isdir = False
        if isdir:
            _remove_tree(q)
        else:
            remove_quietly(q)
    try:
        os.rmdir(str(p))
    except OSError:
        pass


# ---- the journal (4.3) ------------------------------------------------------------

# Every op's fields in the crate's order. `name` of a place is written only once
# decided; a remove's `sha` is written even when null.
_OP_FIELDS = {"vacate": ("path", "sha"), "backup": ("path", "to"), "place": ("path", "sha", "size", "dll", "name"),
              "remove": ("path", "sha", "why"), "rmdir": ("path",), "touch": ("path",), "loader": (),
              "receipts": (), "state": (), "stamp": ("tag",)}
_OP_READ = {
    "vacate": [("path", _jstr, _REQ), ("sha", _jsha, _REQ)],
    "backup": [("path", _jstr, _REQ), ("to", _jstr, _REQ)],
    "place": [("path", _jstr, _REQ), ("sha", _jsha, _REQ), ("size", _u64, _REQ), ("dll", _jbool, _REQ),
              ("name", _jopt(_jstr), None)],
    "remove": [("path", _jstr, _REQ), ("sha", _jopt(_jsha), None), ("why", _jenum("takeback", "prune"), _REQ)],
    "rmdir": [("path", _jstr, _REQ)], "touch": [("path", _jstr, _REQ)], "loader": [], "receipts": [], "state": [],
    "stamp": [("tag", _jstr, _REQ)]}


def _journal_op(v):
    if not isinstance(v, dict) or v.get("op") not in _OP_READ:
        raise _Json("a journal op this tool does not know: {!r}".format(v))
    op = _jstruct(v, _OP_READ[v["op"]], "journal op")
    op["op"] = v["op"]
    return op


_F_JOURNAL = [
    ("format", _u32, _REQ),
    ("target", _jrec([("tag", _jstr, _REQ), ("manifest", _jstr, _REQ), ("lineage", _jenum("dev", "public"), _REQ)],
                     "journal target"), _REQ),
    ("from", _jrec([("tag", _jstr, _REQ), ("leaving", _jbool, _REQ)], "journal from"), _REQ),
    ("ops", _jlist(_journal_op), _REQ), ("next", _u64, _REQ)]


def depot_load_journal_raw(game):
    """(journal, SHA-256 of journal.json's bytes) with its progress applied, None
    when there is none; _Json when one is there and cannot be read."""
    d = depot_dir(game)
    try:
        with open(str(d / "journal.json"), "rb") as f:
            raw = f.read()
    except FileNotFoundError:
        return None
    except OSError as e:
        raise _Json(str(e))
    j = _jstruct(json_strict(raw), _F_JOURNAL, "journal.json")
    digest = hashlib.sha256(raw).hexdigest()
    prog = _load_progress(d / "journal.next")
    if prog is not None and prog[0] == digest:
        _apply_progress(j, prog[1], prog[2])
    return j, digest


def depot_load_journal(game):
    """The journal with its progress applied, None when there is none; _Json when
    one is there and cannot be read."""
    got = depot_load_journal_raw(game)
    return None if got is None else got[0]


def _load_progress(path):
    """journal.next as (journal digest, next, {op index: DLL name}); None when it is
    missing or not readable, which only means a replay from an earlier `next`."""
    try:
        with open(str(path), "rb") as f:
            doc = json_strict(f.read())
    except (OSError, _Json):
        return None
    if (not isinstance(doc, dict) or not isinstance(doc.get("journal"), str)
            or type(doc.get("next")) is not int or doc["next"] < 0):
        return None
    names = {}
    given = doc.get("names") if isinstance(doc.get("names"), dict) else {}
    for k, v in given.items():
        if _digits(k) and isinstance(v, str):
            names[int(k)] = v
    return doc["journal"], doc["next"], names


def _apply_progress(j, nxt, names):
    """`next` only ever forward and never past the end; a DLL name only where none
    was decided yet (journal.rs apply_progress)."""
    j["next"] = min(max(j["next"], nxt), len(j["ops"]))
    for i, name in names.items():
        if 0 <= i < len(j["ops"]):
            op = j["ops"][i]
            if op["op"] == "place" and op["dll"] and op.get("name") is None:
                op["name"] = name


def _op_json(op):
    out = {"op": op["op"]}
    for k in _OP_FIELDS[op["op"]]:
        if k == "name" and op.get("name") is None:
            continue
        out[k] = op.get(k)
    return out


class JournalFile(object):
    """journal.json, written once, and its progress in journal.next (SPEC 4.3 as
    amended at integration, the launcher's journal.rs).

    WHY TWO FILES. A fresh install's journal lists about 17,000 ops (2.5 MB);
    rewriting it every 64 ops would write 600 MB on a USB 2 drive. So the journal
    is written whole once, and the progress goes to journal.next, a few hundred
    bytes written by temp file and rename:
    {"journal":"<SHA-256 of journal.json's bytes>","next":N,"names":{"<op>":"<name>"}}.
    A rename never tears, where a number rewritten in place can (a power cut part
    way through "99" -> "190" can leave "990" and skip ops). The digest ties the
    progress to its journal: a progress file of another journal is ignored, and
    a lost or unreadable one only means a replay from an earlier `next`, which
    every op survives (each proves itself done). `names` holds each DLL's on-disk
    name, recorded before its rename."""

    def __init__(self, folder, digest=None):
        self.dir = Path(folder)
        self.path = self.dir / "journal.json"
        self.progress = self.dir / "journal.next"
        self.digest = digest
        self.next = 0
        self.names = {}

    def write(self, j):
        """A new journal: any progress file is an older journal's, so it goes first."""
        remove_quietly(self.progress)
        body = _jdump({"format": j["format"], "target": j["target"], "from": j["from"],
                       "ops": [_op_json(o) for o in j["ops"]], "next": j["next"]})
        depot_write(self.path, body)
        self.digest = hashlib.sha256(body).hexdigest()
        self.next = j["next"]
        self.names = {}

    def carry_on(self, j):
        """Progress for a journal another run wrote (bound by `digest`), from where it stands."""
        self.next = j["next"]
        self.names = dict((i, op["name"]) for i, op in enumerate(j["ops"])
                          if op["op"] == "place" and op["dll"] and op.get("name") is not None)

    def _flush(self):
        doc = {"journal": self.digest, "next": self.next}
        if self.names:
            doc["names"] = dict((str(i), self.names[i]) for i in sorted(self.names))
        depot_write(self.progress, _jdump(doc))

    def set_next(self, j, n):
        """Ops before n are done. OSError when the progress cannot be written; the
        caller goes on (a replay from an earlier `next` is slower, never wrong)."""
        j["next"] = n
        self.next = n
        self._flush()

    def decide(self, i, name):
        """A DLL's on-disk name for op i, recorded before the rename it governs."""
        self.names[i] = name
        self._flush()

    def clear(self):
        """The switch is done: the journal goes, then its progress."""
        remove_quietly(self.path)
        remove_quietly(self.progress)


# ---- the folder lock (4.5) ------------------------------------------------------------

def depot_host_id():
    """The first 8 hex digits of the SHA-256 of the computer's name, as every ODG tool writes it."""
    if os.name == "nt":
        name = os.environ.get("COMPUTERNAME", "")
    else:
        try:
            name = os.uname().nodename
        except (AttributeError, OSError):
            name = ""
    return hashlib.sha256(name.encode("utf-8")).hexdigest()[:8]


def _pid_alive(pid):
    if pid <= 0:
        return False
    if os.name == "nt":
        # os.kill on Windows ends the process whatever the signal, so ask the kernel instead.
        import ctypes
        k32 = ctypes.WinDLL("kernel32", use_last_error=True)
        h = k32.OpenProcess(0x1000, False, int(pid))  # PROCESS_QUERY_LIMITED_INFORMATION
        if not h:
            return ctypes.get_last_error() == 5  # access denied: it is there
        try:
            code = ctypes.c_ulong()
            if not k32.GetExitCodeProcess(h, ctypes.byref(code)):
                return True
            return code.value == 259  # STILL_ACTIVE
        finally:
            k32.CloseHandle(h)
    try:
        os.kill(pid, 0)
    except ProcessLookupError:
        return False
    except PermissionError:
        return True
    except (OSError, OverflowError):
        return False
    return True


class DepotLock(object):
    """depot/lock: one updater per game folder, whichever tool it is (the launcher,
    its Linux fork or this one). Another machine's lock is taken over when older
    than two hours; this machine's when its process is gone (or after a day, against
    a reused pid), never by age alone: a slow download keeps its lock (F9)."""

    def __init__(self, game):
        self.path = depot_dir(game) / "lock"
        self.held = False
        self.mine = None

    def acquire(self):
        if self.held:
            return True
        try:
            self.path.parent.mkdir(parents=True, exist_ok=True)
        except OSError:
            return False
        for _ in range(3):
            try:
                fd = os.open(str(self.path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
            except FileExistsError:
                if not self._stale():
                    return False
                try:
                    os.remove(str(self.path))
                except FileNotFoundError:
                    pass
                except OSError:
                    return False
                continue
            except OSError:
                return False
            try:
                self.mine = {"pid": os.getpid(), "tool": DEPOT_TOOL, "host": depot_host_id(), "started": now_unix()}
                os.write(fd, _jdump(self.mine))
                os.fsync(fd)
            finally:
                os.close(fd)
            self.held = True
            return True
        return False

    def _stale(self):
        try:
            with open(str(self.path), "rb") as f:
                info = _jstruct(json_strict(f.read()), _F_LOCK, "lock")
        except (OSError, _Json):
            # Half written by a tool that died between creating and filling it.
            try:
                return time.time() - os.stat(str(self.path)).st_mtime > LOCK_UNREADABLE_SECS
            except OSError:
                return True
        if info["host"] == depot_host_id():
            return not _pid_alive(info["pid"]) or now_unix() - info["started"] > LOCK_SAME_HOST_SECS
        return now_unix() - info["started"] > LOCK_STALE_SECS

    def release(self):
        """Deletes the file only while it is still ours: a lock another tool took over is its."""
        if self.held:
            try:
                with open(str(self.path), "rb") as f:
                    info = _jstruct(json_strict(f.read()), _F_LOCK, "lock")
                ours = self.mine is not None and info["pid"] == self.mine["pid"] and info["started"] == self.mine["started"]
            except (OSError, _Json):
                ours = False
            if ours:
                remove_quietly(self.path)
            self.held = False


# ---- what is on disk (6.5) -------------------------------------------------------------

class Seen(object):
    """One file as listed: its on-disk relative name, size and mtime (ns)."""
    __slots__ = ("name", "size", "mtime_ns")

    def __init__(self, name, size, mtime_ns):
        self.name, self.size, self.mtime_ns = name, size, mtime_ns

    def __repr__(self):
        return "Seen({!r}, {}, {})".format(self.name, self.size, self.mtime_ns)


class Observation(object):
    """Files keyed by their lowercase path (observe.rs Observation)."""

    def __init__(self):
        self.files = {}
        self.listed_dirs = 0

    def add(self, name, size, mtime_ns):
        k = akey(name)
        cur = self.files.get(k)
        # Two spellings in one Linux folder: the first in string order, as Disk picks.
        if cur is None or name < cur.name:
            self.files[k] = Seen(name, size, mtime_ns)

    def get(self, rel):
        return self.files.get(akey(rel))

    def lookup(self, rel):
        k = akey(rel)
        return self.files.get(k), self.files.get(k + OFF)


def _list_into(ob, disk, folder, deep):
    try:
        entries = list(os.scandir(str(folder)))
    except OSError:
        return
    ob.listed_dirs += 1
    base = disk.rel(folder)
    for e in entries:
        try:
            if e.is_symlink():
                continue
            if e.is_dir(follow_symlinks=False):
                if deep:
                    _list_into(ob, disk, Path(e.path), True)
                continue
            if not e.is_file(follow_symlinks=False):
                continue
            st = e.stat(follow_symlinks=False)
        except OSError:
            continue
        ob.add(base + "/" + e.name if base and base != "." else e.name, st.st_size, st.st_mtime_ns)


def depot_observe(game, rels, deep=()):
    """6.5: (on-disk name, size, mtime) of every file in every folder that holds
    one of `rels`, from directory listings alone (scandir answers the stat from
    the listing; no file is opened). `deep` folders are listed with everything
    under them (a retired map, so nothing of the player's in it is missed)."""
    disk = Disk(game)
    ob = Observation()
    folders = {}
    for rel in rels:
        parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
        folders.setdefault(akey(parent), parent)
    for _k, frel in sorted(folders.items()):
        p = disk.find(frel) if frel else Path(game)
        if p is not None and p.is_dir():
            _list_into(ob, disk, p, False)
    for rel in deep:
        p = disk.find(rel)
        if p is not None and p.is_dir() and not p.is_symlink():
            _list_into(ob, disk, p, True)
    return ob


def depot_read_legacy(disk):
    """The legacy stamps (adopt::read_legacy): a stamp is None when missing or empty."""
    def stamp(rel):
        return disk.read(rel).strip() or None

    def pairs(rel):
        out = []
        for line in disk.read(rel).splitlines():
            line = line.strip()
            if line:
                name, tab, digest = line.partition("\t")
                out.append((name.strip(), digest.strip()) if tab else (line, ""))
        return out

    shipped = [line.strip().replace("\\", "/") for line in disk.read(SHIPPED_LIST).splitlines()]
    return {"installed_release": stamp(STAMP), "forgotten": stamp(FORGOTTEN), "parts": pairs(PARTS_STAMP),
            "mapdata": pairs(MAP_STAMP), "shipped": [s for s in shipped if s and not s.startswith("#")]}


# ---- adoption (6.7) ------------------------------------------------------------------------

def depot_active_pieces(ob):
    """The pieces NewSlots reads, as the listing saw them (Pack.PickSource): .001,
    .002 ... up to the first missing number, unless a whole content.odgpak is
    strictly newer than .001 (in .NET's 100 ns ticks). None when it is not the
    pieces NewSlots reads."""
    first = ob.get(PIECE_BASE + ".001")
    if first is None:
        return None
    whole = ob.get(PIECE_BASE)
    if whole is not None and whole.mtime_ns // 100 > first.mtime_ns // 100:
        return None
    out = []
    for i in range(1, MAX_PIECE + 1):
        s = ob.get("{}.{:03d}".format(PIECE_BASE, i))
        if s is None:
            break
        out.append(s)
    return out


def depot_known_seal(m, ob, local_seal):
    """The pak.known entry the local pieces hold: its seal is theirs, and its piece
    count and sizes are theirs (6.7.3)."""
    local = depot_active_pieces(ob)
    if local_seal is None or local is None:
        return None
    for k in m.pak["known"]:
        if k["seal"] == local_seal and len(k["pieces"]) == len(local) \
                and all(pc["s"] == s.size for pc, s in zip(k["pieces"], local)):
            return k
    return None


def depot_known_hashes(m, ob, local_seal):
    """What adoption knows without reading: each local piece's hash, from the
    known seal it holds, by the key of its on-disk name."""
    k, local = depot_known_seal(m, ob, local_seal), depot_active_pieces(ob)
    if k is None or local is None:
        return {}
    return dict((akey(s.name), pc["h"]) for pc, s in zip(k["pieces"], local))


def _inside(p, d):
    """`p` is inside the folder `d` (with or without its slash), ignoring case."""
    d = akey(d.rstrip("/"))
    k = akey(p)
    return len(k) > len(d) and k.startswith(d) and k[len(d)] == "/"


def depot_trust(m, legacy, ob, local_seal):
    """6.7 steps 2 to 5: which manifest paths an adopted install may take as
    shipped without reading them ("trusted"), by the pak's seal, the maps'
    receipts and the release's adoption hint. `local_seal` is the seal id of
    the local pieces (depot_active_pieces), or None."""
    trust = {}
    by_seal = "trusted" if depot_known_seal(m, ob, local_seal) is not None else "untrusted"
    for p in m.piece_paths():
        trust[p] = by_seal
    for mp in m.maps:
        receipt = any(akey(n) == akey(mp["zip"]) and d == mp["legacy"] for n, d in legacy.get("mapdata") or [])
        t = "trusted" if receipt and ob.get(mp["dir"] + "/manifest.bin") is not None else "untrusted"
        for f in m.files:
            if _inside(f["p"], mp["dir"]):
                trust.setdefault(f["p"], t)
    hint = _hint(m, legacy)
    ov = hint.get("overflow") if hint is not None else None
    part_ok = ov is not None and any(akey(n) == akey(ov["part"]) and d in ov["digests"]
                                     for n, d in legacy.get("parts") or [])
    for f in m.files:
        p = f["p"]
        if p in trust:
            continue
        if hint is None or any(akey(d) == akey(p) for d in hint["differs"]):
            trust[p] = "untrusted"
        elif ov is not None and any(_inside(p, d) for d in ov["dirs"]):
            trust[p] = "trusted" if part_ok else "untrusted"
        else:
            trust[p] = "trusted"
    return trust


def _hint(m, legacy):
    """The adoption hint for the installed legacy tag (none on a lineage switch,
    an unknown tag or missing stamps)."""
    tag = legacy.get("installed_release")
    if tag is None:
        return None
    return next((a for a in m.adopt if a["tag"] == tag), None)


class Staged(object):
    """What is staged (stage.json, reconciled with stage/): whole files by hash and
    size, and the verified parts of files being assembled."""

    def __init__(self):
        self.whole = {}
        self.parts = {}

    @classmethod
    def from_index(cls, ix):
        st = cls()
        for sha, e in ((ix or {}).get("files") or {}).items():
            if e.get("done"):
                st.whole[sha] = e["s"]
            elif e.get("parts"):
                st.parts[sha] = (e["s"], set(e["parts"]))
        return st

    def has(self, sha, size):
        return self.whole.get(sha) == size

    def has_part(self, sha, size, k):
        p = self.parts.get(sha)
        return p is not None and p[0] == size and k in p[1]


# ---- the plan (6.6 to 6.8, 6.13, 6.15) ---------------------------------------------------
#
# A pure function: what to keep, build, move, back up and remove, from the
# manifest, what is installed and what is on disk. It reads nothing itself, so
# the shared vectors run it on a virtual disk and must get the Rust plan's
# answer. The rules, in the order they apply, are the crate's plan.rs:
#
# 1. have: the state's tag, else .installed_release, else the forgotten stamp.
#    Same lineage, ahead and the listing not exact: kept newer, nothing else.
# 2. Each manifest file against its file on disk (a DLL under its recorded
#    name, else X.dll, else X.dll.odg-off): missing needs the content; REPAIR
#    hashes it when its size is right; with a state, a record whose stat still
#    holds gives its hash unread, else the right size is hashed; adopted, a
#    trusted file of the right size is current, a trusted piece takes its
#    seal's hash, an untrusted piece is read when its size is any piece's, any
#    other untrusted file when its size is right. Equal hashes are current;
#    a known other hash makes the old bytes a source for other files.
# 3. Runtime-written files follow 6.15 and are never sources.
# 4. Removals: the prune of retired maps, then take-back when leaving; only
#    where may_delete agrees and only files still as shipped, unless they have
#    no record (the legacy list, a DLL's twin) or are a retired map's
#    runtime-written files.
# 5. Sources, first that applies: an empty file is built from nothing; staged;
#    a local file of that hash (files leaving their path first, then by path);
#    per part of a big file a staged part, a slice of a known local file whose
#    own entry (installed or target manifest) holds that part, else a fetch.
#    The last user of a leaving file moves it, earlier ones copy it, and a
#    file that moves is not also removed.
# 6. The loader when it is not ready; .001 touched when a whole content.odgpak
#    could win over the pieces.

def _same_stat(rec, size, mtime_ns):
    """4.1: same size, and the same mtime or one exactly an hour away (give or take 2 s)."""
    if rec["s"] != size:
        return False
    d = abs(rec["t"] - mtime_ns)
    return d == 0 or abs(d - DST_SHIFT_NS) <= DST_SLACK_NS


def _plan_frame(m, inst, opts):
    """(have, kept newer, leaving) of 6.4 steps 2 to 4."""
    state, legacy = inst.get("state"), inst.get("legacy") or {}
    tag = m.tag
    if state is not None:
        have = state["tag"]
    elif legacy.get("installed_release") is not None:
        have = legacy["installed_release"]
    else:
        have = legacy.get("forgotten") or ""
    same = lineage(have) == lineage(tag)
    ahead = bool(have) and not opts.get("exact") and same and compare(have, tag) == "ahead"
    leaving = bool(have) and (not same or compare(have, tag) == "ahead"
                              or bool(state is not None and state["forgotten"])
                              or (legacy.get("forgotten") is not None and legacy.get("installed_release") is None))
    return have, ahead, leaving


def depot_state_matches(state, legacy):
    """4.1: a state still describes the install when its tag is the stamp's or the
    forgotten stamp's; else it is adopted again."""
    return state["tag"] in (legacy.get("installed_release"), legacy.get("forgotten"))


def _record(h, s, path, b):
    r = {"h": h, "s": s.size, "t": s.mtime_ns}
    if s.name != path:
        r["n"] = s.name
    if b is not None:
        r["b"] = b
    return r


def _target_name(f, ob):
    """6.8.1: a top-level DLL keeps .odg-off when only that name exists; any other
    file keeps the spelling already on disk."""
    on, off = ob.lookup(f["p"])
    if is_top_dll(f["p"]):
        if on is not None:
            return on.name
        if off is not None:
            return off.name
        return f["p"]
    return on.name if on is not None else f["p"]


class _Known(object):
    """A local file whose content is known: a possible source."""
    __slots__ = ("name", "path", "h", "leaving")

    def __init__(self, name, path, h, leaving):
        self.name, self.path, self.h, self.leaving = name, path, h, leaving


class _Planner(object):
    def __init__(self, m, inst, ob, trust, opts, staged):
        self.m, self.ob, self.trust, self.opts = m, ob, trust or {}, opts
        self.state = inst.get("state")
        self.old = inst.get("manifest")
        self.legacy = inst.get("legacy") or {}
        self.staged = staged if staged is not None else Staged()
        self.records = {}
        if self.state is not None:
            for p, r in sorted(self.state["files"].items()):
                self.records[akey(p)] = (p, r)
        self.have, self.ahead, self.leaving = _plan_frame(m, inst, opts)
        self.hint = _hint(m, self.legacy)
        pieces = m.piece_paths()
        self.piece_keys = set(akey(p) for p in pieces)
        self.piece_sizes = set(m.file(p)["s"] for p in pieces if m.file(p) is not None)

    def record(self, path):
        hit = self.records.get(akey(path))
        return hit[1] if hit is not None else None

    def observed(self, f):
        """The file on disk for manifest file `f`: under its recorded name when that
        is there, else (a DLL) X.dll, else X.dll.odg-off. A toggle renames a DLL and
        keeps its size and mtime, so a switched mod is still current."""
        r = self.record(f["p"])
        if r is not None and r.get("n"):
            s = self.ob.get(r["n"])
            if s is not None:
                return s
        on, off = self.ob.lookup(f["p"])
        return (on or off) if is_top_dll(f["p"]) else on

    def _hashed(self, o, sha, hashes):
        if hashes is None:
            return "hash", None
        h = hashes.get(akey(o.name))
        if h is None:
            raise PlanError("missing_hash", o.name)
        if h == sha:
            return "current", None
        return "needs", (None if h == ZERO_SHA else h)

    def classify(self, f, o, hashes):
        """("current" | "needs" | "hash", the hash of what is on disk when known)."""
        sha = self.m.fsha(f)
        if o is None:
            return "needs", None
        right = o.size == f["s"]
        if self.opts.get("repair"):
            return self._hashed(o, sha, hashes) if right else ("needs", None)
        if self.state is not None:
            r = self.record(f["p"])
            if r is not None and _same_stat(r, o.size, o.mtime_ns):
                return ("current", None) if r["h"] == sha else ("needs", r["h"])
            return self._hashed(o, sha, hashes) if right else ("needs", None)
        trusted = self.trust.get(f["p"]) == "trusted"
        if akey(f["p"]) in self.piece_keys:
            if trusted:
                # the known seal's hash, which the caller put in `hashes`
                return ("current", None) if hashes is None else self._hashed(o, sha, hashes)
            return self._hashed(o, sha, hashes) if o.size in self.piece_sizes else ("needs", None)
        if trusted:
            return ("current", None) if right else ("needs", None)
        return self._hashed(o, sha, hashes) if right else ("needs", None)

    def x_action(self, f, local):
        """6.15 for a runtime-written file on disk whose hash is `local`."""
        new = self.m.fsha(f)
        if local == ZERO_SHA:
            # unreadable now (an antivirus, another program): the player's edits may be in
            # it, so it is backed up first; a backup that cannot read it stops the switch
            # rather than replace edits nobody read (F8, as the launcher does)
            return "backup", None
        if local == new:
            return "current", None
        if self.state is None:
            if self.hint is not None and not any(akey(d) == akey(f["p"]) for d in self.hint["differs"]):
                return "keep", local
            return "backup", None
        r = self.record(f["p"])
        base = r.get("b") if r is not None else None
        if base is None and self.old is not None and self.old.file(f["p"]) is not None:
            base = self.old.fsha(self.old.file(f["p"]))
        if base == new:
            return "keep", local
        if base == local:
            return "place", None
        return "backup", None

    def retired_maps(self):
        """(name as installed, folder) of each installed map a target that lists maps
        no longer ships, never Custom, only while its manifest.bin is there."""
        if not self.m.maps:
            return []
        if self.state is not None:
            names = list(self.state["maps"])
        else:
            names = []
            for n, _d in self.legacy.get("mapdata") or []:
                k = akey(n)
                if k.startswith("mapdata_") and k.endswith(".zip") and len(k) > len("mapdata_.zip"):
                    names.append(k[len("mapdata_"):-len(".zip")])
        out, seen = [], set()
        for name in names:
            k = akey(name)
            if any(akey(mp["name"]) == k for mp in self.m.maps) or k == "custom" or k in seen:
                continue
            seen.add(k)
            d = MAPS_DIR + name
            if self.ob.get(d + "/manifest.bin") is not None:
                out.append((name, d))
        return out

    def candidates(self, may_delete):
        """The prune and take-back candidates before their checks, and the map
        folders the prune touches."""
        out, taken, map_dirs = [], set(), []

        def push(name, rec, force, why):
            o = self.ob.get(name)
            if o is None or not safe_rel(o.name) or any(akey(l) == akey(o.name) for l in LAUNCHER_OWNED):
                return False
            k = akey(o.name)
            if k in taken:
                return False
            taken.add(k)
            if not may_delete(o.name):
                return False
            out.append({"name": o.name, "rec": rec, "force": force, "why": why})
            return True

        for _name, d in self.retired_maps():
            hit = False
            if self.state is not None:
                for p, r in sorted(self.state["files"].items()):
                    if not _inside(p, d):
                        continue
                    old = self.old.file(p) if self.old is not None else None
                    force = r.get("b") is not None or (old is not None and old.get("x") == 1)
                    hit = push(r.get("n") or p, r, force, "prune") or hit
            else:
                for p in self.legacy.get("shipped") or []:
                    if _inside(p, d):
                        hit = push(p, None, False, "prune") or hit
            if hit:
                map_dirs.append(d)
        if self.leaving:
            if self.state is not None:
                for p, r in sorted(self.state["files"].items()):
                    if self.m.file(p) is not None:
                        continue
                    # the recorded name first (a toggle renames without telling the
                    # state, and keeps the stat); a DLL's other name, if there too, is
                    # a twin without a record
                    names = [r.get("n") or p] + ([p, p + OFF] if is_top_dll(p) else [])
                    matched, seen = False, set()
                    for name in names:
                        if akey(name) in seen:
                            continue
                        seen.add(akey(name))
                        if self.ob.get(name) is None:
                            continue
                        push(name, None if matched else r, False, "takeback")
                        matched = True
            else:
                for p in self.legacy.get("shipped") or []:
                    if self.m.file(p) is not None:
                        continue
                    push(p, None, False, "takeback")
                    if is_top_dll(p):
                        push(p + OFF, None, False, "takeback")
        return out, map_dirs


def depot_needs_hash(m, inst, ob, trust, opts, may_delete=None, staged=None):
    """The on-disk names whose bytes must be read before planning (6.6, 6.7, 6.13,
    6.15). A trusted piece needs no read: its seal's hash goes into `hashes`."""
    p = _Planner(m, inst, ob, trust, opts, staged)
    if p.ahead:
        return []
    out = set()
    for f in m.files:
        o = p.observed(f)
        if o is None:
            continue
        if f["x"]:
            out.add(o.name)
        elif p.classify(f, o, None)[0] == "hash":
            out.add(o.name)
    cands, _dirs = p.candidates(may_delete or (lambda rel: True))
    for c in cands:
        r = c["rec"]
        o = ob.get(c["name"])
        if r is not None and not c["force"] and o is not None and not _same_stat(r, o.size, o.mtime_ns) \
                and o.size == r["s"]:
            out.add(c["name"])
    return sorted(out, key=pkey)


class Plan(object):
    """What one run does (plan.rs Plan). Builds are files assembled in stage/;
    parts are ("staged",), ("local_file", name), ("local_slice", name, offset)
    or ("fetch", object number)."""

    def __init__(self, tag):
        self.tag = tag
        self.leaving = False
        self.kept_newer = None
        self.keep = []
        self.builds = []
        self.moves = []
        self.backups = []
        self.removes = []
        self.rmdirs = []
        self.loader = False
        self.loader_parts = []
        self.touch_001 = False
        self.fetch_objects = []
        self.download_bytes = 0
        self.stage_bytes = 0

    def nothing_to_do(self):
        return self.kept_newer is None and not (self.builds or self.moves or self.backups or self.removes
                                                or self.rmdirs or self.loader or self.touch_001)


def _slice_source(m, old, known, want):
    """A slice of a known local big file whose own entry (in the installed manifest,
    then the target) holds a part with hash `want`: every unchanged part of a
    changed big file is copied from the old copy instead of fetched."""
    for k in sorted(known, key=lambda k: pkey(k.path)):
        for man in ([old] if old is not None else []) + [m]:
            e = man.file(k.path)
            if e is None or len(e["o"]) < 2 or man.fsha(e) != k.h:
                continue
            for j, n in enumerate(e["o"]):
                if n < len(man.objects) and man.objects[n][5] == want:
                    return ("local_slice", k.name, j * PART_SIZE)
    return None


def depot_plan(m, inst, ob, hashes, trust, opts, may_delete, staged=None):
    """6.8. `inst` is {"state", "manifest" (the installed one), "legacy"}; `hashes`
    maps the key of an on-disk name to its SHA-256 for every name needs_hash
    asked for (and the pieces a seal proved); `opts` holds repair, exact and
    loader_needed; `may_delete(name)` is 6.13's guard; `staged` is Staged."""
    p = _Planner(m, inst, ob, trust, opts, staged)
    plan = Plan(m.tag)
    if p.ahead:
        plan.kept_newer = p.have
        return plan
    plan.leaving = p.leaving

    # 1. every manifest file
    needs, known = [], []
    for f in m.files:
        sha = m.fsha(f)
        o = p.observed(f)
        target = _target_name(f, ob)
        if f["x"]:
            if o is None:
                needs.append((f, sha, target, True))
                continue
            local = hashes.get(akey(o.name))
            if local is None:
                raise PlanError("missing_hash", o.name)
            act, h = p.x_action(f, local)
            if act == "current":
                plan.keep.append((f["p"], _record(sha, o, f["p"], sha)))
            elif act == "keep":
                plan.keep.append((f["p"], _record(h, o, f["p"], sha)))
            else:
                if act == "backup":
                    plan.backups.append({"path": o.name, "to": o.name + ".odg-local.bak"})
                needs.append((f, sha, target, True))
            continue
        cls, local = p.classify(f, o, hashes)
        if cls == "current":
            plan.keep.append((f["p"], _record(sha, o, f["p"], None)))
            known.append(_Known(o.name, f["p"], sha, False))
        elif cls == "needs":
            if local is not None and o is not None:
                known.append(_Known(o.name, f["p"], local, True))
            needs.append((f, sha, target, False))
        else:
            raise PlanError("missing_hash", o.name if o is not None else f["p"])

    # 2. removals, and the files they make movable
    cands, map_dirs = p.candidates(may_delete)
    removes = []
    for c in cands:
        o = ob.get(c["name"])
        if o is None:
            continue
        r = c["rec"]
        expect = None
        if r is not None and not c["force"]:
            if not (_same_stat(r, o.size, o.mtime_ns) or (o.size == r["s"] and hashes.get(akey(c["name"])) == r["h"])):
                continue        # changed since ODG put it there: the player's now
            known.append(_Known(o.name, c["name"], r["h"], True))
            expect = r["h"]
        removes.append({"path": o.name, "expect": expect, "why": c["why"]})
    # recorded files that stay where they are can still be copied from
    if p.state is not None:
        for path, r in sorted(p.state["files"].items()):
            if m.file(path) is not None:
                continue
            o = ob.get(r.get("n") or path)
            if o is not None and _same_stat(r, o.size, o.mtime_ns) and r.get("b") is None \
                    and not any(akey(k.name) == akey(o.name) for k in known):
                known.append(_Known(o.name, path, r["h"], False))
    known.sort(key=lambda k: (not k.leaving, pkey(k.path)))
    first = {}
    for i, k in enumerate(known):
        first.setdefault(k.h, i)

    # 3. sources
    uses = collections.OrderedDict()
    parts_of = []
    for ni, (f, sha, _target, _x) in enumerate(needs):
        if f["s"] == 0:
            parts_of.append([])
            continue
        if p.staged.has(sha, f["s"]):
            parts_of.append([("staged",)])
            continue
        ki = first.get(sha)
        if ki is not None:
            if known[ki].leaving:
                uses.setdefault(ki, []).append(ni)
            parts_of.append([("local_file", known[ki].name)])
            continue
        if len(f["o"]) > 1:
            parts = []
            for j, n in enumerate(f["o"]):
                if p.staged.has_part(sha, f["s"], j):
                    parts.append(("staged",))
                else:
                    parts.append(_slice_source(m, p.old, known, m.objects[n][5]) or ("fetch", n))
            parts_of.append(parts)
        else:
            parts_of.append([("fetch", f["o"][0])])
    moved_by = dict((users[-1], ki) for ki, users in uses.items())
    moved = set()
    for ni, ((f, sha, target, x), parts) in enumerate(zip(needs, parts_of)):
        if ni in moved_by:
            k = known[moved_by[ni]]
            moved.add(akey(k.name))
            plan.moves.append({"from": k.name, "to": f["p"], "sha": sha, "size": f["s"], "target_name": target})
            continue
        plan.builds.append({"path": f["p"], "target_name": target, "sha": sha, "size": f["s"], "parts": parts,
                            "x": x, "objects": list(f["o"])})
    plan.removes = sorted((r for r in removes if akey(r["path"]) not in moved), key=lambda r: pkey(r["path"]))

    # every folder between a pruned file and its retired map's folder, that folder
    # included (the switch removes only those that end up empty)
    rmd = {}
    for d in map_dirs:
        for r in plan.removes:
            if r["why"] != "prune" or not _inside(r["path"], d):
                continue
            at = r["path"]
            while "/" in at:
                parent = at[:at.rfind("/")]
                if len(parent) < len(d):
                    break
                rmd.setdefault(akey(parent), parent)
                at = parent
    plan.rmdirs = [rmd[k] for k in sorted(rmd)]

    # 4. the loader, when it is not ready
    loader_fetch = []
    if opts.get("loader_needed"):
        plan.loader = True
        lsha, lsize = m.fsha(m.loader), m.loader["s"]
        if p.staged.has(lsha, lsize):
            plan.loader_parts = [("staged",)]
        else:
            plan.loader_parts = [("fetch", n) for n in m.loader["o"]]
            loader_fetch = list(m.loader["o"])

    # 5. .001 is touched when a whole content.odgpak could win over the pieces (8.10)
    whole = ob.get(PIECE_BASE)
    first_piece = PIECE_BASE + ".001"
    if whole is not None and m.file(first_piece) is not None:
        placed = any(akey(b["path"]) == akey(first_piece) for b in plan.builds) \
            or any(akey(mv["to"]) == akey(first_piece) for mv in plan.moves)
        kept = next((r for path, r in plan.keep if akey(path) == akey(first_piece)), None)
        plan.touch_001 = placed or (kept is not None and whole.mtime_ns // 100 > kept["t"] // 100)

    # 6. what to fetch, and what it costs
    fetch = set(loader_fetch)
    for b in plan.builds:
        for part in b["parts"]:
            if part[0] == "fetch":
                fetch.add(part[1])
        if not b["parts"] or not all(part[0] == "staged" for part in b["parts"]):
            plan.stage_bytes += b["size"]
    if loader_fetch:
        plan.stage_bytes += m.loader["s"]
    plan.fetch_objects = sorted(fetch)
    plan.download_bytes = sum(m.objects[n][2] for n in fetch)
    return plan


def _part_dict(p):
    if p[0] == "staged":
        return {"staged": True}
    if p[0] == "local_file":
        return {"local_file": p[1]}
    if p[0] == "local_slice":
        return {"local_slice": {"path": p[1], "offset": p[2]}}
    return {"fetch": p[1]}


def depot_plan_dict(plan):
    """The plan as the shared vectors write it (15.2): every list sorted by path,
    moves by `to` then `from`, fetch_objects ascending."""
    return {
        "tag": plan.tag, "leaving": plan.leaving, "kept_newer": plan.kept_newer,
        "keep": sorted((k for k, _r in plan.keep), key=pkey),
        "builds": [{"path": b["path"], "target_name": b["target_name"], "sha": b["sha"],
                    "parts": [_part_dict(p) for p in b["parts"]]}
                   for b in sorted(plan.builds, key=lambda b: pkey(b["path"]))],
        "moves": [{"from": mv["from"], "to": mv["to"]}
                  for mv in sorted(plan.moves, key=lambda mv: (pkey(mv["to"]), pkey(mv["from"])))],
        "backups": [{"path": b["path"], "to": b["to"]} for b in sorted(plan.backups, key=lambda b: pkey(b["path"]))],
        "removes": [{"path": r["path"], "why": r["why"]} for r in sorted(plan.removes, key=lambda r: pkey(r["path"]))],
        "rmdirs": sorted(plan.rmdirs, key=pkey),
        "loader": plan.loader, "touch_001": plan.touch_001,
        "fetch_objects": sorted(plan.fetch_objects),
        "download_bytes": plan.download_bytes, "stage_bytes": plan.stage_bytes}


def depot_plan_text(plan, m, source):
    """last-plan.txt (4.4): what this run decided, for support and crash reports."""
    pack_of = {}
    for n in plan.fetch_objects:
        p = m.packs[m.objects[n][0]]
        pack_of[n] = "{}/{}".format(m.stores[p["store"]]["tag"], p["name"])
    lines = ["ODG for Linux {}: plan for {} (file list {})".format(TOOL_VERSION, plan.tag, m.sha),
             source,
             "leaving the installed build: {}".format("yes" if plan.leaving else "no"),
             "keep {}, build {}, move {}, back up {}, remove {}, folders {}, loader {}".format(
                 len(plan.keep), len(plan.builds), len(plan.moves), len(plan.backups), len(plan.removes),
                 len(plan.rmdirs), "yes" if plan.loader else "no"),
             "download {:,} B in {} objects, stage {:,} B".format(plan.download_bytes, len(plan.fetch_objects),
                                                                 plan.stage_bytes),
             ""]
    acts = []
    for b in sorted(plan.builds, key=lambda b: pkey(b["path"])):
        fetched = [p[1] for p in b["parts"] if p[0] == "fetch"]
        if fetched:
            acts.append("FETCH {} {:,} B from {}".format(b["path"], sum(m.objects[n][2] for n in fetched),
                                                         pack_of.get(fetched[0], "?")))
        elif b["parts"] and b["parts"][0][0] == "local_file":
            acts.append("COPY {} -> {}".format(b["parts"][0][1], b["path"]))
        elif b["parts"] and b["parts"][0][0] == "local_slice":
            acts.append("SLICE {}".format(b["path"]))
        else:
            acts.append("STAGED {}".format(b["path"]))
    for mv in sorted(plan.moves, key=lambda mv: pkey(mv["to"])):
        acts.append("MOVE {} -> {}".format(mv["from"], mv["to"]))
    for k, r in sorted(plan.keep, key=lambda kr: pkey(kr[0])):
        if r.get("b") is not None and r["h"] != r["b"]:
            acts.append("KEEP-LOCAL {}".format(k))
    for b in plan.backups:
        acts.append("BACKUP {} -> {}".format(b["path"], b["to"]))
    for r in sorted(plan.removes, key=lambda r: pkey(r["path"])):
        acts.append("REMOVE {} ({})".format(r["path"], r["why"]))
    for d in sorted(plan.rmdirs, key=pkey):
        acts.append("RMDIR {}".format(d))
    if plan.touch_001:
        acts.append("TOUCH {}.001".format(PIECE_BASE))
    if len(acts) > 2000:
        acts = acts[:2000] + ["... and {} more".format(len(acts) - 2000)]
    return "\n".join(lines + acts) + "\n"


# ---- room on the drive (6.9) ---------------------------------------------------------

def depot_alloc_unit(path):
    """The drive's allocation unit: every file takes whole units. His exFAT D: has
    1 MiB clusters, where a public fresh install of 11.2 GB takes 21.25 GB (I3 open
    issue 2). statvfs's f_frsize, else f_bsize, else 4096 (Windows has no statvfs)."""
    if depot_test_mode() and os.environ.get("ODG_TEST_ALLOC_UNIT", "").isdigit():
        return max(int(os.environ["ODG_TEST_ALLOC_UNIT"]), 1)
    try:
        st = os.statvfs(str(path))
    except (AttributeError, OSError):
        return 4096
    for unit in (getattr(st, "f_frsize", 0), getattr(st, "f_bsize", 0)):
        if isinstance(unit, int) and unit > 0:
            return unit
    return 4096


def depot_free_bytes(path):
    """Free bytes on the drive with `path`, or None when that cannot be told."""
    if depot_test_mode() and os.environ.get("ODG_TEST_FREE_BYTES", "").isdigit():
        return int(os.environ["ODG_TEST_FREE_BYTES"])
    try:
        return shutil.disk_usage(str(path)).free
    except OSError:
        return None


def depot_space_need(plan, m, unit):
    """6.9: every file this run builds in stage/, each rounded up to whole allocation
    units (a multi-part file at its full size: its .part is made that size), plus 4
    ranges of 32 MiB in flight and a margin of 256 MiB. The files and the sum the
    plan reports (stage_bytes) are the same; only the rounding is added here."""
    def on_disk(n):
        return -(-n // unit) * unit

    # a hash placed at several paths takes one more file for each path but the first:
    # the switch copies those (20.2), as the launcher counts them
    need, seen = 0, set()
    for b in plan.builds:
        first = b["sha"] not in seen
        seen.add(b["sha"])
        if first and b["parts"] and all(part[0] == "staged" for part in b["parts"]):
            continue
        need += on_disk(b["size"])
    if plan.loader and any(part[0] == "fetch" for part in plan.loader_parts):
        need += on_disk(m.loader["s"])
    return need + 4 * RANGE_CAP + (256 << 20)


# ---- downloads (6.10, 11.3) -----------------------------------------------------------

LOCKED = "Another ODG tool is updating this game folder"
STOPPED = "The last update stopped while installing. Run this again to finish it."


class _Fatal(Exception):
    """A download or staging failure that ends this run. Nothing is switched and
    what is staged stays for the next try."""


class _SwitchStop(Exception):
    """The switch stopped at an op. The journal stays, and the next run finishes it."""

    def __init__(self, why, path=""):
        Exception.__init__(self, why)
        self.why = why
        self.path = path


class _Range(object):
    """One closed range bytes=start-end of one pack, the objects inside it, and the
    range files it arrived in (the main file, then continuations)."""

    def __init__(self, rid, pack, start, end, objects):
        self.id, self.pack, self.start, self.end, self.objects = rid, pack, start, end, objects
        self.files = []

    def __len__(self):
        return self.end + 1 - self.start


def depot_coalesce(m, objects, gap=RANGE_GAP, cap=RANGE_CAP, first_id=0):
    """The ranges for `objects` (fetch::coalesce): by pack, then offset; neighbours
    merge while the gap is at most `gap` and the range at most `cap`; a bigger
    object gets a range of its own. Only single closed ranges are ever asked
    for: GitHub's CDN answers 501 to anything else."""
    objs, seen = [], set()
    for n in objects:
        if n in seen or n >= len(m.objects):
            continue
        seen.add(n)
        pack, off, clen = m.objects[n][:3]
        if clen > 0:
            objs.append((pack, off, clen, n))
    objs.sort()
    out = []
    for pack, off, clen, n in objs:
        end = off + clen - 1
        if out:
            r = out[-1]
            merged = max(end, r.end)
            if r.pack == pack and off <= r.end + 1 + gap and merged + 1 - r.start <= cap:
                r.end = merged
                r.objects.append(n)
                continue
        out.append(_Range(first_id + len(out), pack, off, end, [n]))
    return out


class _NoRedirect(urllib.request.HTTPRedirectHandler):
    def redirect_request(self, req, fp, code, msg, headers, newurl):
        return None


def _location(url, key=None, headers=None, data=None, method=None, timeout=30):
    """One request that does not follow a redirect: (status, Location, body).
    A key never travels further than the host it was meant for."""
    opener = urllib.request.build_opener(_NoRedirect)
    try:
        r = opener.open(_request(url, key, headers, data, method), timeout=timeout)
    except urllib.error.HTTPError as e:
        loc = e.headers.get("Location") if e.headers is not None else None
        try:
            body = e.read(1 << 20)
        except (OSError, http.client.HTTPException):
            body = b""
        finally:
            e.close()
        return e.code, loc, body
    try:
        return r.status, None, r.read(4 << 20)
    finally:
        r.close()


def _checked_address(url, what):
    """A signed address from github.com or the account server, used only when it is
    https (or http to a stand-in on this PC) with nothing in it that could carry a
    second header line."""
    try:
        u = urllib.parse.urlsplit(url)
        u.port  # a port that is not a number raises here, not in the middle of a download
        ok = not any(ord(c) <= 32 or ord(c) == 127 for c in url) and (
            (u.scheme == "https" and bool(u.hostname)) or (u.scheme == "http" and u.hostname in ("127.0.0.1", "localhost")))
    except (TypeError, ValueError, AttributeError):
        ok = False
    if not ok:
        raise _Fatal("the address given for {} is not one this tool downloads from".format(what))
    return url


class _Concat(object):
    """A range's files read as one stream: the main file, then its continuations."""

    def __init__(self, paths):
        self.parts = [(p, os.path.getsize(str(p))) for p in paths]
        self.pos = 0
        self.fh = None
        self.fh_i = -1

    def seek(self, pos):
        self.pos = pos

    def read(self, n):
        out = bytearray()
        start = 0
        for i, (p, size) in enumerate(self.parts):
            if n <= 0:
                break
            if self.pos < start + size:
                if self.fh_i != i:
                    self.close()
                    self.fh = open(str(p), "rb")
                    self.fh_i = i
                self.fh.seek(self.pos - start)
                chunk = self.fh.read(min(n, start + size - self.pos))
                out += chunk
                self.pos += len(chunk)
                n -= len(chunk)
            start += size
        return bytes(out)

    def close(self):
        if self.fh is not None:
            self.fh.close()
            self.fh = None
            self.fh_i = -1


def _decode(src, o, write):
    """Decode object `o` (stored, or one raw DEFLATE stream) from `src`, which stands
    at its first encoded byte, into `write`, hashing what is written. None when
    the bytes are the object's; else why not. The output stops the moment it
    would pass rawlen, so a hostile stream cannot fill the disk (codec.rs)."""
    _pack, _off, clen, rawlen, codec, sha = o
    if rawlen > PART_SIZE or (codec == 0 and clen != rawlen):
        return "the file list gives the object impossible sizes"
    h = hashlib.sha256()
    written = 0
    left = clen
    d = zlib.decompressobj(-15) if codec == 1 else None
    while left > 0:
        chunk = src.read(min(left, 1 << 20))
        if not chunk:
            return "the object stops after {} of {} bytes".format(clen - left, clen)
        left -= len(chunk)
        if d is None:
            out = chunk
        elif d.eof:
            continue        # what is left of clen after the stream's end is not part of it
        else:
            try:
                out = d.decompress(chunk, rawlen - written + 1)
            except zlib.error as e:
                return "the object's compressed bytes are damaged ({})".format(e)
        if written + len(out) > rawlen:
            return "the object decodes to more bytes than the file list says"
        if out:
            h.update(out)
            write(out)
            written += len(out)
    if d is not None and not d.eof:
        return "the object's compressed bytes end before their last block"
    if written != rawlen:
        return "the object holds {} of {} bytes".format(written, rawlen)
    if h.hexdigest() != sha:
        return "the object's bytes do not match the file list"
    return None


def _is_link(p):
    """A symlink, or on Windows a junction (a reparse point): rmdir never follows nor
    removes one."""
    try:
        st = os.lstat(str(p))
    except OSError:
        return False
    return stat.S_ISLNK(st.st_mode) or bool(getattr(st, "st_file_attributes", 0) & 0x400)


def _is_busy(e):
    return isinstance(e, PermissionError) or getattr(e, "errno", None) in (errno.EBUSY, errno.ETXTBSY,
                                                                            errno.EACCES, errno.EPERM)


def _sha_file(p):
    try:
        return sha256_file(p)
    except OSError:
        return ZERO_SHA


def _copy_over(src, dest):
    """Copy src onto dest through dest.odg-new, so dest is never half written."""
    tmp = Path(str(dest) + ".odg-new")
    with open(str(src), "rb") as fi, open(str(tmp), "wb") as fo:
        shutil.copyfileobj(fi, fo, 1 << 20)
        fo.flush()
        os.fsync(fo.fileno())
    os.replace(str(tmp), str(dest))


def _writable(p):
    """Clear a read-only bit on a file about to be replaced (Windows refuses otherwise)."""
    try:
        if p.is_file() and not os.access(str(p), os.W_OK):
            os.chmod(str(p), stat.S_IMODE(os.stat(str(p)).st_mode) | stat.S_IWUSR)
    except OSError:
        pass


class _Fetch(object):
    """The ranges of one run: up to 4 threads (1 at 1 MB/s while OPERATOR runs),
    each with its own keep-alive connection per host. A public pack's signed
    address is resolved once through github.com and used for 40 minutes; a dev
    pack's comes from the account server's /dev/sign (A1: from /dev/asset when
    that answers 401, 404 or 405). A 403 from storage resolves or signs that
    pack again once. A short answer queues only the rest as a continuation.
    At full speed the game is looked for every second: once OPERATOR starts,
    one thread carries on at 1 MB/s, the others hand back what is left of their
    ranges and stop (his rule: no heavy disk work while he plays)."""

    def __init__(self, depot, m, ranges, workers, rate, total):
        self.depot = depot
        self.m = m
        self.workers = max(1, workers)
        self.allowed = self.workers
        self.rate = rate
        self.running_rate = RUNNING_RATE
        self.game_started = False
        self.stop = threading.Event()
        self.cv = threading.Condition()
        self.todo = collections.deque({"r": r, "cont": 0, "start": r.start, "tries": 0, "bad": 0, "forbid": 0}
                                      for r in ranges)
        self.active = 0
        self.fatal = None
        self.next_id = max([r.id for r in ranges] + [-1]) + 1
        self.redone = set()
        self.where = {}
        self.url_lock = threading.Lock()
        self.fallback = False
        self.need_dev = sorted(set(r.pack for r in ranges if m.store_of(r.pack)["access"] != "public"))
        self.ctx = None
        self.got = 0
        self.t0 = time.monotonic()
        # the rate counts from here and from this many bytes (moved when the game starts)
        self.rate_t0 = self.t0
        self.rate_base = 0
        self.bar = Bar("downloading", total)
        self.said_slow = 0.0

    def run(self):
        if not self.todo:
            return
        watch = None
        if not self.rate:
            watch = threading.Thread(target=self._watch, name="odg-game-watch")
            watch.daemon = True
            watch.start()
        try:
            with concurrent.futures.ThreadPoolExecutor(self.workers) as ex:
                futs = [ex.submit(self._worker, wi) for wi in range(self.workers)]
                for f in futs:
                    f.result()
        finally:
            self.stop.set()
            if watch is not None:
                watch.join()
        self.bar.done()
        if self.fatal is not None:
            raise self.fatal

    # -- OPERATOR starting part way --

    def _watch(self):
        """game_running() (the /proc scan) every GAME_POLL_SECS while the download runs."""
        while not self.stop.wait(GAME_POLL_SECS):
            try:
                running = game_running()
            except Exception:       # a /proc that cannot be read never stops a download
                running = False
            if running:
                self._throttle()
                return

    def _throttle(self):
        """One thread at 1 MB/s from now on: the others stop at their next 64 KiB and
        hand back the rest of their ranges, keeping every byte they received."""
        with self.cv:
            if self.game_started:
                return
            self.game_started = True
            self.allowed = 1
            self.rate = self.running_rate
            self.rate_t0 = time.monotonic()
            self.rate_base = self.got
            self.cv.notify_all()
        if TTY:
            sys.stdout.write("\n")
        say("  OPERATOR started, so the rest of the update downloads slowly and installs once it is closed.")

    # -- scheduling --

    def _push(self, item):
        with self.cv:
            self.todo.append(item)
            self.cv.notify_all()

    def _worker(self, wi=0):
        conns = {}
        try:
            while True:
                with self.cv:
                    while not self.todo and self.active and self.fatal is None and wi < self.allowed:
                        self.cv.wait(0.5)
                    if self.fatal is not None or not self.todo or wi >= self.allowed:
                        self.cv.notify_all()
                        return
                    item = self.todo.popleft()
                    self.active += 1
                try:
                    self._one(item, conns, wi)
                except _Fatal as e:
                    with self.cv:
                        self.fatal = self.fatal or e
                except Exception as e:      # a bug must stop the run, never hang the other threads
                    with self.cv:
                        self.fatal = self.fatal or _Fatal("the download stopped: {}: {}".format(type(e).__name__, e))
                finally:
                    with self.cv:
                        self.active -= 1
                        self.cv.notify_all()
        finally:
            for c in conns.values():
                c.close()

    def _pause(self, tries):
        time.sleep(_test_pause(RANGE_PAUSES[min(tries, len(RANGE_PAUSES)) - 1]))

    def _slow(self):
        now = time.monotonic()
        with self.cv:
            if now - self.said_slow < 30:
                return
            self.said_slow = now
        if TTY:
            sys.stdout.write("\n")
        say("  GitHub is slow, still trying.")

    def _one(self, item, conns, wi=0):
        r = item["r"]
        name = "r{:06d}.bin".format(r.id) if item["cont"] == 0 else "r{:06d}.c{}.bin".format(r.id, item["cont"])
        path = self.depot.dl_dir / name
        url = self._url(r.pack)
        status, got = self._get(conns, url, item["start"], r.end, path, wi)
        if status == _HANDED_BACK:
            # OPERATOR started and this thread stops: what it received is kept and the
            # rest of the range waits for the one thread that carries on (not a retry)
            if got:
                r.files.append(path)
            else:
                remove_quietly(path)
            self._push(dict(item, cont=item["cont"] + (1 if got else 0), start=item["start"] + got))
            return
        if status == 206:
            if got:
                r.files.append(path)
            else:
                remove_quietly(path)
            if item["start"] + got > r.end:
                self._complete(r)
                return
            tries = 0 if got else item["tries"] + 1
            if tries >= RANGE_TRIES:
                raise _Fatal("a download of {} keeps stopping".format(self._pack_name(r.pack)))
            if tries:
                self._pause(tries)
            self._push(dict(item, cont=item["cont"] + (1 if got else 0), start=item["start"] + got, tries=tries))
            return
        remove_quietly(path)
        if status == 403:
            if item["forbid"]:
                raise _Fatal("GitHub refused {} (HTTP 403)".format(self._pack_name(r.pack)))
            self._url(r.pack, refresh=True)
            self._push(dict(item, forbid=1))
            return
        if status == 404:
            raise _Fatal("{} is missing from GitHub (HTTP 404). The release and its store disagree".format(
                self._pack_name(r.pack)))
        if status in (200, 416, 501, -1):
            if item["bad"]:
                # the pack and the offset, so a report says which bytes GitHub got wrong
                raise _Fatal("{} answered a byte range wrongly (HTTP {}), at byte {}".format(
                    self._pack_name(r.pack), status, item["start"]))
            self._push(dict(item, bad=1))
            return
        if status in (0, 408, 429) or 500 <= status <= 599:
            tries = item["tries"] + 1
            if tries >= RANGE_TRIES:
                raise _Fatal("GitHub kept failing ({})".format("no answer" if status == 0 else "HTTP {}".format(status)))
            self._slow()
            self._pause(tries)
            self._push(dict(item, tries=tries))
            return
        raise _Fatal("{} answered HTTP {}".format(self._pack_name(r.pack), status))

    def _complete(self, r):
        try:
            failed = self.depot.stage(r, r.files)
        finally:
            for p in r.files:
                remove_quietly(p)
            r.files = []
        if not failed:
            return
        with self.cv:
            twice = [n for n in failed if n in self.redone]
            self.redone.update(failed)
            first = self.next_id
            again = depot_coalesce(self.m, failed, first_id=first)
            self.next_id += len(again)
        if twice:
            raise _Fatal("object {} of {} is damaged on GitHub (it failed its check twice)".format(
                twice[0], self._pack_name(self.m.objects[twice[0]][0])))
        for nr in again:
            self._push({"r": nr, "cont": 0, "start": nr.start, "tries": 0, "bad": 0, "forbid": 0})

    def _pack_name(self, pack):
        return self.m.packs[pack]["name"] if pack < len(self.m.packs) else "pack {}".format(pack)

    # -- addresses --

    def _url(self, pack, refresh=False):
        with self.url_lock:
            w = self.where.get(pack)
            if w is not None and not refresh and time.time() < w[1]:
                return w[0]
            if self.m.store_of(pack)["access"] == "public":
                self.where[pack] = [self._resolve_public(pack), time.time() + URL_REUSE_SECS]
            else:
                now = time.time()
                packs = [pack] if refresh else [p for p in self.need_dev
                                                if p == pack or p not in self.where or now >= self.where[p][1]]
                self._sign(packs)
            return self.where[pack][0]

    def _retrying(self, what, call):
        """A request to github.com or the account server, tried 5 times when the
        answer is no answer or a server error."""
        for tries in range(1, RANGE_TRIES + 1):
            try:
                status, loc, body = call()
            except (urllib.error.URLError, OSError, http.client.HTTPException) as e:
                status, loc, body = 0, None, str(getattr(e, "reason", e)).encode("utf-8", "replace")
            if status in (0, 408, 429) or 500 <= status <= 599:
                if tries < RANGE_TRIES:
                    self._slow()
                    time.sleep(_test_pause(RANGE_PAUSES[min(tries, len(RANGE_PAUSES)) - 1]))
                    continue
                raise _Fatal("{}: {}".format(what, "no answer" if status == 0 else "HTTP {}".format(status)))
            return status, loc, body
        raise _Fatal(what)

    def _resolve_public(self, pack):
        p = self.m.packs[pack]
        s = self.m.stores[p["store"]]
        url = "{}/{}/releases/download/{}/{}".format(github_web(), s["repo"], urllib.parse.quote(s["tag"], safe=""),
                                                     urllib.parse.quote(p["name"], safe=""))
        status, loc, _body = self._retrying("github.com did not give the address of " + p["name"],
                                            lambda: _location(url, headers={"Accept": "application/octet-stream"}))
        if status in (301, 302, 303, 307, 308) and loc:
            return _checked_address(urllib.parse.urljoin(url, loc), p["name"])
        if status == 404:
            raise _Fatal("{} is missing from GitHub (HTTP 404). The release and its store disagree".format(p["name"]))
        if status == 200:
            return url
        raise _Fatal("github.com answered HTTP {} for {}".format(status, p["name"]))

    def _sign(self, packs):
        """Signed addresses for dev packs: /dev/sign 64 at a time (10.2); before the
        server knows that route (A1: 401, 404 or 405), /dev/asset one by one."""
        up = self.depot.up
        by_id = {}
        for p in packs:
            by_id.setdefault(self.m.packs[p]["asset_id"], []).append(p)
        ids = sorted(by_id)
        tag = self.depot.tag
        if not self.fallback:
            for i in range(0, len(ids), SIGN_BATCH):
                batch = ids[i:i + SIGN_BATCH]
                headers = account_headers()
                headers["Content-Type"] = "application/json"
                body = json.dumps({"tag": tag, "ids": batch}).encode("utf-8")
                status, _loc, raw = self._retrying(
                    "the account server did not sign the download list",
                    lambda: _location(server() + "/api/launcher/dev/sign", key=up.key, headers=headers,
                                      data=body, method="POST"))
                if status in (401, 404, 405):
                    self.fallback = True
                    break
                doc = {}
                try:
                    doc = json.loads(raw.decode("utf-8"))
                except ValueError:
                    pass
                if status == 409:
                    raise _Fatal("your channel moved to {} while this was updating. Run this again".format(
                        (doc.get("tag") if isinstance(doc, dict) else "") or "another build"))
                if status != 200 or not isinstance(doc, dict) or not isinstance(doc.get("urls"), list):
                    why = doc.get("error") if isinstance(doc, dict) else ""
                    raise _Fatal("the account server refused the download list (HTTP {}{})".format(
                        status, ": " + str(why) if why else ""))
                got = set()
                for u in doc["urls"]:
                    if not isinstance(u, dict) or u.get("id") not in by_id or not isinstance(u.get("url"), str):
                        continue
                    # `expires` is on the server's clock: believed only when it is 11 to 60
                    # minutes ahead of this one; otherwise the shortest life a fresh answer
                    # can have (20 minutes), so a wrong clock neither loops nor overstays
                    now = time.time()
                    left = (u["expires"] if type(u.get("expires")) is int else 0) - now
                    if not 660 <= left <= 3600:
                        left = 1200
                    url = _checked_address(u["url"], self.m.packs[by_id[u["id"]][0]]["name"])
                    for p in by_id[u["id"]]:
                        self.where[p] = [url, now + left - RESIGN_BEFORE_SECS]
                    got.add(u["id"])
                if set(batch) - got:
                    raise _Fatal("the account server left files out of the download list")
            if not self.fallback:
                return
        for aid in ids:
            status, loc, body = self._retrying(
                "the account server did not give the address of a download",
                lambda: _location("{}/api/launcher/dev/asset?id={}".format(server(), aid), key=up.key,
                                  headers=account_headers()))
            if status not in (301, 302, 303, 307, 308) or not loc:
                if status == 403 and b"not part of the build" in (body or b""):
                    # a release whose dev packs sit in a store, met by a server without /dev/sign
                    raise _Fatal("this dev build needs the account server's update before it can be downloaded")
                raise _Fatal("the account server refused a file of this build (HTTP {})".format(status))
            url = _checked_address(urllib.parse.urljoin(server() + "/", loc), self.m.packs[by_id[aid][0]]["name"])
            for p in by_id[aid]:
                self.where[p] = [url, time.time() + URL_REUSE_SECS]

    # -- one range --

    def _conn(self, conns, u):
        k = (u.scheme, u.hostname, u.port)
        c = conns.get(k)
        if c is None:
            if u.scheme == "https":
                if self.ctx is None:
                    self.ctx = ssl.create_default_context()
                c = http.client.HTTPSConnection(u.hostname, u.port or 443, timeout=60, context=self.ctx)
            else:
                c = http.client.HTTPConnection(u.hostname, u.port or 80, timeout=60)
            conns[k] = c
        return k, c

    def _get(self, conns, url, start, end, path, wi=0):
        """GET bytes=start-end into `path`: (status, bytes received). 0 is no answer,
        -1 an answer for another range, _HANDED_BACK this thread stopping because
        OPERATOR started (the bytes received are in `path`). Never sends a key: the
        address is the key."""
        u = urllib.parse.urlsplit(url)
        k, c = self._conn(conns, u)
        target = (u.path or "/") + ("?" + u.query if u.query else "")
        try:
            c.request("GET", target, headers={"Range": "bytes={}-{}".format(start, end), "User-Agent": USER_AGENT,
                                              "Accept": "application/octet-stream"})
            resp = c.getresponse()
        except (OSError, http.client.HTTPException):
            c.close()
            conns.pop(k, None)
            return 0, 0
        if resp.status != 206:
            # 200 is the whole pack, possibly 300 MB: hang up instead of reading it
            if resp.status == 200 or resp.length is None or resp.length > (1 << 20):
                c.close()
                conns.pop(k, None)
            else:
                try:
                    resp.read()
                except (OSError, http.client.HTTPException):
                    c.close()
                    conns.pop(k, None)
            return resp.status, 0
        want = end - start + 1
        cr = re.match(r"bytes (\d+)-(\d+)/", resp.getheader("Content-Range") or "")
        if cr:
            if int(cr.group(1)) != start:
                c.close()
                conns.pop(k, None)
                return -1, 0
            want = min(want, int(cr.group(2)) - start + 1)
        got, broken, handed = 0, False, False
        try:
            f = open(str(path), "wb")
        except OSError as e:
            c.close()
            conns.pop(k, None)
            raise _Fatal("could not write the download: {}".format(e))
        with f:
            while got < want and not broken:
                if wi >= self.allowed:
                    handed = True
                    break
                # Small reads, so a line that drops loses at most 64 KiB of what arrived:
                # everything written here is kept and only the rest is asked for again.
                try:
                    chunk = resp.read(min(65536, want - got))
                except http.client.IncompleteRead as e:
                    chunk, broken = e.partial, True
                except (OSError, http.client.HTTPException):
                    chunk, broken = b"", True
                if not chunk:
                    broken = True
                    break
                try:
                    f.write(chunk)
                except OSError as e:
                    c.close()
                    conns.pop(k, None)
                    if getattr(e, "errno", None) == errno.ENOSPC:
                        raise _Fatal("the disk filled up during the download. Free some space; what is staged is kept")
                    raise _Fatal("could not write the download: {}".format(e))
                got += len(chunk)
                self._progress(len(chunk))
        if broken or handed or not resp.isclosed():
            c.close()
            conns.pop(k, None)
        return (_HANDED_BACK if handed else 206), got

    def _progress(self, n):
        with self.cv:
            self.got += n
            got = self.got
            self.bar.update(min(got, self.bar.total) if self.bar.total else got)
            rate, t0, base = self.rate, self.rate_t0, self.rate_base
        if rate:
            # every thread sleeps off the lead of all of them together: the rate is
            # the run's, not each connection's
            ahead = (got - base) / float(rate) - (time.monotonic() - t0)
            if ahead > 0:
                time.sleep(ahead)


# ---- the depot run ------------------------------------------------------------------------

def _dropped_journal_paths(ops):
    """The paths a dropped journal names (a DLL's recorded name too), each with the hash its place wrote."""
    out = []
    for op in ops:
        kind = op.get("op")
        if kind == "place":
            name = op.get("name") or op["path"]
            out.append((name, op["sha"]))
            if name != op["path"]:
                out.append((op["path"], None))
        elif kind in ("vacate", "remove", "touch"):
            out.append((op["path"], None))
    return out


class Depot(object):
    """The depot route of one install run (SPEC 6.2 to 6.16) for an Updater, and the
    recovery of a switch an earlier run left half done. Everything it says goes
    through the Updater (say and fail), like the legacy route."""

    def __init__(self, updater):
        self.up = updater
        self.inst = updater.inst
        self.game = Path(updater.inst.game)
        self.disk = updater.disk
        self.dir = depot_dir(self.game)
        self.stage_dir = self.dir / "stage"
        self.view = StageView(self.stage_dir)
        # the target loader's hash while it is assembled at stage/loader.zip (None when a
        # manifest file has the same bytes: then it stages in its shard like that file)
        self.loader_sha = None
        self.dl_dir = self.dir / "dl"
        self.lock = DepotLock(self.game)
        self.tag = ""
        self.ix = None
        self.ix_lock = threading.Lock()
        self.ix_dirty_at = 0.0
        self.dests = {}
        self.part_counts = {}
        self.man = None
        self.old_state = None
        self.loader_installed = False
        self.loader_error = None
        self._keep = None
        self.removed = {"takeback": 0, "prune": 0}
        self.addons = None

    def close(self):
        self.lock.release()

    # -- 6.2 --

    def recover(self):
        """Finish a switch an earlier run left half done, before the listing and
        without the network. "nothing", "finished:<tag>" or "pending:<why>"."""
        if not depot_switch_pending(self.game):
            return "nothing"
        if not self.lock.acquire():
            return "pending:{}.".format(LOCKED)
        if game_running():
            return "pending:The last update finishes installing when OPERATOR is closed."
        try:
            got = depot_load_journal_raw(self.game)
        except _Json as e:
            return "pending:The last update's journal could not be read ({}); the next update repairs the pack.".format(e)
        if got is None:
            return "nothing"
        j, digest = got
        jf = JournalFile(self.dir, digest)
        jf.carry_on(j)
        self.old_state = depot_load_state(self.game)
        self._keep = None
        try:
            self._run_journal(j, jf, replay=True)
        except _SwitchStop as e:
            return "pending:" + e.why
        return "finished:" + j["target"]["tag"]

    # -- 6.3 to 6.16 --

    def run(self, doc, assets, exact, acct):
        """uptodate, installed, kept, deferred, tooold or failed."""
        up = self.up
        self.tag = tag = str(doc.get("tag_name") or "")
        # known before anything returns, so tidy never offers to delete what the game has open
        up.running = game_running()
        a = asset_named(assets, MANIFEST_ASSET)
        digest = str(a.get("digest") or "").strip()
        hexd = digest[7:].strip().lower() if digest[:7].lower() == "sha256:" else ""
        if not HEX64.fullmatch(hexd):
            up.fail("The release's file list has no checksum, so nothing was changed.")
            return "failed"
        if not self.lock.acquire():
            up.fail("{}, so nothing was changed. Run this again once it is done.".format(LOCKED))
            return "failed"
        say("Reading the file list for {}".format(tag))
        try:
            gz = self.fetch_manifest(a, hexd)
        except (Problem, OSError) as e:
            up.fail("Could not read the file list for {}: {}".format(tag, e))
            return "failed"
        try:
            m = depot_parse_gz(gz)
            self.validate(m, tag, "broker" if up.src == "broker" else "public",
                          [x["id"] for x in assets if type(x.get("id")) is int])
        except DepotError as e:
            if e.code == "rule 1":
                try:
                    need = depot_peek_gz(gz)[1].get(CLIENT_KIND, "")
                except DepotError:
                    need = ""
                return self._too_old(need)
            up.fail("The release's file list is not valid ({}), so nothing was changed.".format(e))
            return "failed"
        need = depot_may_apply(m, CLIENT_KIND, TOOL_VERSION)
        if need is not None:
            return self._too_old(need)
        self.man = m

        # 6.3.3: a journal for another build is dropped; the full check below repairs what it left half done.
        # Once it is gone nothing else says the install is half switched: the state and the stamp still
        # name the old build. So neither the fast path nor "kept" may answer, adoption trusts nothing,
        # and the records of the paths the journal names are put right first (the launcher's F2 rule).
        half, touched = False, None
        try:
            j = depot_load_journal(self.game)
        except _Json:
            j = {"target": {"manifest": ""}, "ops": None}
        if j is not None and j["target"]["manifest"] != m.sha:
            half = True
            if j.get("ops") is None:
                say("Dropping an unreadable switch journal; no file's record is trusted.")
            else:
                say("Dropping the unfinished switch to {}.".format(j["target"].get("tag", "?")))
                touched = _dropped_journal_paths(j["ops"])
            JournalFile(self.dir).clear()
            j = None
        shutil.rmtree(str(self.dl_dir), ignore_errors=True)

        legacy = depot_read_legacy(self.disk)
        state = depot_load_state(self.game)
        if state is not None and half:
            if touched is None:
                state["files"] = {}
            else:
                self._reread_touched(state, touched)
        if state is not None and not depot_state_matches(state, legacy):
            state = None        # 4.1: a state the stamps do not back is adopted again
        self.old_state = state
        inst = {"state": state, "manifest": self._installed_manifest(state), "legacy": legacy}
        # a half-switched install goes to the listed build even when it is older
        opts = {"repair": False, "exact": bool(exact) or half, "loader_needed": not self._loader_ready()}
        have, ahead, leaving = _plan_frame(m, inst, opts)
        if ahead:
            say("Pack {} kept (ahead of the release, the release is {}).".format(have, tag))
            return "kept"
        if (j is None and not half and state is not None and state["manifest"] == m.sha
                and legacy["installed_release"] == tag and self._quick_ok(m)):
            say("The pack is up to date ({}).".format(tag))
            return "uptodate"

        running = up.running
        self._load_index(m)
        staged = Staged.from_index(self.ix)
        ob = self.observe(m, "full", state, legacy)
        trust, known = ({}, {}) if state is not None or half else self.adopt_trust(m, legacy, ob)
        names = depot_needs_hash(m, inst, ob, trust, opts, self._may_delete, staged)
        hashes = dict(known)
        hashes.update(self._hash(names, state is None))
        try:
            plan = depot_plan(m, inst, ob, hashes, trust, opts, self._may_delete, staged)
        except PlanError as e:
            up.fail("This update could not be planned ({}), so nothing was changed.".format(e))
            return "failed"
        source = "installed {}{}".format(have or "nothing", " (adopted from the legacy stamps)" if state is None else "")
        try:
            depot_write(self.dir / "last-plan.txt", depot_plan_text(plan, m, source).encode("utf-8"))
        except OSError:
            pass
        # what no plan needs goes from stage/ (the launcher's stage::sweep), before anything returns
        self._sweep_stage(plan, m)
        if plan.nothing_to_do():
            # Nothing to place: the receipts, the state (fresh stats) and the stamp go
            # through a three-op journal, so a crash part way is finished like any
            # switch. While OPERATOR runs nothing at all is written (the launcher's rule).
            if not running:
                try:
                    self.switch(plan, m, have)
                except _SwitchStop as e:
                    up.fail(e.why)
                    return "failed"
            say("The pack is up to date ({}).".format(tag))
            return "uptodate"
        count = len(plan.builds) + len(plan.moves)
        say("Update {}: {} file{}, {} to download".format(tag, count, "" if count == 1 else "s", mb(plan.download_bytes)))

        # 6.9: room for everything staged, in whole allocation units, the ranges in flight and a margin
        if plan.stage_bytes:
            where = self.dir if self.dir.is_dir() else self.game
            room = depot_space_need(plan, m, depot_alloc_unit(where))
            free = depot_free_bytes(where)
            if free is not None and free < room:
                up.fail("Not enough space on the drive with the game: {:.1f} GB needed, {:.1f} GB free.".format(
                    room / 1e9, free / 1e9))
                return "failed"
        if running:
            say("OPERATOR is running, so the update to {} downloads slowly now and installs once it is closed.".format(tag))
        try:
            started = self.download(plan, m, 1 if running else DEPOT_WORKERS, RUNNING_RATE if running else None)
        except _Fatal as e:
            up.fail("The update to {} stopped: {}. Nothing was changed; what arrived is kept.".format(tag, e))
            return "failed"
        # OPERATOR started while it downloaded: the rest came on one slow connection, and
        # the switch waits for the next run with the game closed, like a start while running
        if running or started or game_running():
            up.running = True
            up.fail("Ready: it installs the next time you run this with OPERATOR closed.")
            return "deferred"
        off_before = switched_off(self.disk)
        try:
            self.switch(plan, m, have)
        except _SwitchStop as e:
            up.fail(e.why)
            return "failed"
        self.disk.forget()
        reapply_off(self.disk, off_before)
        taken = self.removed["takeback"]
        if taken:
            if lineage(have) != lineage(tag):
                which = "test" if lineage(have) == "dev" else "public"
            else:
                which = "newer"
            say("Removed {} file{} only the {} build had.".format(taken, "" if taken == 1 else "s", which))
        pruned = self.removed["prune"]
        if pruned:
            say("Removed {} file{} of maps the release no longer ships.".format(pruned, "" if pruned == 1 else "s"))
        if self.loader_installed:
            say("Mod loader installed.")
        up.changed = True
        if self.loader_error:
            # the pack is complete and stamped; the next run plans the loader again
            up.fail("Pack {} installed, but the mod loader could not be installed ({}). Run this again to try "
                    "the loader again.".format(tag, self.loader_error))
            return "failed"
        say("Pack {} installed.".format(tag))
        return "installed"

    def _reread_touched(self, state, touched):
        """Put right the state records of the paths a dropped journal names, reading those files once:
        a file its place wrote is recorded with that hash (ours); a file still as its record says keeps
        the record with today's stat; anything else (missing, unreadable, edited) loses its record."""
        files = state["files"]
        by_key = {}
        for path, r in files.items():
            by_key[akey(path)] = path
            if r.get("n"):
                by_key[akey(r["n"])] = path
        for name, placed in touched:
            path = by_key.get(akey(name))
            got = None
            p = self.disk.find(name)
            if p is not None and p.is_file():
                try:
                    st = os.stat(str(p))
                    got = (_sha_file(p), st.st_size, st.st_mtime_ns)
                except OSError:
                    got = None
            if got is not None and got[0] == placed:
                key = path if path is not None else (name[:-len(OFF)] if name.endswith(OFF) else name)
                old = files.get(key) or {}
                rec = {"h": got[0], "s": got[1], "t": got[2]}
                if key != name:
                    rec["n"] = name
                if old.get("b"):
                    rec["b"] = old["b"]
                files[key] = rec
            elif got is not None and path is not None and files.get(path, {}).get("h") == got[0]:
                files[path]["s"], files[path]["t"] = got[1], got[2]
            elif path is not None:
                files.pop(path, None)

    def _too_old(self, need):
        self.up.fail("This update needs ODG for Linux {} or newer. The pack stays as it is; this tool updates "
                     "itself from the site as soon as that version is there.".format(need or "(a newer version)"))
        return "tooold"

    def fetch_manifest(self, a, hexd):
        """The file list from depot/manifests/<sha256>.json.gz when a copy with that
        hash is there, else downloaded and checked against the release's digest."""
        path = self.dir / "manifests" / (hexd + ".json.gz")
        try:
            data = path.read_bytes()
            if hashlib.sha256(data).hexdigest() == hexd:
                return data
        except OSError:
            pass
        path.parent.mkdir(parents=True, exist_ok=True)
        tmp = path.with_name(path.name + ".part")
        try:
            self.up.fetch(a, tmp, "file list")
        except Problem:
            remove_quietly(tmp)
            raise
        os.replace(str(tmp), str(path))
        return path.read_bytes()

    def validate(self, man, tag, source, listing_ids):
        depot_validate(man, tag, source, listing_ids, test=depot_test_mode())

    def _installed_manifest(self, state):
        if state is None or not HEX64.fullmatch(state["manifest"]):
            return None
        try:
            data = (self.dir / "manifests" / (state["manifest"] + ".json.gz")).read_bytes()
            if hashlib.sha256(data).hexdigest() != state["manifest"]:
                return None
            return depot_parse_gz(data)
        except (OSError, DepotError):
            return None

    def _loader_ready(self):
        return loader_state(self.disk)[0] in ("ready", "held")

    def _quick_ok(self, m):
        """6.4.5, the fast path of an unchanged release: every top-level DLL under
        either name, every piece, every map's manifest.bin and the two lists, with
        their sizes, from three listings. And the loader, which this tool checks
        on every run anyway."""
        rels = [f["p"] for f in m.files if is_top_dll(f["p"])] + m.piece_paths() + list(LIST_FILES[1:])
        rels += [mp["dir"] + "/manifest.bin" for mp in m.maps]
        ob = depot_observe(self.game, rels)
        for f in m.files:
            p = f["p"]
            if is_top_dll(p):
                on, off = ob.lookup(p)
                if not ((on is not None and on.size == f["s"]) or (off is not None and off.size == f["s"])):
                    return False
            elif p in rels and (ob.files.get(akey(p)) is None or ob.files[akey(p)].size != f["s"]):
                return False
        return self._loader_ready()

    def observe(self, m, scope="full", state=None, legacy=None):
        """6.5 for this game folder, plus the parts stage.json says are staged."""
        if scope == "quick":
            return depot_observe(self.game, [f["p"] for f in m.files if is_top_dll(f["p"])] + m.piece_paths())
        rels = [f["p"] for f in m.files] + [m.pak["base"]]
        maps = []
        if state is not None:
            rels += list(state["files"])
            rels += [r["n"] for r in state["files"].values() if r.get("n")]
            maps = list(state["maps"])
        elif legacy is not None:
            rels += [line for line in legacy["shipped"] if safe_rel(line)]
            maps = [n[len("mapdata_"):-len(".zip")] for n, _d in legacy["mapdata"]
                    if akey(n).startswith("mapdata_") and akey(n).endswith(".zip")]
        shipped = set(akey(mp["name"]) for mp in m.maps)
        deep = [MAPS_DIR + n for n in maps if map_name_shape(n) and akey(n) not in shipped]
        return depot_observe(self.game, rels, deep)

    def _local_seal(self, ob):
        """The seal id of the pieces NewSlots reads here (as the listing saw them),
        or None when it reads a whole file, nothing, or something that is not a pack."""
        pieces = depot_active_pieces(ob)
        if not pieces:
            return None
        try:
            return seal_id([self.game.joinpath(*s.name.split("/")) for s in pieces])
        except SealError:
            return None

    def adopt_trust(self, man, legacy, seen):
        """6.7: (trust of every manifest path, the piece hashes the local seal proves)."""
        seal = self._local_seal(seen)
        return depot_trust(man, legacy, seen, seal), depot_known_hashes(man, seen, seal)

    def plan(self, man, state, legacy, seen, hashes, trust, repair, exact, old=None, loader_needed=False):
        return depot_plan(man, {"state": state, "manifest": old, "legacy": legacy}, seen, hashes, trust,
                          {"repair": repair, "exact": exact, "loader_needed": loader_needed}, self._may_delete,
                          Staged.from_index(self.ix))

    def _hash(self, names, adoption):
        """SHA-256 of each name, read in order with one progress line. A file that
        cannot be read gets a hash no file has, so it is fetched, never trusted."""
        out = {}
        if not names:
            return out
        paths = [(n, self.game.joinpath(*n.split("/"))) for n in names]
        total = 0
        for _n, p in paths:
            try:
                total += os.path.getsize(str(p))
            except OSError:
                pass
        bar = Bar("checking your files, one time" if adoption else "checking your files", total)
        done = 0
        for n, p in paths:
            h = hashlib.sha256()
            try:
                with open(str(p), "rb") as f:
                    while True:
                        chunk = f.read(1 << 20)
                        if not chunk:
                            break
                        h.update(chunk)
                        done += len(chunk)
                        bar.update(done)
                out[akey(n)] = h.hexdigest()
            except OSError:
                out[akey(n)] = ZERO_SHA
        bar.done()
        return out

    def _may_delete(self, rel):
        """6.13: inside Mods, never the free mods, an add-on's files, the player's
        own maps, the add-on records, or anything reached through a link."""
        if not depot_may_delete_name(rel):
            return False
        if self.addons is None:
            self.addons = addon_files(self.disk)
        if rel.lower() in self.addons:
            return False
        p = self.disk.find(rel)
        mods = self.disk.find("Mods")
        if p is not None and (os.path.islink(str(p)) or (mods is not None and through_link(p.parent, mods))):
            return False
        return True

    # -- the stage folder (20.5) --

    def _where(self, sha, suffix=""):
        """(folder, name) in stage/ of a file being staged: the target loader at the
        top as loader.zip, everything else in its shard. The switch's ops never ask
        this: what they vacate and place is always in its shard (stage_where)."""
        if sha == self.loader_sha:
            return "", STAGE_LOADER + suffix
        return stage_where(sha, suffix)

    def _staged_ok(self, sha, size):
        """Staged whole with this size, by the stage's listings. loader.zip is not named
        by its hash, so it counts only while stage.json says it holds this one."""
        if self.view.size(*self._where(sha)) != size:
            return False
        if sha != self.loader_sha:
            return True
        e = (self.ix or {}).get("files", {}).get(sha)
        return bool(e and e.get("done") and e["s"] == size)

    def _set_loader(self, m):
        lsha = m.fsha(m.loader) if m.loader else None
        self.loader_sha = lsha if lsha and not any(m.fsha(f) == lsha for f in m.files) else None

    # -- stage.json --

    def _load_index(self, m):
        """stage.json, without what stage/ no longer backs: a whole file gone, of another
        size or older than 14 days; parts whose .part is gone, of another size or old.
        Each shard folder is read once (StageView), never one lookup per file."""
        self._set_loader(m)
        ix = None
        try:
            with open(str(self.dir / "stage.json"), "rb") as f:
                ix = _stage_index_from(json_strict(f.read()))
        except (OSError, _Json):
            ix = None
        if ix is None:
            ix = {"format": DEPOT_FORMAT, "target": m.sha, "files": {}}
        ix["target"] = m.sha
        now = time.time()
        for sha in list(ix["files"]):
            e = ix["files"][sha]
            if not e.get("done") and not e.get("parts"):
                del ix["files"][sha]
                continue
            where = self._where(sha, "" if e.get("done") else ".part")
            if self.view.size(*where) != e["s"] or self.view.age(where[0], where[1], now) > STAGE_MAX_AGE:
                # an old one goes in the sweep below; a plan never counts on it, so the
                # sweep cannot take away a file the plan was made to reuse
                del ix["files"][sha]
        self.ix = ix

    def _save_index(self, force=False):
        with self.ix_lock:
            if self.ix is None or (not force and time.monotonic() - self.ix_dirty_at < 2):
                return
            self.ix_dirty_at = time.monotonic()
            data = _stage_index_bytes(self.ix)
        try:
            depot_write(self.dir / "stage.json", data)
        except OSError:
            pass

    def _staged_done(self, sha, size):
        with self.ix_lock:
            self.ix["files"][sha] = {"s": size, "done": True, "parts": []}
        self._save_index()

    def _sweep_stage(self, plan, m):
        """4.2 and 20.5: what the plan builds stays (<h2>/<sha> and <h2>/<sha>.part in
        their own shard, and loader.zip while stage.json says it holds the planned
        loader); anything else in stage/ goes, and so does anything older than 14 days:
        flat files a build before 20.5 left at the top, temp files, a file in the wrong
        shard, other folders. Range files belong in dl/; one found at the top is left to
        the age rule. Unwanted names go without a stat; only kept ones are aged."""
        want = set(b["sha"] for b in plan.builds)
        lsha = m.fsha(m.loader) if plan.loader else None
        if lsha:
            want.add(lsha)
        v = self.view
        now = time.time()
        for name in sorted(v.entries("")):
            kind = v.kind("", name)
            if kind == "dir" and _SHARD_NAME.fullmatch(name):
                for n in sorted(v.entries(name)):
                    sha = n[:-5] if n.endswith(".part") else n
                    keep = (sha in want and sha != self.loader_sha and stage_where(sha)[0] == name
                            and v.kind(name, n) == "file" and v.age(name, n, now) <= STAGE_MAX_AGE)
                    if not keep:
                        v.delete(name, n)
                continue
            if name == STAGE_LOADER:
                e = self.ix["files"].get(lsha) if lsha and lsha == self.loader_sha else None
                keep = (e is not None and bool(e.get("done")) and v.size("", name) == e["s"]
                        and v.age("", name, now) <= STAGE_MAX_AGE)
            else:
                keep = kind == "file" and bool(_RANGE_NAME.fullmatch(name)) and v.age("", name, now) <= STAGE_MAX_AGE
            if not keep:
                v.delete("", name)
        with self.ix_lock:
            for sha in list(self.ix["files"]):
                if sha not in want:
                    del self.ix["files"][sha]
        self._save_index(force=True)

    # -- 6.11 --

    def download(self, plan, m, workers=DEPOT_WORKERS, rate=None):
        """6.10 and 6.11: every planned file complete and verified in stage/ before
        anything in Mods moves: local copies first (a copy that does not verify is
        fetched instead), then the ranges, each staged as it lands (stage). True when
        OPERATOR started while it downloaded: the rest came on one slow connection."""
        self.man = m
        self._set_loader(m)
        self.view.ensure("")
        self.dl_dir.mkdir(parents=True, exist_ok=True)
        self.dests = {}
        self.part_counts = {}
        jobs = [(b["sha"], b["size"], b["parts"], b["objects"]) for b in plan.builds]
        if plan.loader:
            jobs.append((m.fsha(m.loader), m.loader["s"], plan.loader_parts, list(m.loader["o"])))
        fetch = set()
        for sha, size, parts, objects in jobs:
            if self._staged_ok(sha, size):
                continue
            if size == 0:
                self._write_staged(sha, b"")
                continue
            if parts and parts[0][0] == "local_file":
                if self._copy_local(parts[0][1], sha, size):
                    continue
                parts = [("fetch", n) for n in objects]
            if len(objects) == 1:
                # whatever the plan said (its "staged" comes from stage.json), it is not
                # staged now, so it is fetched
                self.dests.setdefault(objects[0], [])
                if ("file", sha, size) not in self.dests[objects[0]]:
                    self.dests[objects[0]].append(("file", sha, size))
                fetch.add(objects[0])
                continue
            self.part_counts[sha] = len(objects)
            with self.ix_lock:
                e = self.ix["files"].get(sha)
                if e is not None and (e.get("done") or self.view.size(*self._where(sha, ".part")) != size):
                    # parts recorded for a .part that is gone or of another size are not there
                    del self.ix["files"][sha]
                have = set((self.ix["files"].get(sha) or {}).get("parts", ()))
            for j, n in enumerate(objects):
                part = parts[j] if j < len(parts) else ("fetch", n)
                if j in have:
                    continue
                if part[0] == "local_slice" and self._slice_local(part[1], part[2], m.objects[n], sha, size, j):
                    continue
                self.dests.setdefault(n, []).append(("part", sha, size, j))
                fetch.add(n)
            self._maybe_finish(sha, size)
        self._save_index(force=True)
        for n in sorted(fetch):
            if m.objects[n][2] == 0:        # an object of no bytes needs no download
                self._stage_object(_Concat([]), m.objects[n], n)
                fetch.discard(n)
        ranges = depot_coalesce(m, sorted(fetch))
        total = sum(len(r) for r in ranges)
        f = _Fetch(self, m, ranges, workers, rate, total)
        try:
            f.run()
        finally:
            self._save_index(force=True)
        # ready to switch: every job staged with its size, from the listings this run
        # already made and kept up to date (never a lookup per file)
        for sha, size, _parts, _objects in jobs:
            if not self._staged_ok(sha, size):
                raise _Fatal("a file of this update did not arrive whole")
        return f.game_started

    def _write_staged(self, sha, data):
        sub, name = self._where(sha)
        self.view.ensure(sub)
        depot_write(self.view.path(sub, name), data)
        self.view.added(sub, name, len(data))
        self._staged_done(sha, len(data))

    def _copy_local(self, name, sha, size):
        """6.11: a local copy, hashed while it is copied; False when it is not the bytes."""
        src = self.game.joinpath(*name.split("/"))
        sub, fin = self._where(sha)
        tmp = self.view.path(*self._where(sha, ".tmp"))
        h = hashlib.sha256()
        n = 0
        try:
            self.view.ensure(sub)
            with open(str(src), "rb") as fi, open(str(tmp), "wb") as fo:
                while True:
                    chunk = fi.read(1 << 20)
                    if not chunk:
                        break
                    h.update(chunk)
                    fo.write(chunk)
                    n += len(chunk)
                fo.flush()
                os.fsync(fo.fileno())
        except OSError as e:
            remove_quietly(tmp)
            if getattr(e, "errno", None) == errno.ENOSPC:
                raise _Fatal("the disk filled up. Free some space; what is staged is kept")
            return False
        if n != size or h.hexdigest() != sha:
            remove_quietly(tmp)
            return False
        os.replace(str(tmp), str(self.view.path(sub, fin)))
        self.view.added(sub, fin, size)
        self._staged_done(sha, size)
        return True

    def _part_file(self, sha, size):
        """The .part of a multi-part file at its full size, made (or sized) unless this
        run already knows it is there with that size."""
        sub, name = self._where(sha, ".part")
        pp = self.view.path(sub, name)
        with self.view.lock:
            have = self.view.size(sub, name)
            if have != size:
                self.view.ensure(sub)
                with open(str(pp), "r+b" if have is not None else "wb") as f:
                    f.truncate(size)
                self.view.added(sub, name, size)
        return pp

    def _slice_local(self, name, offset, o, sha, size, j):
        """A part read from a local file at `offset`, checked against the part's hash."""
        src = self.game.joinpath(*name.split("/"))
        try:
            with open(str(src), "rb") as fi:
                fi.seek(offset)
                data = fi.read(o[3])
        except OSError:
            return False
        if len(data) != o[3] or hashlib.sha256(data).hexdigest() != o[5]:
            return False
        try:
            with open(str(self._part_file(sha, size)), "r+b") as fo:
                fo.seek(j * PART_SIZE)
                fo.write(data)
                fo.flush()
                os.fsync(fo.fileno())
        except OSError as e:
            raise _Fatal("could not write a staged file: {}".format(e))
        self._part_done(sha, size, j)
        return True

    def _part_done(self, sha, size, j):
        with self.ix_lock:
            e = self.ix["files"].setdefault(sha, {"s": size, "done": False, "parts": []})
            if j not in e["parts"]:
                e["parts"].append(j)
        self._maybe_finish(sha, size)
        self._save_index()

    def _maybe_finish(self, sha, size):
        """A multi-part file whose every part is written and verified becomes
        stage/<h2>/<sha>; the parts tile the file, so it is not read again."""
        with self.ix_lock:
            e = self.ix["files"].get(sha)
            count = self.part_counts.get(sha)
            if e is None or e.get("done") or count is None or len(set(e["parts"])) < count:
                return
            sub, name = self._where(sha, ".part")
            if self.view.size(sub, name) is None:
                return
            fsub, fin = self._where(sha)
            os.replace(str(self.view.path(sub, name)), str(self.view.path(fsub, fin)))
            self.view.removed(sub, name)
            self.view.added(fsub, fin, size)
            self.ix["files"][sha] = {"s": size, "done": True, "parts": []}

    def stage(self, rng, files):
        """6.11 for one completed range: slice it into its objects (`files` are the
        range file and its continuations, read as one), decode and verify each into
        its staged file. Returns the objects that failed: only those are fetched
        again, whatever else shared their range."""
        m = self.man
        failed = []
        src = _Concat(files)
        try:
            for n in sorted(rng.objects, key=lambda n: m.objects[n][1]):
                o = m.objects[n]
                src.seek(o[1] - rng.start)
                if not self._stage_object(src, o, n):
                    failed.append(n)
        finally:
            src.close()
        return failed

    def _stage_object(self, src, o, n):
        with self.ix_lock:
            dests = list(self.dests.get(n) or [])
        if not dests:
            return True
        first = dests[0]
        try:
            ok = self._decode_into(src, o, first)
            if ok:
                for d in dests[1:]:
                    self._copy_dest(first, d, o)
        except OSError as e:
            if getattr(e, "errno", None) == errno.ENOSPC:
                raise _Fatal("the disk filled up. Free some space; what is staged is kept")
            raise _Fatal("could not write a staged file: {}".format(e))
        if ok:
            with self.ix_lock:
                self.dests.pop(n, None)
        return ok

    def _decode_into(self, src, o, dest):
        if dest[0] == "file":
            sha, size = dest[1], dest[2]
            sub, fin = self._where(sha)
            tmp = self.view.path(*self._where(sha, ".tmp"))
            self.view.ensure(sub)
            with open(str(tmp), "wb") as out:
                why = _decode(src, o, out.write)
                if why is None:
                    out.flush()
                    os.fsync(out.fileno())
            if why is not None:
                remove_quietly(tmp)
                return False
            os.replace(str(tmp), str(self.view.path(sub, fin)))
            self.view.added(sub, fin, size)
            self._staged_done(sha, size)
            return True
        _kind, sha, size, j = dest
        with open(str(self._part_file(sha, size)), "r+b") as out:
            out.seek(j * PART_SIZE)
            why = _decode(src, o, out.write)
            if why is None:
                out.flush()
                os.fsync(out.fileno())
        if why is not None:
            return False
        self._part_done(sha, size, j)
        return True

    def _copy_dest(self, first, d, o):
        """An object two destinations need: decoded once, copied from the first."""
        if first[0] == "file":
            where, off = self._where(first[1]), 0
        else:
            where, off = self._where(first[1], ".part"), first[3] * PART_SIZE
            if self.view.size(*where) is None:      # that part was its last: stage/<h2>/<sha> now
                where = self._where(first[1])
        with open(str(self.view.path(*where)), "rb") as fi:
            fi.seek(off)
            data = fi.read(o[3])
        if d[0] == "file":
            self._write_staged(d[1], data)
        else:
            with open(str(self._part_file(d[1], d[2])), "r+b") as fo:
                fo.seek(d[3] * PART_SIZE)
                fo.write(data)
                fo.flush()
                os.fsync(fo.fileno())
            self._part_done(d[1], d[2], d[3])

    # -- 6.12 --

    def _journal_ops(self, plan, m):
        """Every op of the switch, in 6.12's order (switch.rs journal_for). A DLL's
        name is left out: it is decided right before its rename."""
        ops = []
        seen = set()
        for mv in sorted(plan.moves, key=lambda mv: (pkey(mv["from"]), mv["from"])):
            if akey(mv["from"]) not in seen:
                seen.add(akey(mv["from"]))
                ops.append({"op": "vacate", "path": mv["from"], "sha": mv["sha"]})
        for b in plan.backups:
            ops.append({"op": "backup", "path": b["path"], "to": b["to"]})
        places = [(b["path"], b["sha"], b["size"]) for b in plan.builds]
        places += [(mv["to"], mv["sha"], mv["size"]) for mv in plan.moves]
        lists = dict((akey(p), i) for i, p in enumerate(LIST_FILES))
        base = akey(m.pak["base"]) + "."

        def piece_no(p):
            k = akey(p)
            if k.startswith(base) and len(k) == len(base) + 3 and _digits(k[len(base):]):
                return int(k[len(base):])
            return None

        data, pieces, dlls, last = [], [], [], []
        for p, sha, size in places:
            op = {"op": "place", "path": p, "sha": sha, "size": size, "dll": is_top_dll(p)}
            if akey(p) in lists:
                last.append((lists[akey(p)], op))
            elif piece_no(p) is not None:
                pieces.append((-piece_no(p), op))
            elif op["dll"]:
                dlls.append(((pkey(p), p), op))
            else:
                data.append(((pkey(p), p), op))
        for group in (data, pieces, dlls, last):
            ops.extend(op for _k, op in sorted(group, key=lambda x: x[0]))
        for why in ("takeback", "prune"):
            for r in sorted((r for r in plan.removes if r["why"] == why), key=lambda r: (pkey(r["path"]), r["path"])):
                ops.append({"op": "remove", "path": r["path"], "sha": r["expect"], "why": why})
        for d in sorted(plan.rmdirs, key=lambda d: (-d.count("/"), pkey(d))):
            ops.append({"op": "rmdir", "path": d})
        if plan.touch_001:
            ops.append({"op": "touch", "path": m.pak["base"] + ".001"})
        if plan.loader:
            ops.append({"op": "loader"})
        ops += [{"op": "receipts"}, {"op": "state"}, {"op": "stamp", "tag": m.tag}]
        return ops

    def switch(self, plan, m, have):
        """6.12: write the whole journal, then run every op. Nothing in Mods changed
        before this point."""
        if plan.loader and self.loader_sha != m.fsha(m.loader):
            # the loader has the bytes of a manifest file, so it was staged in that file's
            # shard; the loader op installs from loader.zip (normally staged right there)
            lsha = m.fsha(m.loader)
            src = self.view.path(*stage_where(lsha))
            z = self.view.path("", STAGE_LOADER)
            if any(b["sha"] == lsha for b in plan.builds) or any(mv["sha"] == lsha for mv in plan.moves):
                _copy_over(src, z)
            else:
                os.replace(str(src), str(z))
                self.view.removed(*stage_where(lsha))
            self.view.added("", STAGE_LOADER, m.loader["s"])
        j = {"format": DEPOT_FORMAT, "target": {"tag": m.tag, "manifest": m.sha, "lineage": m.lineage},
             "from": {"tag": have or "", "leaving": plan.leaving}, "ops": self._journal_ops(plan, m), "next": 0}
        # What the switch checks removals and finished places against, and what the
        # state op carries forward: the state as it is now (switch.rs `before`).
        self.old_state = depot_load_state(self.game)
        self._keep = dict(plan.keep)
        jf = JournalFile(self.dir)
        try:
            jf.write(j)
        except OSError as e:
            raise _SwitchStop("Could not write the update's journal ({}), so nothing was changed.".format(e))
        self._run_journal(j, jf, replay=False)

    def _run_journal(self, j, jf, replay):
        """Run the ops from `next`. Every op is idempotent, so a replay from the last
        flushed `next` ends in the target install whoever wrote the journal (6.12.4;
        switch.rs has the same rules, and the vectors README states them for both)."""
        ops = j["ops"]
        last_place = {}
        for i, op in enumerate(ops):
            if op["op"] == "place":
                last_place[op["sha"]] = i
        kill = -1
        if depot_test_mode() and os.environ.get("ODG_TEST_DEPOT_KILL_AT", "").isdigit():
            kill = int(os.environ["ODG_TEST_DEPOT_KILL_AT"])
        i = j["next"]
        done_here = 0
        dirs = set()
        flushed_i, flushed_t = i, time.monotonic()
        bar = Bar("installing " + j["target"]["tag"], len(ops)) if len(ops) - i > 200 else None
        while i < len(ops):
            op = ops[i]
            self._op(j, jf, i, op, replay, last_place, dirs)
            i += 1
            done_here += 1
            if done_here == kill:
                os._exit(9)     # the test of a crash mid-switch: nothing after this op is flushed
            if bar is not None:
                bar.update(i)
            # Flushed at once when the vacates and backups are done (a replay then starts
            # past them), after a file of 64 MiB or more (a replay re-reads at most one),
            # and at the end; else every 64 ops or 0.5 s. A flush that fails is skipped:
            # progress is a lower bound, and a replay proves every op again.
            boundary = op["op"] in ("vacate", "backup") and (i == len(ops) or ops[i]["op"] not in ("vacate", "backup"))
            big = op["op"] == "place" and op["size"] >= PART_SIZE
            if (boundary or big or i == len(ops) or i - flushed_i >= JOURNAL_FLUSH_OPS
                    or time.monotonic() - flushed_t >= JOURNAL_FLUSH_SECS):
                for d in dirs:
                    _fsync_dir(d)
                dirs.clear()
                try:
                    jf.set_next(j, i)
                except OSError:
                    pass
                flushed_i, flushed_t = i, time.monotonic()
        if bar is not None:
            bar.done()
        jf.clear()
        _fsync_dir(self.dir)
        v = self.view
        remove_quietly(v.path("", STAGE_LOADER))
        v.removed("", STAGE_LOADER)
        # what this switch placed by copy is no longer needed in stage/ (switch.rs
        # sweep_consumed), nor a .part beside a finished file; each shard is read once
        for sha in sorted(set(op["sha"] for op in ops if op["op"] == "place")):
            for suffix in ("", ".part"):
                sub, name = stage_where(sha, suffix)
                if v.kind(sub, name) == "file":
                    v.delete(sub, name)
        if self.ix is not None:
            with self.ix_lock:
                for sha in list(self.ix["files"]):
                    e = self.ix["files"][sha]
                    if v.size(*self._where(sha, "" if e.get("done") else ".part")) is None:
                        del self.ix["files"][sha]
            self._save_index(force=True)
        # 20.5: a shard folder this switch left empty goes
        v.drop_empty_shards()

    def _retry(self, fn, path):
        """6.12.3: a file an antivirus or another program holds is tried 10 times over
        about 5 s; then the switch stops where it is and the next run finishes it."""
        for tries in range(SWITCH_TRIES):
            try:
                return fn()
            except OSError as e:
                if not _is_busy(e) or tries + 1 == SWITCH_TRIES:
                    if _is_busy(e):
                        raise _SwitchStop("{} is in use by another program. Close it and run this again; "
                                          "the update finishes then.".format(path), path)
                    raise _SwitchStop("Could not write {} ({}). Run this again to finish the update.".format(path, e),
                                      path)
                time.sleep(SWITCH_PAUSE)

    def _op(self, j, jf, i, op, replay, last_place, dirs):
        kind = op["op"]
        if kind == "vacate":
            self._op_vacate(j, op, replay, dirs)
        elif kind == "backup":
            self._op_backup(j, op, replay)
        elif kind == "place":
            self._op_place(j, jf, i, op, last_place, dirs)
        elif kind == "remove":
            self._op_remove(op, dirs)
        elif kind == "rmdir":
            self._op_rmdir(op)
        elif kind == "touch":
            p = self.disk.find(op["path"])
            if p is not None and p.is_file():
                try:
                    self._retry(lambda: os.utime(str(p), None), op["path"])
                except _SwitchStop:
                    pass    # the files themselves are right; the next run touches it again (8.10)
        elif kind == "loader":
            self._op_loader()
        elif kind == "receipts":
            self._op_receipts(j)
        elif kind == "state":
            self._write_state(self._target(j), None, j)
        elif kind == "stamp":
            self._op_stamp(op)

    def _dll_name(self, j, path):
        """6.14 (switch.rs decide_dll_name): X.dll when it is there; else X.dll.odg-off
        when that is there, or when this journal moved the switched-off copy's old
        bytes aside and not X.dll's (it was off, and stays off); else X.dll."""
        on, off = self.disk.find(path), self.disk.find(path + OFF)
        if on is not None and on.is_file():
            return self.disk.rel(on)
        if off is not None and off.is_file():
            return self.disk.rel(off)

        def vacated(name):
            return any(o["op"] == "vacate" and akey(o["path"]) == akey(name) for o in j["ops"])

        if vacated(path + OFF) and not vacated(path):
            return path + OFF
        return path

    @staticmethod
    def _size_of(j, sha):
        """The size of the staged file `sha`: from the first place that uses it."""
        for o in j["ops"]:
            if o["op"] == "place" and o["sha"] == sha:
                return o["size"]
        return None

    def _op_vacate(self, j, op, replay, dirs):
        sub, name = stage_where(op["sha"])
        if self.view.size(sub, name) is not None:
            # On a fresh switch a copy made while staging already holds these bytes: the
            # source still goes, or a take-back or prune source would stay under its old
            # path (F7, as the launcher does). On a replay the staged file can be this
            # vacate's own work and the path may be refilled by now: nothing is touched.
            if not replay:
                src = self.disk.find(op["path"])
                want = self._size_of(j, op["sha"])
                try:
                    if src is not None and src.is_file() and (want is None or os.stat(str(src)).st_size == want):
                        _writable(src)
                        self._retry(lambda: os.remove(str(src)), op["path"])
                        dirs.add(src.parent)
                        self.disk.forget(src.parent)
                except OSError:
                    pass
            return
        src = self.disk.find(op["path"])
        try:
            size = os.stat(str(src)).st_size if src is not None and src.is_file() else None
        except OSError:
            size = None
        if size is None:
            return
        # Another size is never the bytes this op moves aside: the path was refilled by a
        # place after this vacate ran, or the file changed since the plan. The place that
        # needs them finds no staged file and stops; the next run fetches them.
        want = self._size_of(j, op["sha"])
        if want is not None and want != size:
            return
        if replay:
            # A place of this journal refills the path: with the same size, only the
            # hash tells whether this vacate already ran.
            refill = next((o for o in j["ops"] if o["op"] == "place"
                           and (akey(o.get("name") or o["path"]) == akey(op["path"])
                                or akey(o["path"]) == akey(op["path"]))), None)
            if refill is not None and refill["size"] == size \
                    and (refill["sha"] == op["sha"] or _sha_file(src) != op["sha"]):
                return
        self.view.ensure(sub)
        _writable(src)
        dst = self.view.path(sub, name)
        self._retry(lambda: os.replace(str(src), str(dst)), op["path"])
        self.view.added(sub, name, size)
        dirs.add(src.parent)
        dirs.add(dst.parent)
        self.disk.forget(src.parent)

    def _op_backup(self, j, op, replay):
        """6.15: the player's edited copy is kept beside the new file, over any older
        backup (the edit on disk now is the one to keep). On a replay it is done when
        the file already holds its place's new bytes: the backup ran before that place."""
        src = self.disk.find(op["path"])
        if src is None or not src.is_file():
            return
        if replay:
            own = next((o for o in j["ops"] if o["op"] == "place" and akey(o["path"]) == akey(op["path"])), None)
            try:
                same = own is not None and os.stat(str(src)).st_size == own["size"] and _sha_file(src) == own["sha"]
            except OSError:
                same = False
            if same:
                return
        try:
            dest = self.disk.place(op["to"])
        except Problem as e:
            raise _SwitchStop(str(e), op["path"])
        self._retry(lambda: _copy_over(src, dest), op["to"])
        self.disk.note(dest)

    def _placed(self, name, op):
        """6.12.4: a place whose staged file is gone is done when its bytes are already
        there, under its name, its manifest path, or (a DLL toggled since) its other
        name: the expected size, and either the state the switch began with records
        those bytes with this very stat, or the hash says so."""
        cands = [name, op["path"]] + ([op["path"] + OFF] if op["dll"] else [])
        rec = ((self.old_state or {}).get("files") or {}).get(op["path"])
        seen = set()
        for c in cands:
            if akey(c) in seen:
                continue
            seen.add(akey(c))
            p = self.disk.find(c)
            if p is None or not p.is_file():
                continue
            try:
                st = os.stat(str(p))
            except OSError:
                continue
            if st.st_size != op["size"]:
                continue
            if rec is not None and rec["h"] == op["sha"] and _same_stat(rec, st.st_size, st.st_mtime_ns):
                return True
            if _sha_file(p) == op["sha"]:
                return True
        return False

    def _op_place(self, j, jf, i, op, last_place, dirs):
        sub, sname = stage_where(op["sha"])
        staged = self.view.path(sub, sname)
        rel = op["path"]
        if op["dll"]:
            if op.get("name") is None:
                # Decided right before the rename and recorded first. A replay that finds
                # no record decides again by the same rule, with the same answer.
                op["name"] = self._dll_name(j, op["path"])
                try:
                    jf.decide(i, op["name"])
                except OSError:
                    pass
            rel = op["name"]
        # from the shard's one listing, kept up to date by this run's renames
        staged_ok = self.view.size(sub, sname) == op["size"]
        if not staged_ok:
            if self._placed(rel, op):
                return
            raise _SwitchStop(STOPPED, op["path"])
        try:
            dest = self.disk.place(rel)
        except Problem as e:
            raise _SwitchStop(str(e), op["path"])
        if dest.is_dir():
            raise _SwitchStop("{} is a folder where the update puts a file. Move it out of the way and run this "
                              "again.".format(self.disk.rel(dest)), op["path"])
        _writable(dest)
        if last_place.get(op["sha"], i) > i:
            # a later place needs the same bytes: this one gets a copy
            self._retry(lambda: _copy_over(staged, dest), op["path"])
        else:
            def move():
                try:
                    os.replace(str(staged), str(dest))
                except OSError as e:
                    if getattr(e, "errno", None) != errno.EXDEV:
                        raise
                    _copy_over(staged, dest)    # another volume behind a link: copy, then let go
                    os.remove(str(staged))
            self._retry(move, op["path"])
            self.view.removed(sub, sname)
        dirs.add(dest.parent)
        self.disk.note(dest)

    def _before_record(self, name):
        """The record, in the state the switch began with, of the file on disk under
        `name`: by the name it was recorded under, or (a DLL toggled since, which a
        rename does not tell the state) by its other name."""
        files = (self.old_state or {}).get("files") or {}
        by_name = {}
        for k, r in files.items():
            by_name.setdefault(akey(r.get("n") or k), r)
        k = akey(name)
        if k in by_name:
            return by_name[k]
        if k.endswith(OFF) and k[:-len(OFF)] in by_name:
            return by_name[k[:-len(OFF)]]
        return by_name.get(k + OFF)

    def _op_remove(self, op, dirs):
        rel = op["path"]
        if not self._may_delete(rel):
            return
        p = self.disk.find(rel)
        if p is None:
            return
        try:
            st = os.lstat(str(p))
        except OSError:
            return
        if not stat.S_ISREG(st.st_mode):
            return      # never a folder, never a link
        if op.get("sha") is not None:
            # Still as shipped: the record says so by stat, or the hash does. Otherwise it
            # changed since ODG put it there and is the player's now.
            r = self._before_record(rel)
            same = r is not None and r["h"] == op["sha"] and _same_stat(r, st.st_size, st.st_mtime_ns)
            if not same and _sha_file(p) != op["sha"]:
                say("Kept {}: it changed since ODG put it there.".format(rel))
                return
        _writable(p)
        self._retry(lambda: os.remove(str(p)), rel)
        self.removed[op["why"]] = self.removed.get(op["why"], 0) + 1
        dirs.add(p.parent)
        self.disk.forget(p.parent)

    def _op_rmdir(self, op):
        """A map folder the prune emptied, with any empty folders inside it. A folder
        still holding a file stays (nothing but files ODG shipped is removed), and a
        link is never followed nor removed."""
        d = self.disk.find(op["path"])
        if d is None or _is_link(d) or not d.is_dir():
            return
        for root, dnames, _fnames in os.walk(str(d), topdown=False):
            for dn in dnames:
                sub = os.path.join(root, dn)
                if _is_link(sub):
                    continue
                try:
                    os.rmdir(sub)
                except OSError:
                    pass
        try:
            os.rmdir(str(d))
        except OSError:
            pass
        self.disk.forget(d.parent)

    def _op_loader(self):
        """The loader from stage/loader.zip. A loader that will not install does not stop
        the switch: the pack is complete, the run reports it, and the next run plans
        the loader again (the launcher's rule)."""
        if self._loader_ready():
            return
        z = self.view.path("", STAGE_LOADER)
        if self.view.size("", STAGE_LOADER) is None:
            self.loader_error = "its download is gone"
            return
        try:
            extract(z, self.disk, "the mod loader")
        except Problem as e:
            self.loader_error = str(e)
            return
        self.disk.forget()
        state, missing = loader_state(self.disk)
        if state not in ("ready", "held"):
            self.loader_error = "still missing " + ", ".join(missing)
            return
        self.loader_installed = True

    def _target(self, j):
        """The target's manifest, from the copy the run that wrote the journal kept."""
        sha = j["target"]["manifest"]
        if self.man is not None and self.man.sha == sha:
            return self.man
        try:
            data = (self.dir / "manifests" / (sha + ".json.gz")).read_bytes()
            if hashlib.sha256(data).hexdigest() != sha:
                raise OSError("its copy is damaged")
            self.man = depot_parse_gz(data)
            return self.man
        except (OSError, DepotError) as e:
            raise _SwitchStop("The file list of {} is gone ({}); the next update repairs the pack.".format(
                j["target"]["tag"], e))

    def _op_receipts(self, j):
        """6.12.2.10: the legacy part receipt goes (any later legacy release fetches its
        parts again), and .installed_mapdata names the maps as their legacy zips. An
        empty map list says nothing about the maps: the receipts stay as they are."""
        m = self._target(j)
        for rel in (PARTS_STAMP, PREV_LIST):
            p = self.disk.find(rel)
            if p is not None:
                self._retry(lambda: os.remove(str(p)), rel)
                self.disk.forget(p.parent)
        if m.maps:
            lines = ["mapdata_{}.zip\t{}".format(mp["name"].lower(), mp["legacy"]) for mp in m.maps]
            self.disk.write(MAP_STAMP, "\n".join(lines) + "\n")

    def _write_state(self, m, adopted, j=None):
        """state.json for the install that now holds `m` (switch.rs derive_state): kept
        files take the plan's records when this run planned the switch; otherwise one
        listing of the folders gives each manifest file present with its size (a DLL
        under either name, a placed file under its recorded name) the manifest hash and
        the stat on disk. A runtime-written file is hashed (b is the shipped hash). A
        file the state the switch began with knows by this stat as other bytes gets no
        record (the next run reads it)."""
        keep = dict((akey(p), r) for p, r in (self._keep or {}).items())
        placed = {}
        if j is not None:
            for op in j["ops"]:
                if op["op"] == "place":
                    placed[akey(op["path"])] = op.get("name")
        ob = depot_observe(self.game, [f["p"] for f in m.files if akey(f["p"]) not in keep])
        old = self.old_state
        files = {}
        for f in m.files:
            k = akey(f["p"])
            sha = m.fsha(f)
            if k in keep:
                files[f["p"]] = keep[k]
                continue
            s = ob.get(placed[k]) if placed.get(k) else None
            if s is None:
                on, off = ob.lookup(f["p"])
                s = (on or off) if is_top_dll(f["p"]) else on
            if s is None:
                continue
            if s.size != f["s"] and not f["x"]:
                continue
            if f["x"]:
                h = sha if k in placed else _sha_file(self.game.joinpath(*s.name.split("/")))
                files[f["p"]] = _record(h, s, f["p"], sha)
            elif k in placed:
                files[f["p"]] = _record(sha, s, f["p"], None)
            else:
                r = ((old or {}).get("files") or {}).get(f["p"])
                if r is not None and _same_stat(r, s.size, s.mtime_ns) and r["h"] != sha:
                    continue
                files[f["p"]] = _record(sha, s, f["p"], None)
        if j is not None:
            # carried from the state this switch replaced (or, on a replay after the
            # state op, the one it wrote); a state of another build is a stale one the
            # run did not use, so the install was adopted from `from`
            if old is not None and old["tag"] in (j["from"]["tag"], j["target"]["tag"]):
                adopted = old.get("adopted_from")
            else:
                adopted = j["from"]["tag"] or None
        st = {"format": DEPOT_FORMAT, "lineage": m.lineage, "repo": m.repo, "tag": m.tag, "manifest": m.sha,
              "written_by": DEPOT_TOOL, "written": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
              "adopted_from": adopted or None, "forgotten": False, "maps": [mp["name"] for mp in m.maps],
              "files": files}
        try:
            depot_save_state(self.game, st)
        except OSError as e:
            raise _SwitchStop("Could not write the record of what is installed ({}). Run this again.".format(e))
        self._prune_manifests(m.sha)

    def _prune_manifests(self, keep):
        """The installed file list and the 3 newest stay in depot/manifests."""
        try:
            entries = [e for e in os.scandir(str(self.dir / "manifests")) if e.name.endswith(".json.gz")]
        except OSError:
            return
        entries.sort(key=lambda e: e.stat().st_mtime, reverse=True)
        for e in entries[3:]:
            if e.name != keep + ".json.gz":
                remove_quietly(e.path)

    def _op_stamp(self, op):
        try:
            p = self.disk.place(STAMP)
            depot_write(p, op["tag"].encode("utf-8"))
        except (OSError, Problem) as e:
            raise _SwitchStop("Could not write {} ({}). Run this again to finish the update.".format(STAMP, e))
        self.disk.note(p)
        self.disk.remove(FORGOTTEN)


def depot_may_delete_name(rel):
    """The name half of 6.13's guard (the launcher's may_delete without the disk):
    inside Mods, never the free mods, the player's own maps or the add-on records."""
    if not safe_rel(rel):
        return False
    lower = rel.replace("\\", "/").lower()
    return not (free_mod_path(lower) or is_custom(lower) or lower == "mods/odg_addons"
                or lower.startswith("mods/odg_addons/"))


# ---------------------------------------------------------------------------
# this tool's own updates
# ---------------------------------------------------------------------------

def self_update():
    """Newer copies of this file are published on the site. A newer one replaces
    this file and runs in its place; an older or equal one never does."""
    if OPTS.get("no_self_update") or os.environ.get("ODG_NO_SELF_UPDATE") or os.environ.get("ODG_SELF_UPDATED"):
        return
    try:
        me = Path(os.path.abspath(sys.argv[0]))
    except Exception:
        return
    if not me.is_file() or me.suffix.lower() != ".py" or not os.access(str(me), os.W_OK):
        return
    try:
        doc = http_json(site() + "/api/releases", headers={"Accept": "application/json"}, timeout=15, attempts=2)
    except (NoAnswer, Refused):
        return
    entry = None
    for a in doc.get("assets") or [] if isinstance(doc, dict) else []:
        if isinstance(a, dict) and a.get("key") == TOOL_KEY:
            entry = a
            break
    if not entry or compare(TOOL_VERSION, str(entry.get("version") or "")) != "behind":
        return
    want = str(entry.get("sha256") or "").lower()
    if len(want) != 64:
        return
    new = me.with_name(me.name + ".new")
    try:
        download(site() + "/dl/" + TOOL_KEY, new, int(entry.get("size") or 0), "ODG for Linux")
        verify(new, int(entry.get("size") or 0), "sha256:" + want)
        text = new.read_text(encoding="utf-8")
        m = re.search(r'^TOOL_VERSION = "([^"]+)"', text, re.M)
        if "ODG_LINUX_TOOL = True" not in text or not m or compare(TOOL_VERSION, m.group(1)) != "behind":
            raise Problem("the published copy is not a newer version of this tool")
        os.chmod(str(new), stat.S_IMODE(os.stat(str(me)).st_mode))
        os.replace(str(new), str(me))
    except (Problem, OSError) as e:
        remove_quietly(new)
        say("This tool could not update itself ({}). Carrying on with version {}.".format(e, TOOL_VERSION))
        return
    say("Updated this tool to version {}. Starting it again.".format(m.group(1)))
    env = dict(os.environ)
    env["ODG_SELF_UPDATED"] = "1"
    os.execve(sys.executable, [sys.executable, str(me)] + sys.argv[1:], env)


# ---------------------------------------------------------------------------
# steam's launch option
# ---------------------------------------------------------------------------

OVERRIDE_RE = re.compile(r"""WINEDLLOVERRIDES\s*=\s*("([^"]*)"|'([^']*)'|(\S*))""", re.I)


def _override_value(m):
    for g in (2, 3, 4):
        if m.group(g) is not None:
            return m.group(g)
    return ""


def override_ok(opts):
    for m in OVERRIDE_RE.finditer(opts or ""):
        for entry in _override_value(m).split(";"):
            if "=" not in entry:
                continue
            dlls, mode = entry.split("=", 1)
            names = [d.strip().lower() for d in dlls.split(",")]
            if ("version" in names or "version.dll" in names) and mode.strip().lower().startswith("n"):
                return True
    return False


def suggested_option(opts):
    opts = (opts or "").strip()
    if not opts:
        return LAUNCH_OPTION
    m = OVERRIDE_RE.search(opts)
    if m:
        val = _override_value(m).strip().rstrip(";")
        new = 'WINEDLLOVERRIDES="{}"'.format(val + ";version=n,b" if val else "version=n,b")
        return opts[:m.start()] + new + opts[m.end():]
    if "%command%" in opts:
        return 'WINEDLLOVERRIDES="version=n,b" ' + opts
    return LAUNCH_OPTION + " " + opts


def launch_options(roots):
    """[(account name, launch options, most recent)] for each Steam account on this
    PC that has OPERATOR in its saved settings."""
    out = []
    users = steam_users(roots)
    for r in roots:
        ud = ci_child(r, "userdata")
        if ud is None:
            continue
        try:
            ids = [d for d in os.listdir(str(ud)) if d.isdigit()]
        except OSError:
            continue
        for d in ids:
            lc = Path(str(ud)) / d / "config" / "localconfig.vdf"
            try:
                text = lc.read_text(encoding="utf-8", errors="replace")
            except OSError:
                continue
            if '"{}"'.format(APPID) not in text:
                continue
            doc = vdf_parse(text)
            apps = dig(doc, "UserLocalConfigStore", "Software", "Valve", "Steam", "apps")
            app = apps.get(APPID) if isinstance(apps, dict) else None
            if not isinstance(app, dict):
                continue
            sid = str(STEAM_BASE + int(d))
            u = users.get(sid, {})
            out.append((u.get("name") or "Steam account " + d, str(app.get("launchoptions") or ""), bool(u.get("recent"))))
    return out


def launch_option_report(inst, verbose=True):
    """True when the launch option is seen. Steam saves it when it closes, so a
    missing one is reported as not seen yet, not as wrong."""
    found = launch_options(inst.roots)
    if not found:
        if verbose:
            say("Launch option: not seen in Steam's saved settings yet.")
        return False
    good = True
    for name, opts, _recent in found:
        if override_ok(opts):
            if verbose:
                say("Launch option: set ({}).".format(name))
        else:
            good = False
            if verbose:
                say("Launch option: missing for {}.".format(name))
                if opts.strip():
                    say("  Yours now:   " + opts.strip())
                    say("  Change it to: " + suggested_option(opts))
    return good


def print_launch_steps(current=""):
    say("")
    say("One step in Steam, once:")
    say("  1. In your Library, right-click OPERATOR and pick Properties.")
    say("  2. Under General, find Launch Options and paste this in:")
    say("")
    say("     " + suggested_option(current))
    say("")
    say("  3. Close the window. Start the game from Steam as usual.")
    say("Without it the game still starts, but no mods load.")


# ---------------------------------------------------------------------------
# what happened last time the game ran
# ---------------------------------------------------------------------------

def last_runs(inst):
    disk = Disk(inst.game)
    ml = disk.find("MelonLoader/Latest.log")
    ml_time = os.path.getmtime(str(ml)) if ml is not None else None
    player = None
    if inst.prefix is not None:
        base = inst.prefix / "drive_c" / "users" / "steamuser" / "AppData"
        player = ci_child(ci_child(ci_child(ci_child(base, "LocalLow"), "VECTOR INTERACTIVE"), "OPERATOR"), "Player.log")
    game_time = os.path.getmtime(str(player)) if player is not None and player.is_file() else None
    loaded, problem = None, None
    if ml is not None:
        try:
            text = ml.read_text(encoding="utf-8", errors="replace")
        except OSError:
            text = ""
        m = None
        for m in re.finditer(r"(\d+) Mods? loaded", text):
            pass
        loaded = int(m.group(1)) if m else None
        if "Failed to load Hostfxr" in text:
            problem = "the mod loader could not start its runtime (Failed to load Hostfxr)"
    return game_time, ml_time, loaded, problem


def when(t):
    return time.strftime("%Y-%m-%d %H:%M", time.localtime(t)) if t else "never"


# ---------------------------------------------------------------------------
# commands
# ---------------------------------------------------------------------------

OPTS = {}


def require_not_root():
    if hasattr(os, "geteuid") and os.geteuid() == 0:
        raise Problem("Run this as yourself, without sudo. Files made by root lock Steam out of the game folder.")


def accept_terms(cfg):
    if cfg.get("terms") == TERMS_VERSION:
        return
    if OPTS.get("accept_terms"):
        cfg["terms"] = TERMS_VERSION
        save_config(cfg)
        return
    if not sys.stdin or not sys.stdin.isatty():
        raise Problem("The terms have not been accepted. Run this in a terminal, or add --accept-terms "
                      "after reading them at " + TERMS_URL)
    say("BEFORE YOU START")
    say("")
    say("This tool installs mods into your copy of OPERATOR. These are the same terms")
    say("the Windows launcher shows. The full text: {} (version {}).".format(TERMS_URL, TERMS_VERSION))
    for head, body in TERMS:
        say("")
        say(head)
        say(wrap(body, 78))
    say("")
    try:
        answer = input("Type yes to accept, or anything else to stop: ").strip().lower()
    except EOFError:
        answer = ""
    if answer != "yes":
        raise Problem("Nothing was changed.")
    cfg["terms"] = TERMS_VERSION
    save_config(cfg)
    a = load_account()
    if a["key"]:
        record_consent(a["key"])
    say("")


def wrap(text, width):
    lines, line = [], ""
    for word in text.split():
        if line and len(line) + 1 + len(word) > width:
            lines.append(line)
            line = word
        else:
            line = (line + " " + word).strip()
    if line:
        lines.append(line)
    return "\n".join(lines)


def header(inst):
    say("ODG for Linux {}".format(TOOL_VERSION))
    say("Game: {}".format(inst.game))
    if not os.access(str(inst.game), os.W_OK):
        raise Problem("This user cannot write to the game folder. If you ever ran this with sudo, "
                      "fix the folder's owner, then run this again.")


def cmd_install(_args):
    require_not_root()
    cfg = load_config()
    accept_terms(cfg)
    self_update()
    inst = resolve_install(OPTS, cfg)
    if OPTS.get("game"):
        cfg["game"] = str(inst.game)
        save_config(cfg)
    header(inst)
    acct = load_account(inst)
    if acct["key"]:
        acct = refresh_account(acct, inst)
        save_account(acct, inst)
        say("Account: {} ({})".format(acct["name"] or "signed in", stamp_of(acct)))
    say("Checking for updates")
    up = Updater(inst)
    try:
        # 6.2: a switch an earlier run left half done is finished first, without the network
        if up.disk.find(DEV_HOLD) is None:
            rec = up.depot.recover()
            if rec.startswith("finished:"):
                say("Finished installing {}, which an earlier run had left part way.".format(rec[len("finished:"):]))
                up.changed = True
            elif rec.startswith("pending:"):
                say(rec[len("pending:"):])
        public_doc = None if is_member(acct) else public_listing()
        up.run(acct, public_doc)
        up.tidy(ask_leftovers=True)
    finally:
        up.depot.close()
    if depot_switch_pending(inst.game) and up.disk.find(DEV_HOLD) is None:
        up.fail(STOPPED)
    disk = up.disk
    state, missing = loader_state(disk)
    have = disk.read(STAMP).strip()
    say("")
    if up.failed:
        say("Finished with problems. Run this again to retry what was skipped.")
    elif have:
        say("ODG {} is installed.".format(have))
    if state == "held":
        say("The mod loader is switched off (vanilla). Run this with 'modded' to turn it back on.")
    elif state != "ready" and not up.running:
        say("The mod loader is not complete: missing {}.".format(", ".join(missing)))
    if not launch_option_report(inst, verbose=False):
        found = launch_options(inst.roots)
        print_launch_steps(found[0][1] if found else "")
        say("Steam saves launch options when it closes. If you have already added it, ignore this.")
    if inst.prefix is None:
        say("")
        say("Start the game from Steam. The first start takes a few minutes while the mod loader")
        say("sets itself up.")
        if acct["key"]:
            say("After that first start, run this again so your membership reaches the game.")
    elif up.changed:
        say("")
        say("Start the game from Steam. The first start after an install takes a few minutes.")
    return 1 if up.failed else 0


def cmd_status(_args):
    cfg = load_config()
    inst = resolve_install(OPTS, cfg)
    disk = Disk(inst.game)
    say("ODG for Linux {}".format(TOOL_VERSION))
    say("Game:         {}".format(inst.game))
    say("Proton:       {}".format("the game has run on this PC" if inst.prefix else "not set up yet (start the game once)"))
    have = disk.read(STAMP).strip()
    latest = ""
    try:
        latest = str(public_listing().get("tag_name") or "")
    except Problem:
        pass
    acct = load_account(inst)
    line = have or "not installed"
    if latest and have and not is_member(acct):
        line += " (up to date)" if have == latest else " (the public release is {})".format(latest)
    say("Pack:         " + line)
    maps = [l for l in disk.read(MAP_STAMP).splitlines() if l.strip()]
    say("Map data:     {} installed".format(len(maps)))
    state, missing = loader_state(disk)
    say("Mod loader:   " + {
        "ready": "ready",
        "held": "switched off (vanilla)",
        "absent": "not installed",
        "incomplete": "damaged, missing " + ", ".join(missing),
    }[state])
    mods = mod_files(disk)
    on = sum(1 for m in mods if m[1])
    say("Mods:         {} on, {} off".format(on, len(mods) - on))
    say("Account:      {}".format("{} ({})".format(acct["name"] or "signed in", stamp_of(acct)) if acct["key"] else "not signed in (free)"))
    say("")
    ok = launch_option_report(inst)
    game_t, ml_t, loaded, problem = last_runs(inst)
    say("Game last ran:       " + when(game_t))
    say("Mod loader last ran: " + when(ml_t) + (", {} mods loaded".format(loaded) if loaded is not None else ""))
    if problem:
        say("Last run problem: " + problem)
    if game_t and (ml_t is None or game_t - ml_t > 120):
        say("")
        say("The game ran without the mod loader last time. The launch option is missing or mistyped.")
        found = launch_options(inst.roots)
        print_launch_steps(found[0][1] if found else "")
    elif not ok and state == "ready":
        print_launch_steps()
    return 0


def squash(s):
    return re.sub(r"[^a-z0-9]", "", s.lower())


def mod_keys(file):
    """What a player might type for a mod: 'Third Person', 'thirdperson', 'Map Loader'."""
    f = file.lower()
    stem = f[:-4] if f.endswith(".dll") else f
    short = stem[:-4] if stem.endswith(".mod") else stem
    name = squash(display_name(file))
    keys = {squash(f), squash(stem), squash(short), name}
    if name.startswith("odg"):
        keys.add(name[3:])
    return keys


def find_mods(disk, names):
    mods = mod_files(disk)
    picked = []
    for want in names:
        w = squash(want)
        if w == "all":
            picked.extend(mods)
            continue
        if not w:
            raise Problem("Name the mod. 'mods' lists them.")
        keys = mod_keys
        exact = [m for m in mods if w in keys(m[0])]
        if not exact:
            exact = [m for m in mods if any(k.startswith(w) for k in keys(m[0]))]
        if not exact:
            raise Problem("No mod called '{}'. 'mods' lists them.".format(want))
        if len(exact) > 1:
            raise Problem("'{}' matches {}. Be more exact.".format(want, ", ".join(display_name(m[0]) for m in exact)))
        picked.append(exact[0])
    seen, out = set(), []
    for m in picked:
        if m[0].lower() not in seen:
            seen.add(m[0].lower())
            out.append(m)
    return out


def cmd_mods(_args):
    inst = resolve_install(OPTS, load_config())
    disk = Disk(inst.game)
    mods = mod_files(disk)
    if not mods:
        say("No mods are installed. Run this tool with no command to install ODG.")
        return 0
    width = max(len(display_name(m[0])) for m in mods)
    for file, on, _off in mods:
        tag = "REQUIRED" if requirement(file) else ("on" if on else "off")
        say("  {}  {}".format(display_name(file).ljust(width), tag))
    say("")
    say("Switch one off with:  python3 odg-linux.py off \"Third Person\"")
    return 0


def cmd_switch(args, enable):
    if not args:
        raise Problem("Name the mod, for example: python3 odg-linux.py {} \"Third Person\"".format("on" if enable else "off"))
    inst = resolve_install(OPTS, load_config())
    disk = Disk(inst.game)
    for file, on, off in find_mods(disk, args):
        name = display_name(file)
        if not enable and requirement(file):
            if len(args) == 1 and args[0].lower() != "all":
                say("{} stays on. {}".format(name, requirement(file)))
            continue
        if set_mod(disk, file, on, off, enable):
            say("{} is {}.".format(name, "on" if enable else "off"))
        else:
            say("{} was already {}.".format(name, "on" if enable else "off"))
    if game_running():
        say("The game is running. The change counts from its next start.")
    return 0


def cmd_vanilla(_args, on):
    inst = resolve_install(OPTS, load_config())
    disk = Disk(inst.game)
    proxy, held = disk.find("version.dll"), disk.find(HELD)
    if not on:
        if proxy is None:
            say("The mod loader is already off." if held else "The mod loader is not installed.")
            return 0
        os.replace(str(proxy), str(inst.game / HELD))
        say("The mod loader is off. The game starts without mods. Turn it back on with: python3 odg-linux.py modded")
    else:
        if held is None:
            say("The mod loader is already on." if proxy else "The mod loader is not installed. Run this tool with no command.")
            return 0
        if proxy is None:
            os.replace(str(held), str(inst.game / "version.dll"))
        else:
            remove_quietly(held)
        say("The mod loader is on again.")
    return 0


def open_browser(url):
    if os.environ.get("ODG_NO_BROWSER"):
        return False
    try:
        import webbrowser
        return webbrowser.open(url)
    except Exception:
        return False


def cmd_signin(args):
    require_not_root()
    cfg = load_config()
    accept_terms(cfg)
    inst = None
    try:
        inst = resolve_install(OPTS, cfg)
    except Problem:
        pass
    acct = load_account(inst)
    how = args[0].strip() if args else ""
    if not how:
        say("Sign in with Patreon, with Steam, or with an invite code from us.")
        try:
            how = input("Type patreon, steam, or your code: ").strip()
        except EOFError:
            how = ""
    if not how:
        raise Problem("Nothing was changed.")
    now = now_unix()
    if how.lower() not in ("patreon", "steam"):
        code = how.upper()
        if len(code) < 6:
            raise Problem("That code is too short.")
        try:
            doc = account_call("/api/launcher/redeem", body={"code": code})
        except NoAnswer as e:
            raise Problem("Could not reach the account server: {}".format(e))
        if doc.get("error"):
            raise Problem(str(doc["error"]))
        key = str(doc.get("key") or "")
        if not key:
            raise Problem("The server did not issue a key.")
        a = from_answer(doc, key, "", now)
    else:
        key = acct["key"] or None
        try:
            if how.lower() == "steam":
                owner = operator_owner(inst) if inst else None
                pc = steam_pc_account(inst.roots if inst else steam_roots())
                doc = account_call("/api/launcher/steam/start", key=key, body={
                    "owner": owner or "", "pc": pc["id"] if pc else "", "pc_name": pc["name"] if pc else ""})
            else:
                doc = account_call("/api/launcher/link/start", key=key, body={})
        except NoAnswer as e:
            raise Problem("Could not reach the account server: {}".format(e))
        if doc.get("error"):
            raise Problem(str(doc["error"]))
        code, secret, url = str(doc.get("code") or ""), str(doc.get("secret") or ""), str(doc.get("url") or "")
        if not (code and secret and url):
            raise Problem("The server did not start a sign-in.")
        say("")
        say("Open this address in your browser and sign in there:")
        say("  " + url)
        say("Your code: " + code)
        open_browser(url)
        say("")
        say("Waiting for the sign-in to finish. Ctrl+C stops waiting.")
        a = None
        deadline = time.time() + 15 * 60
        while time.time() < deadline:
            time.sleep(2)
            try:
                doc = account_call("/api/launcher/link/poll?code={}&secret={}".format(
                    urllib.parse.quote(code), urllib.parse.quote(secret)))
            except NoAnswer:
                continue
            if doc.get("error"):
                raise Problem(str(doc["error"]))
            if doc.get("pending") or not doc.get("key"):
                continue
            a = from_answer(doc, str(doc["key"]), "", now_unix())
            break
        if a is None:
            raise Problem("That sign-in expired. Start again.")
    save_account(a, inst)
    record_consent(a["key"])
    say("Signed in as {} ({}).".format(a["name"] or "a member", stamp_of(a)))
    if inst is not None and inst.prefix is None:
        say("Start the game once, then run this tool again, so your membership reaches the game.")
    else:
        say("Run this tool with no command to get the build your tier is on.")
    return 0


def cmd_signout(_args):
    inst = None
    try:
        inst = resolve_install(OPTS, load_config())
    except Problem:
        pass
    save_account(blank_account(), inst)
    say("Signed out. This PC gets the public release from now on.")
    return 0


def cmd_uninstall(_args):
    require_not_root()
    cfg = load_config()
    inst = resolve_install(OPTS, cfg)
    disk = Disk(inst.game)
    if game_running():
        raise Problem("Close OPERATOR first.")
    if disk.find(DEV_HOLD) is not None:
        raise Problem("This folder is held by .odg-dev-hold. Nothing was removed.")
    mods = disk.find("Mods")
    entries, seen = [], set()

    def add(rel):
        lower = rel.lower()
        if not safe_rel(rel) or is_custom(lower) or lower in seen:
            return
        seen.add(lower)
        p = disk.find(rel)
        if p is None:
            return
        try:
            st = os.lstat(str(p))
        except OSError:
            return
        if stat.S_ISLNK(st.st_mode):
            return
        entries.append((disk.rel(p), 0 if stat.S_ISDIR(st.st_mode) else st.st_size, stat.S_ISDIR(st.st_mode)))

    stems = []
    for rel in read_list(disk.read(SHIPPED_LIST)):
        add(rel)
        if rel.lower().endswith(".dll"):
            add(rel + OFF)
        if rel.startswith("Mods/") and "/" not in rel[5:] and rel.lower().endswith(".dll"):
            stems.append(rel[5:].split(".")[0])
    for file in KNOWN:
        real = ci_child(mods, file) if mods else None
        name = real.name if real is not None else file
        add("Mods/" + name)
        add("Mods/" + name + OFF)
        stems.append(name.split(".")[0])
    for m in UNINSTALL_MARKERS:
        add(m)
    for rel, _s, _d in stale_scan(disk):
        add(rel.rstrip("/"))
    stems = [s.lower() for s in stems if len(s) >= 3]
    others = []
    if mods is not None:
        for n in os.listdir(str(mods)):
            low = n.lower()
            if (low.endswith(".log") or low.endswith(".txt")) and os.path.isfile(str(mods / n)):
                if any(len(low) > len(s) and low.startswith(s) and low[len(s)] in "._-" for s in stems):
                    add("Mods/" + n)
        ours = {e[0].lower() for e in entries}
        for n in os.listdir(str(mods)):
            low = n.lower()
            if (low.endswith(".dll") or low.endswith(".dll" + OFF)) and os.path.isfile(str(mods / n)) \
                    and "mods/" + low not in ours:
                others.append(n[:-len(OFF)] if low.endswith(OFF) else n)
    proxy = disk.find("version.dll") or disk.find(HELD)
    loader = proxy is not None and is_melonloader(proxy)
    size = sum(s for _r, s, _d in entries)
    say("ODG for Linux {}".format(TOOL_VERSION))
    say("Game: {}".format(inst.game))
    say("")
    say("This removes the ODG pack: {} files and folders, {}.".format(len(entries), mb(size)))
    say("It keeps OPERATOR, your saves, your mod settings and outfits, and your own maps.")
    take_loader = False
    if loader:
        if others:
            say("Other mods use the mod loader: {}.".format(", ".join(sorted(set(others)))))
            take_loader = ask("Remove the mod loader too?", default=False)
        else:
            say("The mod loader goes too.")
            take_loader = True
    if not (OPTS.get("yes") or ask("Remove ODG now?", default=False)):
        raise Problem("Nothing was changed.")
    removed, failed = delete_entries(disk, entries)
    # Folders the pack owned and left empty, deepest first. Never Custom.
    dirs = set()
    for rel, _s, _d in entries:
        p = inst.game / rel
        while p.parent != inst.game and mods is not None and mods in p.parents:
            p = p.parent
            if p == mods:
                break
            if "/maploader_assets/custom" not in disk.rel(p).lower():
                dirs.add(p)
    for d in sorted(dirs, key=lambda x: len(x.parts), reverse=True):
        try:
            d.rmdir()
        except OSError:
            pass
    diag = ci_child(ci_child(inst.game, "UserData"), "ODG Diagnostics")
    if diag is not None and diag.is_dir() and not diag.is_symlink():
        shutil.rmtree(str(diag), ignore_errors=True)
    if take_loader:
        for name in ("version.dll", HELD):
            p = ci_child(inst.game, name)
            if p is not None and is_melonloader(p):
                try:
                    os.remove(str(p))
                except OSError as e:
                    failed.append((name, str(e)))
        if ci_child(inst.game, "version.dll") is None and ci_child(inst.game, HELD) is None:
            ml = ci_child(inst.game, "MelonLoader")
            if ml is not None and ml.is_dir() and not ml.is_symlink():
                shutil.rmtree(str(ml), ignore_errors=True)
        for d in ("Plugins", "UserLibs", "Mods"):
            p = ci_child(inst.game, d)
            if p is not None:
                try:
                    p.rmdir()
                except OSError:
                    pass
    # the depot's record, stage and journal go with the pack (11.1)
    depot_remove_all(inst.game)
    data = ci_child(ci_child(inst.game, "UserData"), "ODGModManager")
    if data is not None and data.is_dir() and not data.is_symlink():
        shutil.rmtree(str(data), ignore_errors=True)
    pa = prefix_account_path(inst)
    if pa is not None:
        remove_quietly(pa)
    remove_quietly(config_dir() / "account.json")
    say("Removed {} item{}.".format(len(removed), "" if len(removed) == 1 else "s"))
    for rel, why in failed[:10]:
        say("  could not remove {}: {}".format(rel, why))
    say("")
    say("Two things left for you:")
    say("  1. In Steam, clear OPERATOR's Launch Options (Properties > General).")
    say("  2. In Steam, right-click OPERATOR > Properties > Installed Files > Verify integrity")
    say("     of game files. That puts back any game file the pack replaced.")
    say("This file, odg-linux.py, can be deleted.")
    return 1 if failed else 0


HELP = """ODG for Linux {v}

  python3 odg-linux.py              install ODG, or update it
  python3 odg-linux.py status       what is installed, and whether Steam is set up
  python3 odg-linux.py mods         list the mods
  python3 odg-linux.py off NAME     switch a mod off ("on NAME" switches it back)
  python3 odg-linux.py vanilla      start the game without mods ("modded" undoes it)
  python3 odg-linux.py signin       sign in with Patreon, Steam, or an invite code
  python3 odg-linux.py signout      forget the sign-in on this PC
  python3 odg-linux.py uninstall    take ODG off this PC

Options:
  --game DIR        your OPERATOR folder, when it is not found by itself
  --yes             answer yes to every question except the terms
  --accept-terms    accept the terms without being asked ({url})
"""


def parse_args(argv):
    opts, rest = {}, []
    i = 0
    while i < len(argv):
        a = argv[i]
        if a in ("-h", "--help"):
            rest.insert(0, "help")
        elif a in ("-y", "--yes"):
            opts["yes"] = True
        elif a == "--accept-terms":
            opts["accept_terms"] = True
        elif a == "--no-self-update":
            opts["no_self_update"] = True
        elif a == "--game" or a.startswith("--game="):
            if a == "--game":
                i += 1
                if i >= len(argv):
                    raise Problem("--game needs the folder after it.")
                opts["game"] = argv[i]
            else:
                opts["game"] = a.split("=", 1)[1]
        elif a.startswith("-") and len(a) > 1:
            raise Problem("Unknown option {}. 'help' lists them.".format(a))
        else:
            rest.append(a)
        i += 1
    cmd = rest[0].lower() if rest else "install"
    return cmd, rest[1:], opts


def main(argv=None):
    if sys.version_info < (3, 8):
        print("ODG for Linux needs Python 3.8 or newer.", file=sys.stderr)
        return 2
    try:
        cmd, args, opts = parse_args(sys.argv[1:] if argv is None else argv)
        OPTS.clear()
        OPTS.update(opts)
        commands = {
            "install": cmd_install, "update": cmd_install,
            "status": cmd_status,
            "mods": cmd_mods, "list": cmd_mods,
            "on": lambda a: cmd_switch(a, True), "off": lambda a: cmd_switch(a, False),
            "vanilla": lambda a: cmd_vanilla(a, False), "modded": lambda a: cmd_vanilla(a, True),
            "signin": cmd_signin, "login": cmd_signin,
            "signout": cmd_signout, "logout": cmd_signout,
            "uninstall": cmd_uninstall,
        }
        if cmd == "help":
            print(HELP.format(v=TOOL_VERSION, url=TERMS_URL))
            return 0
        if cmd not in commands:
            raise Problem("Unknown command '{}'. 'help' lists them.".format(cmd))
        lock = _lock()
        try:
            return commands[cmd](args) or 0
        finally:
            if lock is not None:
                lock.close()
    except Problem as e:
        print("", file=sys.stderr)
        print(str(e), file=sys.stderr)
        return 1
    except KeyboardInterrupt:
        print("\nStopped. Run this again to finish.", file=sys.stderr)
        return 130


def _lock():
    """One copy at a time: two unpacking over one game folder would mix packs."""
    try:
        import fcntl
    except ImportError:
        return None
    d = config_dir()
    try:
        d.mkdir(parents=True, exist_ok=True)
        f = open(str(d / "lock"), "w")
    except OSError:
        return None
    try:
        fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
    except OSError:
        f.close()
        raise Problem("ODG for Linux is already running in another window.")
    return f


if __name__ == "__main__":
    sys.exit(main())
