From 9bb8a8451ebf215ba86c3c3b63a419247c4a6e58 Mon Sep 17 00:00:00 2001 From: Thomas Walker Lynch Date: Tue, 10 Nov 2026 13:37:29 +0000 Subject: [PATCH] . --- .gitignore | 2 +- 0pus_AccountTree | 0 administrator/tool/skeleton_update | 621 ++++++++++++++ developer/authored/.gitkeep | 0 .../deprecated/tester}/scratchpad/.gitignore | 0 developer/design_AccountTree.html | 761 ++++++++++++++++++ document/design_AccountTree.html | 28 +- setup | 52 +- shared/project-dependency/Python.org | 95 +-- shared/tool/env_to_emacs | 74 +- shared/tool/makefile/Harmony.mk | 4 +- shared/tool/setup | 2 + shared/tool/version | 2 + 13 files changed, 1527 insertions(+), 114 deletions(-) create mode 100644 0pus_AccountTree create mode 100755 administrator/tool/skeleton_update create mode 100644 developer/authored/.gitkeep rename {tester => developer/authored/deprecated/tester}/scratchpad/.gitignore (100%) create mode 100644 developer/design_AccountTree.html diff --git a/.gitignore b/.gitignore index 9538920..905d115 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,6 @@ __pycache__/ *.swp .DS_Store -# SubU +# AccountTree __pycache__ *.sqlite3 diff --git a/0pus_AccountTree b/0pus_AccountTree new file mode 100644 index 0000000..e69de29 diff --git a/administrator/tool/skeleton_update b/administrator/tool/skeleton_update new file mode 100755 index 0000000..716e342 --- /dev/null +++ b/administrator/tool/skeleton_update @@ -0,0 +1,621 @@ +#!/usr/bin/env python3 +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- +""" +skeleton_update + +Brings a project onto the current Harmony skeleton. Skeleton files are +copied in from a Harmony checkout, and .gitignore and shared/tool/version are +merged with the project's own lines. The project's files are then sorted: + + a file in a place Harmony has, such as developer/authored/ or a directory + of the skeleton, stays where it is; + a file the plan moves is renamed to its new place; + a file under a plan's duplicate tree is deleted when the project holds a + file with the same content elsewhere, and is otherwise deprecated; + every other file is deprecated: moved, with its path kept, under + developer/authored/deprecated/, so nothing is lost and the original place + of each file can be read from its new path. + +The plan is a Python file giving what is particular to one project. A +project already laid out after Harmony needs no plan, and the tool then +updates its skeleton files only. The plan's names are: + + MOVE {old path: new path}; a path ending in / moves a directory tree + DEPRECATE paths that are deprecated even though Harmony has their place + DUPLICATE trees whose files are copies of files kept elsewhere + KEEP paths kept as they are, including skeleton files the project + has its own version of + EXECUTABLE paths, after any move, given execute permission where read + permission is given + +The files sorted are those git tracks or would track: the work tree less what +the project's .gitignore ignores. Ignored files, such as a local Python +environment, are left where they are and listed at the end of the log. +Compiled Python caches are deleted. Directories left empty are removed. + +Commit the project first; run refuses a work tree with uncommitted changes. +Afterward, 'git add -A' records the moves, and git recognizes each moved file +as a rename by its content, so 'git log --follow' follows its history. Run +again after committing, and the log reports nothing left to do. + +Usage: skeleton_update run [] + skeleton_update preview run [] + help | -h | --help print this message (also printed when given no arguments) + run bring onto the skeleton of + preview run print the log of what run would do, change nothing +""" + +import sys ,os ,shutil ,filecmp ,subprocess ,runpy ,hashlib +from pathlib import Path + + +# -------------------------------------------------------------------------------- +# what the tool knows about Harmony + +DEPRECATED_DIR = "developer/authored/deprecated" + +# Skeleton files a project does not take: it has its own. +SKELETON_SKIP_FILE = {"README.md" ,"LICENSE" ,"0pus_Harmony"} +SKELETON_SKIP_TREE = ("developer/authored/ExampleGreet.make/",) + +# Skeleton files merged with the project's version rather than replaced. +MERGED_FILE = (".gitignore" ,"shared/tool/version") + +# Files at the top of a project that belong to the project. +PROJECT_ROOT_FILE = {"LICENSE" ,"README.md" ,".gitattributes" ,".gitmodules"} + +# Trees in which every file belongs to the project. +PROJECT_TREE = ( + "administrator/authored/", + "administrator/document/", + "developer/authored/", + "developer/document/", + "developer/experiment/", + "document/", + "shared/authored/", + "tester/authored/", + "tester/document/", +) + +# Placeholder files of the old skeleton; Harmony uses .gitkeep. +PLACEHOLDER_NAME = {".githolder"} + +CACHE_DIR_NAME = {"__pycache__"} + +PLAN_NAME = ("MOVE" ,"DEPRECATE" ,"DUPLICATE" ,"KEEP" ,"EXECUTABLE") + + +# -------------------------------------------------------------------------------- +# the log + +TAG_WIDTH = 18 + +# the tag a preview prints in place of each past-tense tag +PREVIEW_TAG = { + "added": "add" ,"deleted": "delete" ,"deprecated": "deprecate" ,"kept": "keep", + "left alone": "leave alone" ,"made executable": "make executable" ,"merged": "merge", + "moved": "move" ,"removed": "remove", + "replaced": "replace", +} + +class Log: + def __init__(self ,preview): + self.preview = preview + self.count = {} + self.error_count = 0 + + def line(self ,tag ,rel ,reason=None): + if self.preview: tag = "would " + PREVIEW_TAG.get(tag ,tag) + text = f" {tag:<{TAG_WIDTH}}{rel}" + if reason: text += f" ({reason})" + print(text) + + def tally(self ,key ,n=1): + self.count[key] = self.count.get(key ,0) + n + + def error(self ,rel ,message): + self.error_count += 1 + print(f" {'error':<{TAG_WIDTH}}{rel} ({message})") + + def summary(self): + print("summary:") + order = [ + ("moved" ,"files moved to their place by the plan"), + ("deprecated" ,"files moved under " + DEPRECATED_DIR), + ("deleted duplicate" ,"duplicate files deleted"), + ("deleted placeholder" ,".githolder files deleted"), + ("deleted cache" ,"__pycache__ directories deleted"), + ("skeleton added" ,"skeleton files added"), + ("skeleton replaced" ,"skeleton files replaced"), + ("skeleton current" ,"skeleton files already current"), + ("skeleton kept" ,"skeleton files kept in the project's own version"), + ("merged" ,"files merged with the skeleton"), + ("merged current" ,"merged files already current"), + ("kept" ,"project files kept in place"), + ("made executable" ,"files made executable"), + ("left alone" ,"ignored files left in place, listed above"), + ("directory removed" ,"empty directories removed"), + ] + for key ,label in order: + print(f" {self.count.get(key ,0):>4} {label}") + if self.error_count: + print(f" {self.error_count:>4} errors") + if self.preview: + print("preview only: nothing was changed.") + + +# -------------------------------------------------------------------------------- +# reading git + +def git(directory ,*argument): + return subprocess.run( + ["git" ,"-C" ,str(directory) ,*argument], + capture_output=True ,text=True ,check=True + ).stdout + +def git_dirty(directory): + return git(directory ,"status" ,"--porcelain").strip() + +def project_file_list(project): + """ + The files git tracks or would track, that exist: tracked files, and + untracked files the project's .gitignore does not ignore. + """ + listing = git(project ,"ls-files" ,"-z" ,"--cached" ,"--others" ,"--exclude-standard").split("\0") + result = set() + for rel in listing: + if rel and ((project / rel).exists() or (project / rel).is_symlink()): + result.add(rel) + return sorted(result) + +def ignored_file_list(project): + listing = git(project ,"ls-files" ,"-z" ,"--others" ,"--ignored" ,"--exclude-standard").split("\0") + return sorted(rel for rel in listing if rel) + +def skeleton_file_list(harmony): + """ + The skeleton is the set of files in Harmony's last commit, as new-project + takes it. + """ + result = [] + for rel in git(harmony ,"ls-tree" ,"-r" ,"--name-only" ,"-z" ,"HEAD").split("\0"): + if not rel or rel in SKELETON_SKIP_FILE or in_tree(rel ,SKELETON_SKIP_TREE): + continue + result.append(rel) + return sorted(result) + + +# -------------------------------------------------------------------------------- +# paths + +def in_tree(rel ,tree_tuple): + return any(rel.startswith(t) for t in tree_tuple) + +def matches(rel ,path_tuple): + """A path names a file exactly, or, ending in /, the tree under it.""" + return any(rel == p or (p.endswith("/") and rel.startswith(p)) for p in path_tuple) + +def moved_path(rel ,move): + for old ,new in move.items(): + if rel == old: + return new + if old.endswith("/") and rel.startswith(old): + return new + rel[len(old):] + return None + +def directory_set(rel_list): + result = set() + for rel in rel_list: + parent = Path(rel).parent + while parent != Path("."): + result.add(parent.as_posix()) + parent = parent.parent + return result + +def content_hash(path): + if path.is_symlink(): + return "link:" + os.readlink(path) + h = hashlib.sha256() + with open(path ,"rb") as f: + for block in iter(lambda: f.read(1 << 16) ,b""): + h.update(block) + return h.hexdigest() + + +# -------------------------------------------------------------------------------- +# file operations; each tests preview immediately before acting + +def same_leaf(a ,b): + if a.is_symlink() or b.is_symlink(): + return a.is_symlink() and b.is_symlink() and os.readlink(a) == os.readlink(b) + return a.is_file() and b.is_file() and filecmp.cmp(a ,b ,shallow=False) + +def copy_leaf(src ,dst ,preview): + if preview: return + dst.parent.mkdir(parents=True ,exist_ok=True) + if dst.is_symlink() or dst.exists(): + dst.unlink() + if src.is_symlink(): + os.symlink(os.readlink(src) ,dst) + else: + shutil.copy2(src ,dst) + +def move_leaf(src ,dst ,preview): + if preview: return + dst.parent.mkdir(parents=True ,exist_ok=True) + os.rename(src ,dst) + +def delete_leaf(path ,preview): + if preview: return + path.unlink() + +def delete_tree(path ,preview): + if preview: return + shutil.rmtree(path) + +def write_text(path ,text ,preview): + if preview: return + path.parent.mkdir(parents=True ,exist_ok=True) + path.write_text(text ,encoding="utf-8") + + +# -------------------------------------------------------------------------------- +# merging + +def merged_gitignore(project_text ,harmony_text ,project_name): + """ + Harmony's lines, then a section holding the project's patterns that + Harmony lacks. The project's comments and blank lines are not carried. + """ + section = "# " + project_name + harmony_line = set(harmony_text.splitlines()) + extra = [] + for line in project_text.splitlines(): + pattern = line.strip() + if not pattern or pattern.startswith("#"): + continue + if pattern not in harmony_line and pattern not in extra: + extra.append(pattern) + text = harmony_text.rstrip("\n") + "\n" + if extra: + text += "\n" + section + "\n" + "\n".join(extra) + "\n" + return text + +def merged_version(project_text ,harmony_text ,project_name): + """Harmony's file, then the project's lines that do not name Harmony.""" + project_line = [l for l in project_text.splitlines() if l.strip() and "Harmony" not in l and l not in harmony_text.splitlines()] + text = harmony_text if harmony_text.endswith("\n") else harmony_text + "\n" + return text + "".join(l + "\n" for l in project_line) + +MERGE_FUNCTION = { + ".gitignore": merged_gitignore, + "shared/tool/version": merged_version, +} + + +# -------------------------------------------------------------------------------- +# the work + +def step_cache(project ,log ,preview): + for dirpath ,dirnames ,filenames in os.walk(project): + dirnames.sort() + if Path(dirpath) == project and ".git" in dirnames: + dirnames.remove(".git") + for name in list(dirnames): + if name in CACHE_DIR_NAME: + rel = Path(dirpath ,name).relative_to(project).as_posix() + delete_tree(project / rel ,preview) + log.line("deleted" ,rel + "/" ,"compiled Python cache") + log.tally("deleted cache") + dirnames.remove(name) + +def decide(rel ,plan ,skeleton_set ,skeleton_dir ,ignored_list): + action = decide_place(rel ,plan ,skeleton_set ,skeleton_dir) + if Path(rel).name == ".gitignore" and action[0] in ("move" ,"duplicate" ,"deprecate"): + prefix = Path(rel).parent.as_posix() + "/" + n = sum(1 for i in ignored_list if i.startswith(prefix)) + if n: + return ("keep" ,rel ,f"it hides {n} ignored file{'s' if n != 1 else ''}, left in place") + return action + +def decide_place(rel ,plan ,skeleton_set ,skeleton_dir): + """ + What becomes of one project file: (action ,new path ,reason). The action is + one of placeholder, keep, keep skeleton, move, duplicate, deprecate, or + skeleton, which the skeleton steps handle. + """ + name = Path(rel).name + if name in PLACEHOLDER_NAME: + return ("placeholder" ,None ,None) + if matches(rel ,plan["KEEP"]): + return ("keep skeleton" if rel in skeleton_set else "keep" ,rel ,None) + new = moved_path(rel ,plan["MOVE"]) + if new is not None: + return ("move" ,new ,None) + if matches(rel ,plan["DUPLICATE"]): + return ("duplicate" ,f"{DEPRECATED_DIR}/{rel}" ,None) + if matches(rel ,plan["DEPRECATE"]): + return ("deprecate" ,f"{DEPRECATED_DIR}/{rel}" ,"by the plan") + if rel in skeleton_set or rel in MERGED_FILE: + return ("skeleton" ,rel ,None) + parent = Path(rel).parent.as_posix() + if (parent == "." and (name in PROJECT_ROOT_FILE or name.startswith("0pus_"))) \ + or in_tree(rel ,PROJECT_TREE) or parent in skeleton_dir: + return ("keep" ,rel ,None) + return ("deprecate" ,f"{DEPRECATED_DIR}/{rel}" ,None) + +def step_sort(project ,plan ,skeleton_set ,file_list ,ignored_list ,log ,preview): + """ + Place every project file that is not a skeleton file. Every decision is + made before anything moves, so a duplicate can name where its twin ends up. + """ + skeleton_dir = directory_set(skeleton_set) + decision = {rel: decide(rel ,plan ,skeleton_set ,skeleton_dir ,ignored_list) for rel in file_list} + + # where the content of each file that stays in the project ends up + kept_at = {} + for rel ,(action ,new ,reason) in decision.items(): + if action in ("keep" ,"keep skeleton" ,"move" ,"deprecate" ,"skeleton"): + kept_at.setdefault(content_hash(project / rel) ,new) + + for rel in file_list: + action ,new ,reason = decision[rel] + if action == "placeholder": + delete_leaf(project / rel ,preview) + log.line("deleted" ,rel ,"placeholder; Harmony uses .gitkeep") + log.tally("deleted placeholder") + elif action == "keep skeleton": + log.line("kept" ,rel ,"the project's own version, by the plan") + log.tally("skeleton kept") + elif action == "keep": + if reason: log.line("kept" ,rel ,reason) + log.tally("kept") + elif action == "move": + place(project ,rel ,new ,"moved" ,f"to {new}" ,log ,preview) + elif action == "duplicate": + twin = kept_at.get(content_hash(project / rel)) + if twin: + delete_leaf(project / rel ,preview) + log.line("deleted" ,rel ,f"the same as {twin}") + log.tally("deleted duplicate") + else: + place(project ,rel ,new ,"deprecated" ,"no copy of it is kept" ,log ,preview) + elif action == "deprecate": + place(project ,rel ,new ,"deprecated" ,reason ,log ,preview) + +def place(project ,rel ,new ,tag ,reason ,log ,preview): + dst = project / new + if dst.exists() or dst.is_symlink(): + log.error(rel ,f"{new} already exists; nothing was moved") + return + move_leaf(project / rel ,dst ,preview) + log.line(tag ,rel ,reason) + log.tally(tag) + +def step_skeleton(project ,harmony ,skeleton_list ,plan ,log ,preview): + for rel in skeleton_list: + if rel in MERGED_FILE or matches(rel ,plan["KEEP"]): + continue + src = harmony / rel + dst = project / rel + if dst.exists() or dst.is_symlink(): + if same_leaf(src ,dst): + log.tally("skeleton current") + continue + copy_leaf(src ,dst ,preview) + log.line("replaced" ,rel ,"from the skeleton") + log.tally("skeleton replaced") + else: + copy_leaf(src ,dst ,preview) + log.line("added" ,rel ,"from the skeleton") + log.tally("skeleton added") + +def step_merge(project ,harmony ,project_name ,plan ,log ,preview): + for rel in MERGED_FILE: + if matches(rel ,plan["KEEP"]): + continue + project_path = project / rel + harmony_text = (harmony / rel).read_text(encoding="utf-8") + project_text = project_path.read_text(encoding="utf-8") if project_path.is_file() else "" + text = MERGE_FUNCTION[rel](project_text ,harmony_text ,project_name) + if text == project_text: + log.tally("merged current") + continue + if not project_path.exists(): + copy_leaf(harmony / rel ,project_path ,preview) + write_text(project_path ,text ,preview) + log.line("merged" ,rel ,"the skeleton's lines with the project's own") + log.tally("merged") + +def step_opus(project ,harmony ,project_name ,log ,preview): + rel = "0pus_" + project_name + if (project / rel).exists(): + return + write_text(project / rel ,"" ,preview) + if not preview: shutil.copymode(harmony / "0pus_Harmony" ,project / rel) + log.line("added" ,rel ,"the file that marks the project root") + log.tally("skeleton added") + +def step_executable(project ,plan ,file_list ,log ,preview): + """ + Give each EXECUTABLE path execute permission wherever it has read + permission. In a preview the file is found at its place before the move. + """ + before = {moved_path(rel ,plan["MOVE"]) or rel: rel for rel in file_list} + for rel in plan["EXECUTABLE"]: + path = project / rel + if not path.is_file(): + path = project / before.get(rel ,rel) + if not path.is_file(): + log.error(rel ,"EXECUTABLE names a file that does not exist") + continue + mode = path.stat().st_mode + wanted = mode | ((mode & 0o444) >> 2) + if wanted == mode: + continue + if not preview: os.chmod(path ,wanted) + log.line("made executable" ,rel) + log.tally("made executable") + +def step_ignored(project ,file_list ,ignored_list ,log): + """ + List ignored files outside scratchpad and consumer, by the highest + directory holding nothing git tracks. + """ + tracked_dir = directory_set(file_list) + group = {} + for rel in ignored_list: + part = rel.split("/") + if "scratchpad" in part[:-1] or part[0] == "consumer" or any(p in CACHE_DIR_NAME for p in part): + continue + top = rel + parent = Path(rel).parent + while parent != Path(".") and parent.as_posix() not in tracked_dir: + top = parent.as_posix() + "/" + parent = parent.parent + group[top] = group.get(top ,0) + 1 + for top ,n in sorted(group.items()): + log.line("left alone" ,top ,f"ignored by git; {n} file{'s' if n != 1 else ''}") + log.tally("left alone" ,n) + +def step_newly_ignored(project ,ignored_list ,log): + """ + A file the run placed where the merged .gitignore ignores it would not be + recorded by 'git add -A'. Such files are reported as errors. + """ + before = set(ignored_list) + for rel in ignored_file_list(project): + part = rel.split("/") + if rel in before or "scratchpad" in part[:-1] or part[0] == "consumer" or any(p in CACHE_DIR_NAME for p in part): + continue + log.error(rel ,"git ignores it now, so 'git add -A' will not record it; add it with 'git add -f'") + +def step_empty_dir(project ,log ,preview): + for dirpath ,dirnames ,filenames in os.walk(project ,topdown=False): + path = Path(dirpath) + if path == project or ".git" in path.relative_to(project).parts: + continue + try: + remaining = list(path.iterdir()) + except FileNotFoundError: + continue + if not remaining: + rel = path.relative_to(project).as_posix() + if not preview: path.rmdir() + log.line("removed" ,rel + "/" ,"empty directory") + log.tally("directory removed") + +def work(harmony ,project ,plan ,preview): + print(f"skeleton_update {'preview ' if preview else ''}run in {project}") + print(f" skeleton from {harmony}") + log = Log(preview) + project_name = project.name + + skeleton_list = skeleton_file_list(harmony) + skeleton_set = set(skeleton_list) + file_list = [r for r in project_file_list(project) if not any(p in CACHE_DIR_NAME for p in r.split("/"))] + ignored_list = [r for r in ignored_file_list(project) if not any(p in CACHE_DIR_NAME for p in r.split("/"))] + + step_cache(project ,log ,preview) + step_sort(project ,plan ,skeleton_set ,file_list ,ignored_list ,log ,preview) + step_skeleton(project ,harmony ,skeleton_list ,plan ,log ,preview) + step_merge(project ,harmony ,project_name ,plan ,log ,preview) + step_opus(project ,harmony ,project_name ,log ,preview) + step_executable(project ,plan ,file_list ,log ,preview) + step_ignored(project ,file_list ,ignored_list ,log) + if preview: + print(" (directories left empty are found and removed by run, not listed in a preview)") + else: + step_empty_dir(project ,log ,preview) + step_newly_ignored(project ,ignored_list ,log) + + log.summary() + return 1 if log.error_count else 0 + + +# -------------------------------------------------------------------------------- +# command line + +def read_plan(path): + plan = {"MOVE": {} ,"DEPRECATE": () ,"DUPLICATE": () ,"KEEP": () ,"EXECUTABLE": ()} + if path is None: + return plan + given = runpy.run_path(str(path)) + for name in PLAN_NAME: + if name in given: + plan[name] = given[name] + if not isinstance(plan["MOVE"] ,dict): + raise ValueError("MOVE must be a dictionary of old path to new path") + for name in ("DEPRECATE" ,"DUPLICATE" ,"KEEP" ,"EXECUTABLE"): + plan[name] = tuple(plan[name]) + for old ,new in plan["MOVE"].items(): + if old.endswith("/") != new.endswith("/"): + raise ValueError(f"MOVE '{old}' to '{new}': a directory moves to a directory, both ending in /") + return plan + +def print_help(): + print(__doc__.strip()) + return 0 + +def CLI(argv=None): + if argv is None: argv = sys.argv[1:] + if not argv or argv[0] in ("help" ,"-h" ,"--help"): return print_help() + + preview = argv[0] == "preview" + if preview: argv = argv[1:] + if not argv: + print("Error: 'preview' needs a command after it." ,file=sys.stderr) + return 1 + + command ,rest = argv[0] ,argv[1:] + if command != "run": + print(f"Error: unknown command '{command}'." ,file=sys.stderr) + print_help() + return 1 + if len(rest) not in (2 ,3): + print("Error: run is given []." ,file=sys.stderr) + return 1 + + harmony = Path(rest[0]).resolve() + project = Path(rest[1]).resolve() + plan_path = Path(rest[2]).resolve() if len(rest) == 3 else None + + if not (harmony / "0pus_Harmony").is_file() or not (harmony / ".git").exists(): + print(f"Error: {harmony} is not a Harmony checkout: it lacks 0pus_Harmony or .git." ,file=sys.stderr) + return 1 + if not (project / ".git").exists(): + print(f"Error: {project} is not a git work tree; the tool sorts the files git tracks." ,file=sys.stderr) + return 1 + if harmony == project: + print("Error: the project is the Harmony checkout itself." ,file=sys.stderr) + return 1 + if plan_path and not plan_path.is_file(): + print(f"Error: plan file {plan_path} does not exist." ,file=sys.stderr) + return 1 + try: + plan = read_plan(plan_path) + except Exception as error: + print(f"Error: plan file {plan_path}: {error}" ,file=sys.stderr) + return 1 + + dirty = git_dirty(project) + if dirty and not preview: + print("Error: the project work tree has uncommitted changes. Commit them first:" ,file=sys.stderr) + print(dirty ,file=sys.stderr) + return 1 + dirty = git_dirty(harmony) + if dirty: + print("Error: the Harmony checkout has uncommitted changes; the skeleton is taken" ,file=sys.stderr) + print("from its last commit, and its files would not match it:" ,file=sys.stderr) + print(dirty ,file=sys.stderr) + return 1 + + return work(harmony ,project ,plan ,preview) + + +if __name__ == "__main__": + try: + sys.exit(CLI()) + except BrokenPipeError: + os.dup2(os.open(os.devnull ,os.O_WRONLY) ,sys.stdout.fileno()) + sys.exit(1) diff --git a/developer/authored/.gitkeep b/developer/authored/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tester/scratchpad/.gitignore b/developer/authored/deprecated/tester/scratchpad/.gitignore similarity index 100% rename from tester/scratchpad/.gitignore rename to developer/authored/deprecated/tester/scratchpad/.gitignore diff --git a/developer/design_AccountTree.html b/developer/design_AccountTree.html new file mode 100644 index 0000000..8866744 --- /dev/null +++ b/developer/design_AccountTree.html @@ -0,0 +1,761 @@ + + + + + AccountTree 5.0 Design + + + + + + + + + + + +

Purpose

+ +

+ AccountTree keeps light containers made of Linux accounts. A person's work is divided among accounts, each with its own files and processes, so that a web browser, an email program and a development project cannot read each other's files, without the cost of a virtual machine. This document gives the design as it is currently envisioned, for the standard numbered 5.0. It covers the accounts, the storage that keeps their data, the systems the storage is attached to, the databases, the requests through which accounts and mounts are changed, the programs that make them, and login. Where a decision is still open it is marked as open. The features planned for later standards are listed at the end. +

+ +

+ document/doc_2.html gives the reasoning behind the flat home directories and the idmapped views. It was written in the earlier terms described below; this document builds on it. +

+ +

Ontology

+ +

Why the terms changed

+ +

+ The project was called SubU, for subordinate user, and spoke of a MasU, a master user, owning its SubUs. In Linux the word user is overloaded. It names the person at the keyboard, human or AI, and it names the account in /etc/passwd that the person uses, as in useradd and user name. The two are different things: one person uses many accounts, and an account has no will of its own. The design came to need the difference everywhere. Two accounts of one person are not the same account, while an account on a system and an account in a storage database can be one account in two places. With one word for both, every sentence about them was ambiguous. +

+ +

+ The MasU and SubU terms had a second fault. Some passages spoke as though there were one MasU at the top, and others as though every SubU could be a MasU. Both were true, because master and subordinate are relative, but the terms gave no word for the top, or for an account with nothing below it. +

+ +

+ So the word user is dropped. A person is the agent; an account is the account. The structure is described in the terms of a tree, which are relative where they should be and fixed where they should be. The project is renamed AccountTree, for what it keeps, and SubU and MasU are retired. The word user survives only in names the project does not own: uid, which is short for user id, useradd and userdel, and the kernel's user namespaces. +

+ +

The account tree

+ +

+ Every account on a system has one parent, and root is at the top. In Linux root already administers every account, so making root the top of the tree describes what is already so, and the tree fits into Linux without changing its shape. Root keeps its one meaning. +

+ +
    +
  • person: the agent who logs in and owns data in the world, human or AI.
  • +
  • account: a Linux account, with its own uid, home, processes and sessions.
  • +
  • account tree: the accounts of a system, with root at the top.
  • +
  • parent account: the account directly above an account. child account: an account directly below one.
  • +
  • login account: a child account of root.
  • +
  • leaf account: an account with no child accounts.
  • +
+ +

+ Every account is entered through its parent. Root enters its child accounts through the system's login programs, the display manager, login at the console and sshd, which run as root and start a session for a person who gives a password or a key. A login account enters its child accounts through AccountTree's request gate, which starts sessions and programs in them; an account below a login account has no password, and the login programs do not enter it. The rule is the same at every level of the tree. Only the means differ, and the means for root's children are the ones Linux already has. +

+ +

+ A parent account reaches a child account's files through a view. +

+ +
    +
+ +

+ An account's name is its parent's name, the middle dot, and its own short name: Thomas·developer·scratch is a child of Thomas·developer, which is a child of the login account Thomas, which is a child of root. In identifiers the wider category comes first, so prose says child account and parent account, and code says account_child and account_parent. +

+ +

Term

+ +
    +
  • storage: a persistent storage device carrying accounts' data, such as the SSD inside a laptop or an encrypted USB drive. Its root is the directory it is mounted at. A storage can be moved between machines.
  • +
  • tree dir: the directory <storage root>/home/AccountTree, holding the storage database and the accounts' homes. On the disk a system runs from it is /home/AccountTree. AccountTree_admin is given a tree dir to name a storage.
  • +
  • system: an installed, running Linux, with its own /etc/passwd, its own uid numbering, and its own /home, into which the accounts kept on attached storage are mounted.
  • +
  • system account: an account in a system's /etc/passwd. storage account: an account in a storage database.
  • +
  • storage database: the SQLite database in a tree dir, describing the storage accounts kept on that storage.
  • +
  • system database: the SQLite database at /var/lib/AccountTree/system.sqlite3, a system's own record. Its system map is the authority for which storage accounts are the same accounts as system accounts.
  • +
  • database uuid: the random identifier a storage database is given when it is made. It names the database, not the storage: a copy of the database carries it too. It is not the storage uuid that a file system is given when it is formatted, which AccountTree does not use.
  • +
  • system uuid: the random identifier a system database is given when it is made, which names the system.
  • +
  • database uid: a storage account's uid in its storage database's own numbering. The account's files on the storage carry it.
  • +
  • system uid: a system account's uid, in /etc/passwd.
  • +
  • account specifier: the name of an account in one place: system <host name> <system account>, or storage <tree dir> <storage account>.
  • +
  • same account: two account specifiers that name one account in two places. The system map records which system accounts are the same accounts as which storage accounts.
  • +
  • view: an idmapped mount through which a parent account sees a child account's files as its own.
  • +
  • request: what a person asks of AccountTree, such as making a child account. A request is an instruction of the request language, given in a journal to the request gate.
  • +
  • gate, guard, interpreter: Man_In_Grey's parts. A journal given to a gate is checked by its guard, and played by its interpreter only when the guard approves it. The gate invoking account is the account that started the gate, which the kernel reports as its real uid; the gate owner account is the one whose effective uid the guard and interpreter run with. Man_In_Grey.html gives the design.
  • +
  • database account: the system account AccountTree, which owns AccountTree's databases and is the only account that opens them, besides root. It holds no other privilege.
  • +
  • answer: the result one interpreter hands back to the program that invoked its gate, on descriptor 3, which that program checks before it acts on it.
  • +
  • unmanaged account: an account AccountTree does not manage, such as root, a service's account, or an account made with useradd by hand.
  • +
+ +

Account and name

+ +

Account name

+ +

+ The middle dot (U+00B7, written ·) is not permitted in a short name, so a full name divides in exactly one way, and the conventional characters _, - and . remain free for use in names. It lies outside the POSIX portable character set. Making an account needs the --badname flag of useradd, and a tool that checks account names against the portable set needs its check widened. In UTF-8 the middle dot is two bytes, and an account name is currently limited to 32 bytes. That limit is set in user space, by the fixed name field of the utmp login records and by useradd, not by the kernel, which knows accounts only by uid. pkg-config prints paths holding the middle dot in a form that is not valid UTF-8; builds that pass such paths through pkg-config need the workaround given in shared/project-dependency/Python.org. +

+ +

+ The name AccountTree is reserved, because on a running system /home/AccountTree holds AccountTree's own files. A system account made for a branch of a storage database is named with a tilde and eight hex digits, as Thomas·try~a7b3247f, described under Copies. That is a default, not a rule on names: the system map, not the name, records which system accounts belong to a branch. +

+ +

Identity of an account

+ +

+ A storage account has an identity in its storage database, and, when the storage is attached to a running system, an extended identity on that system. +

+ +

+ In its storage database, an account is identified by the database uuid together with its database uid. Its files on the storage are owned by that database uid, and root, mounting the storage by hand without AccountTree, sees that database uid on them. This pair is all the storage database needs to identify the account. +

+ +

+ On a running system, the account also needs a system uid, because that numbering is the one /etc/passwd and the kernel use there. A row of the system map gives it: the row pairs the storage account's identity with a system account. The account's extended identity is its identity in its storage database, joined through the system map to the system. +

+ +

Storage and system

+ +

Storage layout

+ +

+ Every storage has the same layout under its root, where the root is the directory the storage is mounted at: / for the disk a system runs from, or a mount point such as /mnt/Eagle for a drive plugged in. Its tree dir is home/AccountTree under the root, so /home/AccountTree or /mnt/Eagle/home/AccountTree. +

+ + + home/AccountTree/ the tree dir: root's, mode 0711 + home/AccountTree/database/ the database account's, mode 0700 + home/AccountTree/database/storage.sqlite3 the storage database + home/AccountTree/account_data/ root's, mode 0700 + home/AccountTree/account_data/<name> the home of each storage account, by name + home/<other> anything else, such as the homes of unmanaged accounts + + +

+ Everything AccountTree keeps on a storage is under its tree dir, so the storage can carry other things beside it, and the storage database sits with the data it describes. account_data holds the accounts' homes; on a running system each is reached through an idmapped mount at /home/<system account>. An account named database cannot collide with the database, because the two are at different levels. +

+ +

+ The ownership divides the tree dir between the two accounts that act on it. SQLite writes a journal file beside a database while a transaction is open, so the account that writes a database needs to write the directory it is in; the database is therefore in a directory of its own, database/, which the database account owns. Everything else is root's. The database account can pass through the tree dir, mode 0711, to reach its directory, but cannot enter account_data, so it cannot rename, move or read a home. A root interpreter that checks a home's owner on disk is then checking something the database account could not have arranged; see Answer check. +

+ +

+ On the disk a system runs from, AccountTree's files are therefore under /home, beside the homes of unmanaged accounts. Distributions already keep /home apart from the operating system: openSUSE on btrfs makes it a subvolume outside the root snapshots, so a rollback of the root leaves it alone, and installers on other file systems commonly offer it as a partition of its own, kept when the system is reinstalled. AccountTree's files inherit this without depending on any file system. Were they on the root file system, a rollback of the root would roll back the storage database and the accounts' files with it. +

+ +

+ A tool that walks /home sees each mounted account's files twice: at /home/<system account>, and at /home/AccountTree/account_data/<name>. A bind mount of the same file system reports the same device number, so options such as rsync -x do not stop at the mount. A backup of /home excludes one of the two. +

+ +

Every storage is the same

+ +

+ A storage is a repository of accounts' data, whether it is built into the machine or plugged in, and a machine can have several. The SSD inside a laptop is attached at boot, before any login, in the same way a USB drive is attached when it is plugged in. Taken out of the laptop and put into a USB enclosure, the same SSD can be attached to another machine, and its accounts used there. +

+ +

Copy and branch

+ +

+ The database uuid names the database, not the storage it is on. A copy of a tree dir, made for a backup or an archive, carries the same database uuid, and is the same database until one of the two is changed. Because a tree dir can be anywhere, an archive kept at /<storage name>/<date>/home/AccountTree can be attached by giving that tree dir. +

+ +

+ A storage moved between systems, plugged into a desktop, then a laptop, then the desktop again, is the ordinary case. A copy used apart from its original is the case to notice: a storage restored from a backup after the first was lost, or a lost drive found again after work went on on its replacement. The record of transactions tells these apart, as git's history does: +

+ +
    +
  • Every attach and every detach is a transaction in the storage database. Two copies used apart therefore have different heads, even when no account was made or changed on either.
  • +
  • The system records every head of each storage database it sees, in its storage_head_seen table: when one of its accounts is made the same account as a system account, and at each attach and detach.
  • +
+ +

+ At attach, the storage's record is compared with those heads: +

+ +
    +
  • forward: its record holds the last head seen. It is the storage seen before, moved on, perhaps on another system. It attaches without comment. The system does not care how far along the storage has gone, only that it has not gone back, or sideways.
  • +
  • backwards: its head is one seen earlier, and another state of the database has been attached since. It is an older copy, or the other line of a branch.
  • +
  • sideways: its record holds an earlier head seen, but not the last one. It is a branch, two copies each changed after they parted, or an older copy whose head this system never saw.
  • +
  • unrelated: its record holds no head seen here; it parted before this system first saw it.
  • +
+ +

+ A storage gone backwards, sideways or unrelated is attached with a warning, and the warning and the kind are recorded. No one is kept from working: a person restoring from a backup attaches the restored storage and goes on, and its line becomes the one this system follows. Merging the two lines afterwards is done by hand in 5.0. +

+ +

+ The two lines can also be kept apart, so that both can be attached at once and their files compared. attach <tree dir> branch makes the storage a branch: it gives the database a new database uuid, records in it the database it came from and the head they parted after, and makes a system account for each storage account that had one on the line it came from, named <system account>~<eight hex digits>. The eight digits begin the hash of the transaction that made the branch, which no other branch has, and which does not change as either line grows. The branch is then a storage database in its own right. +

+ +

+ The same database attached twice would mount two homes at one place. When a storage's homes are already mounted from another copy of it, attach mounts nothing, and says to detach the other copy or attach this one as a branch. +

+ +

+ Bringing two lines together again is a merge. For the storage database, a merge replays the transactions of one line onto the other where they do not conflict. For the accounts' files, it is the integration problem of 7.0, of which merging two lines of one storage is the simplest case. +

+ +

+ These checks establish continuity, not authenticity. The hashes are not secret: they are in the storage database and the system database, and anyone who can read either can copy them into a fabricated database. Authenticity needs a secret, which is the subject of the keys planned for 6.0. +

+ +

Uids are a storage database's own

+ +

+ The uids in a storage database are that database's own numbering. Files on the storage carry them. A storage database never adopts the uids of a system it is attached to; its accounts are mounted onto the system through idmapped mounts, which translate database uids to system uids as the files are used. A storage can therefore be moved from machine to machine without changing its files. +

+ +

+ A storage database gives out database uids counting down from 1999999999, while systems give out system uids counting up from low numbers. A database uid is therefore not also the system uid of an account, and a file on a storage, seen without its mount, does not appear to belong to a system account it does not belong to, except in the cases given under Limit of a uid. The band 1879048192 to 2147352575 is left unused by systemd's allocation of uids, so a storage database has about 120 million database uids before it leaves the band. An account brought into AccountTree from an existing one can keep its existing uid, with the keyword uid of account make; for that account the coincidence remains, behind the root-only directories of the tree dir. +

+ +

+ A storage database never reuses a database uid. A database uid given up by a deallocated account stays retired, because files carrying it can remain on the storage, in a backup, or on another storage they were copied to. +

+ +

Limit of a uid

+ +

+ Two numberings meet on a running system: database uids, on the files of storage, and system uids, in /etc/passwd. Each has limits, set by different parties, and each limit is checked where the party that sets it is known. +

+ +

+ The kernel. Linux uids are 32 bits. 4294967295 is (uid_t)−1 and means no uid; 65534 is the overflow uid shown for a uid that cannot be mapped; neither is given out. A database uid is never used as a system uid. At run time it meets only the idmapped mount, which accepts any uid below 4294967295, so the kernel's limit is the same on every system and a storage moved between systems is unaffected by it. +

+ +

+ The file system. ext4, xfs and btrfs store 32-bit uids. JFFS2 and Minix store 16 bits, and FAT and exFAT store none; such storage cannot carry AccountTree accounts. This limit belongs to the storage and travels with it, so it is checked once, when the storage is made: storage make sets the owner of a test file to the first database uid, reads it back, and refuses the storage if the two differ. That check is not yet built. +

+ +

+ The system. UID_MAX in /etc/login.defs bounds only the uids useradd picks on its own; AccountTree gives the system uid. A system's range of system uids is set by system make, 2000 to 59999 by default, and is the system's own. A storage moved to a system with a smaller range keeps its database uids, since they never become that system's uids; the smaller range limits only how many of the storage's accounts the system can make system accounts for, and is-same-account reports when the range is used up. +

+ +

+ Where the two numberings can still meet. A database uid equals the system uid of some account when: +

+ +
    +
  • an account was brought in with the keyword uid of account make, keeping its existing uid. This is deliberate.
  • +
  • a system's own range reaches into the band. system make refuses a range that reaches 1879048192.
  • +
  • accounts from outside AccountTree have uids in the band. A directory service mapping Active Directory accounts through SSSD gives out slices of 200000 uids up to 2000200000 by default, which is inside the band, and container uid ranges in /etc/subuid can be set anywhere.
  • +
  • a storage database has given out nearly all its band, after about 120 million accounts. account make refuses below 1879048192. Should a database ever need more, a later standard chooses the next band and records it in the database, so that databases already made keep theirs.
  • +
+ +

+ At attach, the database uids of the storage are to be compared with the system uids in use, from the system's account database, which includes directory services, and from /etc/subuid. Each coincidence is logged with the system account that would appear to own the raw files. A coincidence is not refused: through the mounts each account sees its own files correctly, and the raw files are behind account_data, which only root can enter. It is reported because a tool that works by uid on the raw files, run as root, would treat them as that account's. This check is not yet built. +

+ +

+ Tools that copy files. Backups and archives copy uids with the files. The old ustar format of tar holds uids up to 2097151, and the odc format of cpio up to 262143; database uids do not fit. GNU tar's default format, the POSIX pax format, cpio's newc format, rsync and btrfs send hold 32-bit uids, and archives of AccountTree storage use one of them. +

+ +

The system

+ +

+ A system is a party in its own right, with its own database. Its system database, at /var/lib/AccountTree/system.sqlite3, holds its system uuid, the range of system uids AccountTree gives out, the system map, every head of each storage database it has seen, and the limits. The system owns the uid numbering of /etc/passwd: it alone decides which storage accounts get system accounts on it and with which system uids, so that record is its own and not any storage's. A storage plugged in can claim nothing; what it names in its own database grants it no system account. +

+ +

+ A system never reuses a system uid. Its own disk keeps files carrying system uids, in /tmp, /var/spool, crontabs and logs, and a reused uid would hand them to another account. The system map therefore keeps every row it has held, retired rows included, and a system account AccountTree makes is never given a system uid that appears in it. +

+ +

+ The system database is under /var, which holds the state programs keep, and not under /etc, which holds configuration. A system whose root file system is snapshotted, as openSUSE's is, keeps /var outside the snapshots. A rollback of the root then rolls back /etc/passwd and leaves the system map as it was: a system account made after the snapshot is lost from /etc/passwd, but its system uid stays recorded, is not given out again, and the system account can be made again from the system map. Were the system database under /etc, the rollback would also forget the system uid, and it could be given to another account while files carrying it remained under /var. +

+ +

Deleting storage

+ +

+ AccountTree_admin storage delete <tree dir> undoes, on this system, what AccountTree did for the accounts of the storage database, then deletes the tree dir. It makes every check first, and a failed check changes nothing: +

+ +
    +
  • the storage database names no other system that has seen it, since that system's map would name a database that no longer exists, unless the keyword forget-other-systems is given;
  • +
  • no system account that is the same account as one of its accounts has a login session;
  • +
  • nothing is mounted inside the tree dir;
  • +
  • the tree dir holds only the storage database and account_data, and account_data holds only homes of the database's accounts;
  • +
  • homes that hold files are deleted only when the keyword delete-account-data is given.
  • +
+ +

+ For each row of this system's map naming one of its accounts, it then unmounts the home, deletes the mount point attach made, recognized as an empty directory owned by root with mode 0000, deletes the system account and its group when AccountTree made them, keeps an adopted system account, and retires the row. Last it deletes account_data, the storage database and the tree dir. Run a second time, it finds nothing to delete and says so. system delete deletes the system database, and is refused while any row of the system map is in use. +

+ +

Database

+ +

+ There are two kinds of database, with two schemas, schema_storage.sql and schema_system.sql. Both keep the same record of changes, described below, and are checked the same way. Each fact is kept by the party with authority over it: a storage database keeps its accounts and their database uids; the system database keeps which of those accounts are the same accounts as system accounts. +

+ +

Storage database table

+ +
    +
  • identity: the database uuid, the uuid it was made with, from which its chain begins, and for a branch, the database it came from and the head they parted after; the series number, the head hash, and when it was made. One row. A storage database has no name for people. A name would travel with every copy of the database, and a name taken from the storage it was made on would come to mislead once the database was copied elsewhere. A database is shown by its database uuid, its series number, its head hash and the time of its last change, and log lines give the first eight digits of the database uuid.
  • +
  • standard: the AccountTree standards this database follows, one row each. Currently 5.0.
  • +
  • account: one row per storage account: database uid, name, the database uid and name of its parent, state (active, deprecated, deallocated), and times. The parent's database uid is empty when the parent is root or is carried by another database.
  • +
  • database_uid: every database uid this database has ever given out, with whether it is in use or retired.
  • +
  • system_seen: the systems that have seen this database, with when that was first and last. For the owner of the storage, and for storage delete, which will not leave a system mapping a database that no longer exists. Never authority.
  • +
  • acquaintance_database and same_account_database: other storage databases this one knows, and which of their accounts are the same accounts as accounts here, by database uid. Written into both databases. Not used in 5.0.
  • +
  • transaction_log: the record of every change, described below.
  • +
+ +

System database table

+ +
    +
  • identity: the system uuid, the series number, the head hash, and when the database was made. One row. The system is shown by its host name, read when it is shown, so a name for people is not kept.
  • +
  • standard: as for a storage database.
  • +
  • system_uid_range: the first and last system uid AccountTree gives a system account it makes. One row.
  • +
  • system_map: one row for each time a system account has been made the same account as a storage account: the system uid and gid, the system account's name, the database uuid and database uid, whether AccountTree made the system account or adopted one that existed, whether the row is in use or retired, and when. At most one row in use names a given system uid, and at most one names a given storage account. A row is retired, never deleted. An adopted system account can be made the same account again once its row is retired; a system uid that appears in any row is never given to a system account AccountTree makes.
  • +
  • storage_head_seen: every head of each storage database this system has seen, with the event (mapped, attached, detached), and at attach, the continuity found.
  • +
  • account_limit: the number of child accounts each system account is allowed below it, by system uid. A system account with no row is allowed none.
  • +
  • transaction_log: the record of every change.
  • +
+ +

Series number and hash chain

+ +

+ A transaction is one change to one database: the SQL statements a command of AccountTree_admin, or a request, runs against that database, recorded together. A command can make transactions in both kinds of database; is-same-account makes one in the storage database and one in the system database. +

+ +

+ Each transaction increments the series number, and records its statements, a description, a random number, and a hash. The hash is taken over the previous transaction's hash, the series number, the description, the statements, and the random number. The first transaction follows a hash made from the database's kind and the uuid it was made with, so no two databases' chains can coincide, and a record cannot be moved from one database into another. A branch keeps its record, and the transaction that made it a branch is in that record. The identity row holds the series number and hash of the last transaction, the head of the chain. +

+ +

+ The chain gives order, integrity and identity of history. It does not give authenticity. Anyone holding a database can extend its chain, or rewrite it and compute new hashes. What a rewrite cannot do is keep the hashes that others have already seen: a system that remembers a head finds that the rewritten record does not hold it. Each party that remembers a head is a witness. A storage database and the system database are readable only by the database account and root, so the record is the database's own history, as a git repository's history is, rather than a log for persons to read. +

+ +

+ The series number orders a database's history; the hash identifies it. A copy that is used goes on to make its own transactions, with the same series numbers as the original's and different contents. Their hashes differ, because the random number differs even when the same change is made to both. So two databases with the same database uuid and series number but different heads are known to have parted. +

+ +

+ Times recorded in a database are for people to read. They are not authoritative, because clocks differ between machines. The series number is authoritative. +

+ +

Standard

+ +

+ A program that opens a database first reads its standard table, and refuses a database that follows no standard the program follows. During a change from one standard to the next, a database lists both. +

+ +

Fallback tool

+ +

+ The databases can be built by hand from SQL scripts kept in the project. They are the means of starting over if a database is lost or damaged. +

+ +
    +
  • A schema script builds an empty database where there was none.
  • +
  • Initialization scripts put in the starting data: the identity row, the standard, existing accounts, the uid range, and limits.
  • +
  • Scripts with their data written into them can write the system map rows for existing system accounts, for a system whose system database is lost.
  • +
+ +

Same account

+ +

+ A storage account's home, account_data/<name> in its tree dir, carries only its database uid. What lets it be used on a system is a row of that system's map, saying that a system account there is the same account as the storage account. The administrator states it: +

+ + + AccountTree_admin is-same-account system <host name> <system account> storage <tree dir> <storage account> [make-system-account] + + +

+ A keyword such as make-system-account is a word given after a command's other arguments, as the command itself is a word, not a flag. The two account specifiers are accepted in either order. In 5.0 the pair is a system account and a storage account; correspondences between two storage databases come later. The host name must be this system's, which confirms that the administrator is on the machine he means. When the system account exists, it is adopted: it must be an account of a person, with a system uid at or above UID_MIN, its home must be /home/<system account>, and it must not already be the same account as another storage account. When it does not exist, the keyword make-system-account makes it, with a system uid from the system's range that no account, no group and no row of the system map has had, provided /home/<system account> is absent or empty, since the new system account would take whatever is there as its home. Without the keyword the command refuses, so that a mistyped name does not make a system account. +

+ +

+ The first time a storage account is made the same account as a system account on a system, the storage database records that system in system_seen. That and the system map row together are what 5.0 calls an introduction; in 6.0 it is also when keys are exchanged. account make with make-system-account makes a storage account and, in the same command, a system account of the same name, the same account as it. +

+ +

+ is-not-same-account undoes it: it unmounts the home if it is mounted from that storage, deletes the mount point attach made, deletes the system account if AccountTree made it and keeps an adopted one, and retires the row. same-account show lists which accounts are the same, with the storage accounts and the persons' system accounts that are not. same-account suggest prints is-same-account commands proposed by matching names, for the administrator to read and run; it changes nothing. Matching names is a guess, and is used nowhere else. +

+ +

+ Someone else's storage, plugged into this system, gets nothing until the administrator makes its accounts the same accounts as system accounts here. There is no guest attach in 5.0: every storage is introduced by the administrator. A storage account named Thomas on that storage is no one here until the system map says so. +

+ +

+ Two storage databases can record that an account of one is the same account as an account of the other, and can record it while a system's map says otherwise. Such a claim grants nothing until a system acts on it, as in mounting a parent account's view of a child account kept on other storage. A system acts only on its own record: it does not infer that two accounts are the same from what storage databases say, and where the claims of storage databases and its own map disagree, it reports the disagreement. A child account made through a request on this system was made by this system, which records the parent relation itself. +

+ +

+ In 5.0 storage databases and systems recognize each other by database uuid and system uuid alone. A copy of a storage database is taken for the original, and gains the system accounts mapped to the original's accounts. That is enough among one person's machines and storage. It is not enough among strangers, because either kind of uuid can be copied; keys are given under standard 6.0. +

+ +

Request

+ +

No server

+ +

+ A person's program asks for what it wants by composing a journal and invoking a gate with it: AccountTree's program is its own Man_In_Grey orchestrator, and no server stands between the person and the gate. The gate's real uid is the account that asked, reported by the kernel, so the guard knows who asks as a fact rather than as a claim in the journal. The place where authority is decided and the place where privilege changes are then one place, the gate, which is what Man_In_Grey was built to be. +

+ +

+ An earlier form of this design had a server, running under an account of its own and receiving requests on a socket. The server learned the requesting account from SO_PEERCRED, but every journal it composed reached a root gate with the server as the gate invoking account, so the root guard would have had to trust the server's word for who asked. The server also wrote the system database, while the root guards checked their constraints against that database, so a compromised server could have written a record and then asked root to act on it. Taking the server away removes both. +

+ +

+ A gate is a function call. The program that invokes it waits, and is given back the gate's exit status and, on descriptor 3, the interpreter's result. A request's answer is therefore tied to the request by the kernel: it arrives on the descriptor of the gate the program started. +

+ +

The gate

+ +
    +
  • The request gate, AccountTree_gate·request, setuid root. Any account invokes it. Its guard and interpreter run with root's effective uid.
  • +
  • The record gate, AccountTree_gate·record, setuid to the database account, owned by it with the group root and mode 4750, so that only root can start it. A root interpreter invokes it to have the databases read or written; its guard and interpreter run with the database account's effective uid.
  • +
  • The administrator's gate, AccountTree_gate·admin, not setuid, mode 0700, owned by root. AccountTree_admin invokes it under root; its guard's constraints allow only root.
  • +
+ +

+ The administrator's gate needs no setuid bit, since root invokes it already holding root's rights, and without the bit the administrator's instructions have no setuid program to reach them by. It is a gate all the same, so that the administrator's journals pass a guard as a person's do, and the code that changes the system lives in one place, the interpreters. Making accounts the same, attaching, detaching and setting limits all go through it, at boot as at any other time. +

+ +

The database account

+ +

+ The databases are opened by the database account and by no root process. A storage database is input from a drive: a copy, a branch, or a drive found again after it was out of its owner's hands, whose file anyone could have rewritten. Opening it means parsing its pages and its schema and walking its whole hash chain, and SQLite's own guidance treats a database file from an untrusted source as an attack surface. Under the database account, a fault that a crafted database reaches gives that account, not root. This is privilege separation in the manner of OpenSSH: the complex parsing of untrusted input runs unprivileged, and the privileged program does a few simple things with what it is told. +

+ +

+ The database account owns /var/lib/AccountTree, where the system database is, and on each storage the tree dir's database/ directory. Its interpreter opens the databases with SQLite's defensive settings, trusted_schema off among them. +

+ +

Who is asking

+ +

+ The request gate records the gate invoking account, its real uid, for its guard and interpreter, as every Man_In_Grey gate does; the account cannot misstate it. That account's name is the prefix of any child account it makes. At the record gate the gate invoking account is root, and the journal names the requesting account as an argument. The record guard trusts that argument because the account that gave it is root, which could do anything anyway; its constraints allow only root to give it journals. +

+ +

The request language

+ +

+ The request language is made of requests, not of the primitive steps that carry them out. A journal that asked for a system uid to be made, or a path to be mounted, would leave the guard to work out what request the steps add up to, and to check every uid and path the composer chose against records it would have to consult again. A request names the work, and the uids, paths and mount options are the interpreter's to choose; none of them can be expressed in the language. +

+ +

+ Each request acts on the requesting account or on an account in the tree below it, with the authority of the requesting account; this is how a child account's window, opened from its parent's, acts with the parent's authority. Each has a result, which the person's program shows. +

+ +
    +
  • make_child_account <short name>: make a child account of the requesting account. Checks: the short name is valid and not in use; the requesting account's limit, and the limit of every account above it in its tree, allow one more. The child account is made in the storage database of the requesting account unless another attached storage is named. Steps: a database uid, never given before, and the account row in the storage database; a system uid, never given before, and the row in the system map; the system account; the home in account_data, owned by the database uid, mode 0700; the mount of it at /home/<name>; the parent's view. Result: the account's name, database uid and system uid.
  • +
  • rename <account> <new short name>: rename a child account. The account, and every account below it, whose names begin with its name, must have no session; all are renamed together. The uids and files do not change; the homes, mount points and views are renamed with them.
  • +
  • deprecate <account>: end the account's sessions, lock it, and mark it deprecated. Its data, name and uid are kept. It still counts against the limits. A deprecated account can be restored.
  • +
  • deallocate <account>: remove a deprecated account. Its system account and views are removed, and its database uid and system uid are retired. What becomes of its data is open: the proposal is to move it to a holding directory on the storage, from which only the administrator removes it.
  • +
  • view [<account>]: report an account and its tree: names, states, storage, and whether each is mounted. Changes nothing. The report is the result, so a person's program learns the records without being able to open the databases.
  • +
  • set_views: mount the requesting account's views of its child accounts that are on attached storage. Views already mounted are left as they are.
  • +
  • enter <account> [<program>]: start a session, or a program, in an account below the requesting account.
  • +
+ +

+ reconcile, which compares the databases with /etc/passwd and the mount table, completes what an interrupted request left undone, makes again any system account lost from /etc/passwd, and reports what differs, is an instruction of the administrator's language, run at boot and on demand. +

+ +

A request's course

+ +

+ For make_child_account: +

+ +
    +
  1. The person's program composes a journal of the one request, and invokes the request gate with descriptor 3 open.
  2. +
  3. The request guard approves it: the journal is well formed, the gate invoking account is a person's account, with a system uid at or above UID_MIN, and the short name is valid. Whether the account is managed, and what it is allowed to do, are questions for the records, asked next.
  4. +
  5. The request interpreter composes a journal for the record gate, naming the requesting account and the short name, and invokes it.
  6. +
  7. The record interpreter checks the tree and the limits against the records, gives out the database uid and the system uid, each within its database's write transaction, records the account and its intended state, and answers with the name, the two uids, and the series number and head hash of each transaction.
  8. +
  9. The request interpreter checks the answer, Answer check below, then makes the system account, the home, the mount point, the mount and the parent's view.
  10. +
  11. The request gate hands back the result: the account's name and uids.
  12. +
+ +

+ A refusal from the record interpreter, for a limit for instance, comes before anything on the system has changed, since the records are asked first. The series numbers and head hashes in the answer are logged with the changes the request interpreter makes, so that every change to the system is tied to the database transaction that recorded it. +

+ +

Giving out a uid

+ +

+ Each database gives out its own uids, and its write lock is what keeps two requests from being given the same one. The choice of the next uid is made inside the write transaction, opened with BEGIN IMMEDIATE, which takes the lock before anything is read; a uid chosen before the transaction opens could be chosen by two requests at once. The code as it stands chooses first and opens the transaction after, in account make and in is-same-account, and is to be changed. +

+ +

+ One check stays outside the lock: that no account made by hand already has the system uid. An account made with useradd between the record interpreter's choice and the request interpreter's useradd -u would hold it. useradd then fails rather than making a duplicate, the request fails, and since the system database recorded the uid first, reconcile finds the request half done. +

+ +

Answer check

+ +

+ The request interpreter trusts the database account less than it trusts itself, so it checks each answer the way a guard checks a journal, against what it asked and against facts the database account cannot forge: +

+ +
    +
  • the account's name is the requesting account's name, the middle dot, and the short name asked for;
  • +
  • the system uid lies in the system's range and is held by no account in the system's account database, and the database uid lies in the band;
  • +
  • before a home is mounted, its owner on disk, read with stat, is the database uid the answer gives; the database account cannot enter account_data, so it cannot have arranged the owner.
  • +
+ +

+ The third check is the one that matters most. An answer that gave one account another's database uid would otherwise mount the second account's files as the first's. +

+ +

Instruction language and constraint

+ +
    +
  • Request, at the request gate: the requests above. Guard: the journal is well formed; the gate invoking account is a person's account; names are valid. The checks that need the records, the tree and the limits, are the record interpreter's, made before anything changes.
  • +
  • Record, at the record gate: the record work each request and each of the administrator's commands needs, such as allocating a child account, mapping a system account to a storage account, recording an attach or a detach, and reading the tree for a view. Guard: the journal is well formed; the gate invoking account is root.
  • +
  • Administrator, at the administrator's gate: system make, storage make, account make, is-same-account, attach, detach, limit set, reconcile and the others of AccountTree_admin. Guard: the journal is well formed; the gate invoking account is root.
  • +
+ +

+ The constraints that limit what root's interpreters do with what they are told stay with them: a new system uid lies in AccountTree's range and appears in no row of the system map; a system account is deleted only when the system map records that AccountTree made it; a home is mounted only from <tree dir>/account_data/<storage account> onto /home/<system account>, with the uid translation of the row of the system map that pairs them; a view is mounted only of a child account's home, at ~/account_child/<short name> of its parent; every mount is nosuid and nodev; a mount point is an empty directory owned by root with mode 0000, made and deleted only at /home/<system account>; a home is deleted only during the deletion of its storage, or of a deallocated account. +

+ +

+ A single request touches /etc/passwd, a home and a mount, so the request gate needs every part of root's rights those take, and the separate account, home, mount and session languages of the earlier form of this design are folded into it. The loss is small: the right to mount, CAP_SYS_ADMIN, is nearly root by itself. The account, home, mount and session code stays in separate modules within the request interpreter. +

+ +

Partial failure

+ +

+ A request makes several changes. The journal for a request is approved whole before any instruction executes, so a request that breaks a constraint changes nothing on the system, and the record interpreter's refusals come before the request interpreter changes anything. A step can still fail while executing, after earlier steps succeeded. The databases record the intended state of each account, so such a failure leaves the intended state recorded and the system short of it; reconcile finishes the work or reports why it cannot. A request is complete only when the system matches the databases. +

+ +

At boot

+ +

+ attach and reconcile are run at boot, before logins are allowed, by a systemd unit that runs AccountTree_admin under root. Two kinds of systemd option are to be left out of that unit, because each would quietly stop the work: +

+ +
    +
  • options that give the unit a mount namespace of its own, such as PrivateMounts or ProtectHome, since a mount made there exists only for the unit, and no account ever sees it;
  • +
  • NoNewPrivileges, since with it set the kernel ignores the setuid bit of the record gate, and the record interpreter would run under root, opening the databases with root's rights.
  • +
+ +

Limit

+ +

+ Making a child account takes a database uid from a storage database, and a system uid and a system account from the system, and the system's are finite. Without a limit, one account could take all of them. Each system account has a limit on the number of accounts in its tree below it, counting accounts at every depth and counting deprecated ones. Limits are kept in the system database. The administrator sets them with AccountTree_admin limit set. A system account with no limit set has a limit of 0, and cannot make child accounts. +

+ +

+ A child account made by a child account counts against the limits of every account above it. The starting value suggested for a login account is 100. +

+ +

Mount

+ +

Idmapped mount

+ +

+ Every AccountTree mount is an idmapped mount: a bind mount that carries a translation of uids, applied by the kernel to every permission check and to every file made through it. Idmapped mounts need Linux 5.12 or later, and a file system that supports them, such as ext4, btrfs or xfs. An idmapped mount is kernel state, with no process behind it, so it survives the end of the program that made it and of any session. A mount through FUSE, such as bindfs, does neither. +

+ +

+ The request interpreter, and the administrator's, make the mount with the kernel's own calls, not by running mount. open_tree clones the source as a mount not yet attached anywhere; mount_setattr gives it nosuid, nodev and the idmap; move_mount places it. The kernel is given the idmap as a user namespace: a child process is made in a new user namespace, its uid and gid maps are written, and the namespace is handed to mount_setattr. A map line reads <id inside> <id outside> <count>; for an idmapped mount the kernel takes the uid on disk, the database uid, as the id inside, and the id outside, the system uid, as the uid seen through the mount. +

+ +

Attaching storage

+ +

+ AccountTree_admin attach <tree dir> verifies the storage database's standards and hash chain, compares its record with the heads seen here, as described under Copy and branch, then reads this system's map for the rows naming that database uuid, and nothing else. For each, it checks that the storage account is active and that the system account exists with its system uid and with its home at /home/<system account>, and mounts the storage account's home there, translating the database uid to the system uid. Storage accounts with no row are logged as skipped. The mounts are nosuid and nodev, since a storage can carry set-user-id programs and device files. Files on the storage owned by database uids the mount does not translate appear owned by nobody. +

+ +

View

+ +

+ A parent account sees each of its child accounts at ~/account_child/<short name>: an idmapped mount of the child account's home that translates the child's uid to the parent's. Files there appear to be the parent's, and files the parent makes there belong to the child. The directory is named for the property each of its entries has, being a child account, so the name is singular. Views are set when storage is attached, for the parent accounts already in session, and at each session's start, through set views. Views are designed in outline only, and come next after this document's other parts are built. +

+ +

Mount point

+ +

+ A mount point must exist before a mount, so /home/<system account> exists for every known account, attached or not. While nothing is mounted on it, it is an empty directory owned by root with mode 0000, and marked immutable. No account can enter it, and nothing can be written into it to be hidden later by the mount. +

+ +

Login and entering

+ +

+ A login account whose storage is not attached has no home. A login into the empty mount point finds no initialization files, and does not read them later when the storage arrives. Login is therefore refused for a managed login account whose home is not mounted. +

+ +

+ The check is in PAM, which every way of logging in passes through: the display manager, the console, ssh, su and machinectl shell. +

+ + + account sufficient pam_succeed_if.so uid = 0 quiet + account required pam_exec.so quiet /usr/local/lib/AccountTree/home_present + + +

+ home_present allows the login when the account is not managed by AccountTree, or when /home/<name> is a mount point, and refuses it otherwise. The first line keeps root able to log in whatever the check does. +

+ +

+ An account below a login account is entered through its parent: the parent gives an enter request to the request gate, whose interpreter allows the session only when the gate invoking account is above the account in the tree. The interpreter hands the session to systemd to start, and returns: the gate waits for its interpreter's verdict, so the interpreter cannot be the session itself. Whether such an account is ever to be entered through the login programs as well is open. +

+ +

+ A session, when it starts, gives a set_views request. The gate knows the account from its real uid, so a session can set only its own views. +

+ +

Program and module

+ +
    +
  • AccountTree: the person's program. Its commands compose requests and give them to the request gate; its window command opens the resource window, which shows the account's child accounts and their states and gives the same requests. Runs under the person's account.
  • +
  • AccountTree_admin: the administrator's program, run under root. Its commands: system make, system show and system delete; storage make, storage show and storage delete, each given a tree dir; account make; is-same-account, is-not-same-account, same-account show and same-account suggest; attach and detach; limit set; and reconcile. Each composes a journal for the administrator's gate. Detaching is refused while a system account it would unmount has a session. Unlocking an encrypted storage and mounting it come later.
  • +
  • The three gates, each with its guard and interpreter, installed owned by root, in directories only root can write, except the record gate, owned by the database account.
  • +
  • home_present: the PAM check.
  • +
  • The schema and initialization scripts.
  • +
+ +

+ Each program keeps the RT convention of a command line interface apart from its work functions, and the modules are imported as libraries: a window or a test calls the functions directly, never the command line. There are two layers, and one boundary between them, the gate. The person's functions, and the administrator's, compose journals and invoke a gate; AccountTree, the window and AccountTree_admin import them. At the gate the kernel says who asks, privilege changes, and the guard decides, by the constraints, what the privilege is used for. The interpreters hold the functions that change the system and the records. +

+ +

+ The modules follow Man_In_Grey's naming: a module is a namespace, a file named for a namespace alone is its main module, and a file specific to it carries the namespace before a middle dot. Each of AccountTree's three instruction languages is a namespace whose files parallel Man_In_Grey's own: +

+ + + Request·Language.py the request language's definition + Request·Guard.py its guard, with its constraints at its top + Request·GuardCheck.py its checks for the guard + Request·Interpreter.py its interpreter + Request·LibraryInstruction.py the implementation of its instructions + Record·... the same five for the record language + Admin·... the same five for the administrator's language + + +

+ The code that does the work is in modules of its own, which the libraries of instructions import. Where a direct call is not yet written, a wrapper function stands in its place, named for what it does here and commented with the direct call it will become. The names are proposed: +

+ +
    +
  • Kernel.py: direct calls, made through ctypes. The idmapped mount through open_tree, mount_setattr and move_mount; unmounting through umount2; the mount table from /proc/self/mountinfo; the immutable flag through the FS_IOC_SETFLAGS ioctl; login sessions through sd_get_uids in libsystemd. Imported by the request and administrator's interpreters.
  • +
  • Account·System.py: reading system accounts directly, through the C library, and changing them through wrappers, system_account_make, system_account_delete and system_group_delete, which run the shadow suite's commands for now. In the future each will be a direct call: the lines of /etc/passwd, /etc/shadow, /etc/group and /etc/gshadow edited under the C library's lock, lckpwdf, each new file written beside the old one and renamed into place.
  • +
  • Database.py, Database·Storage.py, Database·System.py and Database·SameAccount.py: the databases, their hash chain, and the work functions on them. Imported by the record interpreter only, so that no root process opens a database.
  • +
  • Window.py, and the person's side of the resource kinds: the person's program.
  • +
+ +

+ In the code as it stands, AccountTree_admin calls the changing functions directly, as root, and opens the databases itself. Moving the changing functions into the interpreters, and the database work into the record interpreter, is the work this design calls for. Keeping the changing functions small and their calls direct also lets them later be moved into the kernel, where an account tree with root at its top fits Linux as it is. +

+ +

Open question

+ +
    +
  • What deallocation does with an account's data.
  • +
  • When the request gate moves from setuid root to only the capabilities its requests need: CAP_SYS_ADMIN to mount, CAP_SETUID and CAP_SETGID for sessions, and the file capabilities for accounts and homes.
  • +
  • How AccountTree's gates, guards and interpreters are installed beside Man_In_Grey's library, and how a guard's check of its modules' ownership extends to that library. Man_In_Grey's install_gate installs Man_In_Grey's own tree; an application's tree is not yet provided for.
  • +
  • Whether the record gate is a Man_In_Grey gate, as proposed, or the request interpreter instead starts a child that gives up root's uid for the database account's and answers on a pipe. The gate keeps the record work in the same form as every other change, logged and guarded; the child needs no setuid program.
  • +
  • A deadline for each gate, so that a hung interpreter, which account A cannot signal once the gate has isolated itself, does not hold a request until root ends it. Man_In_Grey lists it as a future feature.
  • +
  • Whether an account below a login account is ever to be entered through the login programs as well.
  • +
  • Whether a parent account's views are removed when its last session ends, or left until the storage is detached.
  • +
  • How resources other than accounts and storage, such as network zones and devices, are recorded in the database.
  • +
+ +

Road map

+ +

+ Standard 5.0 is the design above. Later standards add the following, each described in its own section. A database lists the standards it follows in its standard table, so a database moves to a later standard by listing it, and a program that follows only an earlier one refuses it. +

+ +
    +
  • 6.0: longer account names; keys, for unlocking storage and recognizing it; signed heads; deduplication; deep freeze.
  • +
  • 7.0: integration of archives and of child copies, and merging of lines; a property file system.
  • +
  • Later, version not yet assigned: guests.
  • +
+ +

+ Guests. A person who plugs storage into someone else's machine and works there, with no administrator introducing it first, makes the machine a display with a processor for whatever storage is brought to it. The guest's storage accounts would be given system accounts at attach, in a band of system uids of their own, recorded in the system map as transient, and deleted at detach, with no rights to anything of the machine's owners. In 5.0 every storage is introduced by the administrator. Guests wait until keys, automatic attach, and the protection of a machine from what a stranger's storage carries, are understood. +

+ +

Standard 6.0

+ +

Longer account name

+ +

+ Account names built from a chain of parent names grow long, and 32 bytes, with two for each middle dot, is soon reached. 6.0 is to allow names of 128 bytes. The limit is in user space: the login records and useradd. Distributions are replacing the utmp records with wtmpdb and lastlog2, which have no fixed name field; openSUSE has done so. What remains is the check in useradd, which goes away when AccountTree edits the account files itself. Whether a system already accepts longer names is found by making a test account with a name past 32 bytes. +

+ +

Key

+ +

+ In 5.0, databases and systems recognize each other by database uuid and system uuid. 6.0 adds two uses of keys. +

+ +

+ Unlocking storage. A storage's file system is encrypted with LUKS2, which keeps up to 32 key slots, any of which opens it. Each person whose accounts are on the storage has a slot, opened by a passphrase or by a hardware token such as a FIDO2 key. When a storage is introduced to a system, the administrator opens it once, and the system adds a slot of its own, with a key it keeps, so that the storage is opened and attached on that system without asking anyone for a password. systemd-cryptenroll already enrolls such slots, and can seal a key in the machine's TPM, the security chip on its motherboard, so that the key is useless away from that machine. A slot can be removed from any machine that can open the storage, which is how a stolen laptop is cut off from it. Opening a storage with a key the system enrolled is also the proof that it is the storage that was introduced, which uuids and hashes cannot give: a fabricated storage cannot be opened with that key. The keys a system keeps make the place they are kept a target in the way /etc/shadow is; sealing them in the TPM is the protection. The disk inside the machine can be treated the same way, its /home opened by the system's own slot at boot, which keeps every storage alike. +

+ +

+ Signed heads. Each database and each system also holds a key pair. Each transaction's hash, which covers every transaction before it, is signed with the private key of the party that wrote it, so that a head can be shown to have been written by whoever holds that key, and not only to follow from the heads before it. The signature is checked with the public key, and the first is-same-account between a system and a storage database records each one's public key in the other's database, alongside its database uuid or system uuid. On attach, each side checks that the other holds the private key matching the public key recorded for it, so a copied database uuid or system uuid alone is refused. A storage is not a running party, so a database's private key is a file on its storage, and a full copy of the database copies the key too; protecting the key, for example inside the storage's encryption, is part of the 6.0 design. +

+ +

+ The migration touches only the identity and acquaintance tables and the programs that read them. A 6.0 database adds the key columns and lists both 5.0 and 6.0 in its standard table while keys are exchanged. The system map, uids and the account table are unchanged, because an account is still identified by its database and its database uid. +

+ +

Deduplication

+ +

+ A storage gathers copies of the same files, from old archives and from copies made between accounts. On btrfs, two files with the same content can be made to share their extents, after which the content is stored once while the files stay separate: a later write to one of them is written apart from the other. The kernel call that does this, FIDEDUPERANGE, compares the two ranges byte for byte before sharing them, so a false match on a hash cannot corrupt data. Tools such as duperemove and bees find candidates by hashing and leave the comparison to that call. +

+ +

+ A background pass over attached storage, started by a systemd timer through the administrator's gate, does the work. The pass keeps an index of content hashes, gives the kernel call the pairs whose hashes match, and so keeps each storage compacted as it is used. On a file system without shared extents, the pass reports duplicates and changes nothing. The same index serves integration in 7.0. +

+ +

Deep freeze

+ +

+ A person's accounts, archives included, do not always fit on one portable storage: 500 GB, say, against more data than that. Most of it is seldom read. Deep freeze divides an account's home into a working part and a frozen part by access time. Files not read for a long time move to the frozen part, which is kept on other storage, and a copy to a storage that cannot hold all of it takes the working part only. Whether the frozen part's directory tree stays visible, with its content brought in when its storage is attached, is open. +

+ +

+ Access times are recorded only when a file system is mounted with atime or relatime. relatime, the Linux default, updates a file's access time at most once a day, which is enough here; under noatime nothing is recorded, and deep freeze cannot work. Bringing a working part back after it was changed elsewhere is the integration problem of 7.0. +

+ +

Standard 7.0

+ +

Integration

+ +

+ Two kinds of integration are needed. The first brings the files of old archives into the current accounts. The old archives have older, less mature directory structures, and the aim is to place their files in the current structure, not to keep the old structures. The content-hash index kept for deduplication shows which old files already exist in the current tree, under any name, and leaves the rest to be placed. +

+ +

+ The second brings back a child copy: part of an account's home copied to a portable storage, changed there, and returned. Its changes are merged into the original. A change to the directory structure on either side is the hard case, as when a file is moved on one side and changed on the other. Two lines of one storage, described under Copy and branch, are merged the same way. +

+ +

Property file system

+ +

+ Both kinds of integration are hard because a file has one place in one tree. A file system organized by properties, or tags, would let a file belong to an old structure and a new one at once, and make a directory a query. AccountTree's contract with storage is that an account's home is a directory mounted at /home/<system account>, and the mounting is one function in the code. A home provided by a property file system changes that function and nothing else, provided the file system supports idmapped mounts or translates uids itself. At that point the AccountTree project is expected to fork. +

+ +
+ + diff --git a/document/design_AccountTree.html b/document/design_AccountTree.html index e4ec119..6b496b0 100644 --- a/document/design_AccountTree.html +++ b/document/design_AccountTree.html @@ -24,7 +24,7 @@ @@ -72,7 +72,7 @@

- Every account is entered through its parent. Root enters its child accounts through the system's login programs, the display manager, login at the console and sshd, which run as root and start a session for a person who gives a password or a key. A login account enters its child accounts through the AccountTree server, which starts sessions and programs in them; an account below a login account has no password, and the login programs do not enter it. The rule is the same at every level of the tree. Only the means differ, and the means for root's children are the ones Linux already has. + Every account is entered through its parent. Root enters its child accounts through the system's login programs, the display manager, login at the console and sshd, which run under root and start a session for a person who gives a password or a key. A login account enters its child accounts through the AccountTree server, which starts sessions and programs in them; an account below a login account has no password, and the login programs do not enter it. The rule is the same at every level of the tree. Only the means differ, and the means for root's children are the ones Linux already has.

@@ -102,8 +102,8 @@

  • account specifier: the name of an account in one place: system <host name> <system account>, or storage <tree dir> <storage account>.
  • same account: two account specifiers that name one account in two places. The system map records which system accounts are the same accounts as which storage accounts.
  • view: an idmapped mount through which a parent account sees a child account's files as its own.
  • -
  • server: the program that changes accounts, databases and mounts on behalf of persons. It runs as an unprivileged account of its own, AccountTree, and makes every change to the system through journal interpreters.
  • -
  • journal interpreter: a small privileged program of Man-In-Grey's design that receives a journal of commands, has its guard check every command against its constraints, and only then executes them. AccountTree has one bespoke journal interpreter for each of its command languages. design_Man-In-Grey.html gives the design.
  • +
  • server: the program that changes accounts, databases and mounts on behalf of persons. It runs under an unprivileged account of its own, AccountTree, and makes every change to the system through journal interpreters.
  • +
  • journal interpreter: a small program of Man_In_Grey's design, run under root through its gate, that receives a journal of commands, has its guard check every command against its constraints, and only then executes them. AccountTree has one bespoke journal interpreter for each of its command languages. Man_In_Grey.html gives the design.
  • unmanaged account: an account AccountTree does not manage, such as root, a service's account, or an account made with useradd by hand.
  • @@ -255,7 +255,7 @@

    - At attach, the database uids of the storage are to be compared with the system uids in use, from the system's account database, which includes directory services, and from /etc/subuid. Each coincidence is logged with the system account that would appear to own the raw files. A coincidence is not refused: through the mounts each account sees its own files correctly, and the raw files are behind the tree dir, which only the server's account and root can enter. It is reported because a tool that works by uid on the raw files, run as root, would treat them as that account's. This check is not yet built. + At attach, the database uids of the storage are to be compared with the system uids in use, from the system's account database, which includes directory services, and from /etc/subuid. Each coincidence is logged with the system account that would appear to own the raw files. A coincidence is not refused: through the mounts each account sees its own files correctly, and the raw files are behind the tree dir, which only the server's account and root can enter. It is reported because a tool that works by uid on the raw files, run under root, would treat them as that account's. This check is not yet built.

    @@ -407,7 +407,7 @@

    - The server owns AccountTree's records: the system database, /var/lib/AccountTree/system.sqlite3, and on each storage the tree dir and its storage database. It reads and writes them with its own rights. Every change to the system itself, an account in /etc/passwd, a home directory owned by a database uid, a mount, a session in another account, needs privilege the server does not have. For those the server composes a journal and hands it to a bespoke journal interpreter, through the interpreter's gate, following the design of Man-In-Grey. + The server owns AccountTree's records: the system database, /var/lib/AccountTree/system.sqlite3, and on each storage the tree dir and its storage database. It reads and writes them with its own rights. Every change to the system itself, an account in /etc/passwd, a home directory owned by a database uid, a mount, a session in another account, needs privilege the server does not have. For those the server composes a journal and hands it to a bespoke journal interpreter, through the interpreter's gate, following the design of Man_In_Grey.

    @@ -421,7 +421,7 @@

    - The requesting account is the principal of every journal the server composes for the request. A journal interpreter sees the server's account as its caller, and accepts a principal named in a journal only because the server's account is listed, in a table root owns, as entitled to name one. So the decision about who asked rests on the kernel at the socket, and the decision about what is allowed rests on the journal interpreter's guardrails. + The requesting account is named in every journal the server composes for the request. A journal interpreter sees the server's account as the orchestrator account, and accepts a requesting account named in a journal only because the server's account is listed in relay_account, in constraints root owns. So the decision about who asked rests on the kernel at the socket, and the decision about what is allowed rests on the journal interpreter's constraints.

    Requests

    @@ -442,7 +442,7 @@

    - The administrator's commands are not requests. AccountTree_admin runs as root, so it can change the system directly, but it composes the same journals and hands them to the same journal interpreters, with root as caller and principal. The code that changes the system then lives in one place, the journal interpreters, and the constraints catch the administrator's mistakes as they catch the server's. Making accounts the same, attaching, detaching and setting limits work while the server is stopped, at boot for instance. The server and AccountTree_admin both write the databases; SQLite's lock lets one transaction in at a time. + The administrator's commands are not requests. AccountTree_admin runs under root, so it can change the system directly, but it composes the same journals and hands them to the same journal interpreters, with root as the orchestrator account and no requesting account named. The code that changes the system then lives in one place, the journal interpreters, and the constraints catch the administrator's mistakes as they catch the server's. Making accounts the same, attaching, detaching and setting limits work while the server is stopped, at boot for instance. The server and AccountTree_admin both write the databases; SQLite's lock lets one transaction in at a time.

    Command languages and constraints

    @@ -455,11 +455,11 @@
  • Account. Commands: make a system account with a given name and system uid; delete a system account; delete its group. Constraints: the name is a valid account name; a new system uid lies in AccountTree's range and appears in no row of the system map; a system account is deleted only when the system map records that AccountTree made it. Edits /etc/passwd, /etc/shadow, /etc/group and /etc/gshadow.
  • Home. Commands: make a storage account's home in account_data, owned by its database uid, mode 0700; delete a home and its files. Constraints: the tree dir holds a storage database whose database uuid the system database knows, and the storage account exists there with that database uid; a home is deleted only during the deletion of its storage, or of a deallocated account.
  • Mount. Commands: make and delete a mount point; mount and unmount a home; mount and unmount a view. Constraints: a home is mounted only from <tree dir>/account_data/<storage account> onto /home/<system account>, where a row of the system map in use pairs the two, with that row's uid translation; a view is mounted only of a child account's home, at ~/account_child/<short name> of its parent; every mount is nosuid and nodev; a mount point is an empty directory owned by root with mode 0000, made and deleted only at /home/<system account>.
  • -
  • Session. Commands: start a session, or a program, in an account. Constraint: the principal is above the account in the tree.
  • +
  • Session. Commands: start a session, or a program, in an account. Constraint: the requesting account is above the account in the tree.
  • - Each command language needs a different part of root's rights: the account language to write files in /etc, the mount language to mount, the session language to change uid. Separate journal interpreters let each hold only its own part, as Man-In-Grey's design on least privilege describes. + Each command language needs a different part of root's rights: the account language to write files in /etc, the mount language to mount, the session language to change uid. Separate journal interpreters let each hold only its own part, as Man_In_Grey's design on least privilege describes.

    Partial failure

    @@ -482,7 +482,7 @@
    • Options that give the service a mount namespace of its own, such as PrivateMounts or ProtectHome. The journal interpreters the server starts inherit its mount namespace, and a mount made there exists only for the service; no account ever sees it.
    • -
    • NoNewPrivileges, and the options that imply it for a service not running as root. With it set, the kernel ignores the setuid bit and file capabilities of every program the service starts, so the gates run without privilege.
    • +
    • NoNewPrivileges, and the options that imply it for a service not running under root. With it set, the kernel ignores the setuid bit and file capabilities of every program the service starts, so the gates run without privilege.

    Limits

    @@ -545,7 +545,7 @@

    - An account below a login account is entered through its parent: the parent sends the server an enter request, and the server composes a journal for the session journal interpreter, whose constraint allows the session only when the principal is above the account in the tree. Whether such an account is ever to be entered through the login programs as well is open. + An account below a login account is entered through its parent: the parent sends the server an enter request, and the server composes a journal for the session journal interpreter, whose constraint allows the session only when the requesting account is above the account in the tree. Whether such an account is ever to be entered through the login programs as well is open.

    @@ -558,7 +558,7 @@

  • AccountTree: the person's program. Its commands send requests to the server; its window command opens the resource window, which shows the account's child accounts and their states and sends the same requests. Runs as the person's account.
  • AccountTree_server: the server, started by its systemd unit as the account AccountTree. Nobody types it.
  • AccountTree_admin: the administrator's program. Root. Its commands: system make, system show and system delete; storage make, storage show and storage delete, each given a tree dir; account make; is-same-account, is-not-same-account, same-account show and same-account suggest; attach and detach; and limit set. At boot, before logins are allowed, attach is run for every storage present, the disk inside the machine included, so that /home is populated. Detaching is refused while a system account it would unmount has a session. Unlocking an encrypted storage and mounting it come later.
  • -
  • The bespoke journal interpreters, one for each command language, account, home, mount and session, each a compiled gate followed by a guard and Man-In-Grey's general interpreter. Installed owned by root, in directories only root can write.
  • +
  • The bespoke journal interpreters, one for each command language, account, home, mount and session, each a compiled gate followed by a guard and Man_In_Grey's general interpreter. Installed owned by root, in directories only root can write.
  • home_present: the PAM check.
  • The schema and initialization scripts.
  • @@ -642,7 +642,7 @@

    - The AccountTree server already runs as a service, and is a natural home for a background pass over attached storage. The pass keeps an index of content hashes, gives the kernel call the pairs whose hashes match, and so keeps each storage compacted as it is used. On a file system without shared extents, the pass reports duplicates and changes nothing. The same index serves integration in 7.0. + The AccountTree server already runs under a service, and is a natural home for a background pass over attached storage. The pass keeps an index of content hashes, gives the kernel call the pairs whose hashes match, and so keeps each storage compacted as it is used. On a file system without shared extents, the pass reports duplicates and changes nothing. The same index serves integration in 7.0.

    Deep freeze

    diff --git a/setup b/setup index 1992fea..d2a452a 100755 --- a/setup +++ b/setup @@ -3,11 +3,24 @@ import os import sys +import json import argparse import tempfile PROJECT_ROLES = ["administrator", "consumer", "developer", "tester"] +# Role shells stack: setup run inside a role shell starts a shell in the new +# role, and exiting it returns to the shell it was started from. A loop +# through setup, such as one started from ~/.bashrc, is stopped at this depth. +DEPTH_MAX = 8 + +# The environment setup was first run from, before any role, recorded so that +# every role shell starts from it rather than from the role shell around it. +BASE_KEY = "RT_SETUP_BASE" + +# The outer role's values, for handing an Emacs shell buffer back on exit. +OUTER_KEY = ("REPO_HOME", "PROJECT", "ROLE", "PATH") + def main(): parser = argparse.ArgumentParser( description="Enter a project role environment by spawning an isolated subshell." @@ -15,19 +28,35 @@ def main(): parser.add_argument("role", choices=PROJECT_ROLES, help="The project role to assume.") args = parser.parse_args() - env = os.environ.copy() + outer = os.environ.copy() - # A role shell started inside another role shell inherits the outer - # project's PATH, REPO_HOME and functions, and a loop through setup would - # start shells without end. RT_SETUP_DEPTH counts the role shells this - # process is inside; any nesting is refused. - depth = env.get("RT_SETUP_DEPTH", "0") + # A role shell started inside another would otherwise inherit the outer + # role's environment: its tool directory on PATH, its variables, and the + # shell functions the shared setup exports. So every role shell starts + # from the base environment, the one setup was first run from, however + # deeply role shells are stacked. RT_SETUP_DEPTH counts the role shells + # this process is inside. + depth = outer.get("RT_SETUP_DEPTH", "0") depth = int(depth) if depth.isdigit() else 0 - if depth > 0: - print(f"setup: already inside a role shell ({env.get('PROJECT', '?')} {env.get('ROLE', '?')}).") - print("setup: type 'exit' to leave it, then run setup again.") + if depth >= DEPTH_MAX: + print(f"setup: already {depth} role shells deep; refusing, to stop a loop through setup.") sys.exit(1) + + base = None + if depth > 0 and BASE_KEY in outer: + try: + base = json.loads(outer[BASE_KEY]) + except ValueError: + base = None + if not isinstance(base, dict): + base = {k: v for k, v in outer.items() if not k.startswith("RT_SETUP_")} + + env = dict(base) + env[BASE_KEY] = json.dumps(base) env["RT_SETUP_DEPTH"] = str(depth + 1) + if depth > 0: + for key in OUTER_KEY: + env[f"RT_SETUP_OUTER_{key}"] = outer.get(key, "") # Python-domain logic: Variables and Paths # The project is the directory holding this file, whatever REPO_HOME the @@ -67,6 +96,11 @@ def main(): 'fi', 'if [[ -n "${INSIDE_EMACS}" ]] && command -v env_to_emacs >/dev/null 2>&1; then', ' env_to_emacs', + ' # inside another role shell: on exit, hand the Emacs buffer back to the outer role', + ' if [[ -n "${RT_SETUP_OUTER_ROLE:-}" ]]; then', + ' rt_setup_env_to_emacs="$(command -v env_to_emacs)"', + ' trap \'REPO_HOME="$RT_SETUP_OUTER_REPO_HOME" PROJECT="$RT_SETUP_OUTER_PROJECT" ROLE="$RT_SETUP_OUTER_ROLE" PATH="$RT_SETUP_OUTER_PATH" "$rt_setup_env_to_emacs"\' EXIT', + ' fi', 'fi', 'rm -f "${BASH_SOURCE[0]}"' ] diff --git a/shared/project-dependency/Python.org b/shared/project-dependency/Python.org index 4799f37..8e67c79 100644 --- a/shared/project-dependency/Python.org +++ b/shared/project-dependency/Python.org @@ -1,74 +1,13 @@ -#+TITLE: Python for SubU +#+TITLE: Python Installation Guide #+AUTHOR: Project Administrator -SubU's tools are Python programs, and its window is drawn with GTK 4 through PyGObject, the GTK bindings for Python. GTK talks to the desktop's display directly, Wayland or X, as the desktop's own programs do. +This project requires a dedicated Python environment. The configuration plugin (`shared/project-dependency/Python.init`) was written targeting Python 3.12.3, though the architecture does not rely on any known version-dependent features. -There are two ways to provide the Python. Choose one. +The build below can be moved. Nothing in it records the absolute directory it was built in where the interpreter depends on it, so the directory can move with a home directory, or to another machine, and keep working. - - The system Python. Least work: install two packages, section 1, and stop there. - - A project Python, built from source as a sibling of the project and linked into it. The project then does not depend on the system's Python version. Build the vanilla Python of section 3 first, then add what SubU needs, section 2. +* 1. Install the development libraries -The role setup decides which one a shell gets. shared/project-dependency/Python.init puts the project Python at the front of PATH when shared/project-dependency/Python-3.12.3 leads to a build that runs. When there is no such build, or it does not run, python3 is the system's. Running the tools with the system Python therefore needs nothing more than not installing the project Python. - -The package names below are for openSUSE; zypper search finds the name of one that differs on another release, and other distributions name the same libraries similarly. - -* 1. Using the system Python - -Install PyGObject for the system Python, and the GTK 4 introspection data. Introspection data is the machine-readable description of GTK's interface, which PyGObject reads at run time to call into GTK. - -#+begin_src bash -sudo zypper install python313-gobject typelib-1_0-Gtk-4_0 -#+end_src - -The 313 in the first name is the system Python's version, 3.13; find yours with python3 --version. - -Check: - -#+begin_src bash -python3 -c 'import gi; gi.require_version("Gtk" ,"4.0"); from gi.repository import Gtk; print("GTK" ,Gtk.get_major_version())' -#+end_src - -* 2. SubU's additions to the project Python - -A Python built from source has only its standard library. The system's PyGObject cannot be borrowed: it is installed in the system Python's own module directory, and is compiled for that Python's version. The project Python gets its own copy, compiled by pip against the GTK libraries already on the system. - -** 2.1 System development files - -pip compiles PyGObject and pycairo, the Python bindings for cairo, the drawing library GTK uses. Compiling needs the C development files of both libraries; the libraries themselves are already present on any desktop that runs GTK programs. The GTK 4 introspection data is read when the window opens. - -#+begin_src bash -sudo zypper install cairo-devel gobject-introspection-devel typelib-1_0-Gtk-4_0 -#+end_src - -** 2.2 Install PyGObject - -Run this in a role shell, so that python3 is the project Python, and with the Python's pkg-config files moved aside for the length of the install: - -#+begin_src bash -mv shared/project-dependency/Python-3.12.3/lib/pkgconfig shared/project-dependency/Python-3.12.3/lib/pkgconfig.off -python3 -m pip install PyGObject -mv shared/project-dependency/Python-3.12.3/lib/pkgconfig.off shared/project-dependency/Python-3.12.3/lib/pkgconfig -#+end_src - -Why the move. The build system pycairo uses, Meson, asks pkg-config where Python's header files are, and pkg-config answers from lib/pkgconfig/python-3.12.pc. When the Python's path holds a character outside ASCII, such as the middle dot of a SubU home directory (/home/Thomas·developer), pkg-config prints that character with a backslash before each of its bytes, which is no longer valid UTF-8, and Meson stops with "'utf-8' codec can't decode byte 0xc2 ... invalid continuation byte". With the .pc files out of sight, Meson asks the Python interpreter itself, and its answer is correct. The .pc files serve only C programs that build against Python, so moving them aside for the install affects nothing else. On a path of plain ASCII the move is unnecessary but harmless. - -If a build stops with "Dependency ... found: NO", the development files of that library are missing; install its -devel package and run pip again. - -** 2.3 Check - -#+begin_src bash -python3 -c 'import gi; gi.require_version("Gtk" ,"4.0"); from gi.repository import Gtk; print("GTK" ,Gtk.get_major_version())' -#+end_src - -* 3. The vanilla project Python - -This is the Harmony recipe for a project Python, unchanged. It is written for Python 3.12.3, though nothing in the project depends on version-specific features. - -The build can be moved. Nothing it depends on records the absolute directory it was built in, so the directory can move with a home directory, or to another machine with compatible system libraries, and keep working. - -** 3.1 Install the development libraries - -Python's configure step looks for the development files of each optional library, and silently leaves out any module whose library it does not find. A build without sqlite3-devel, for example, has no sqlite3 module, and the gap shows only when a program imports it. Install these first. +Python's configure step looks for the development files of each optional library, and silently leaves out any module whose library it does not find. A build without sqlite3-devel, for example, has no sqlite3 module, and the gap shows only when a program imports it. Install these first. The names are for openSUSE; zypper search finds the name of one that differs. #+begin_src bash sudo zypper install gcc make sqlite3-devel libopenssl-devel libffi-devel zlib-devel \ @@ -77,9 +16,10 @@ sudo zypper install gcc make sqlite3-devel libopenssl-devel libffi-devel zlib-de zlib-ng-compat-devel serves in place of zlib-devel, and the two conflict; if zypper reports that conflict, keep the one already installed. -** 3.2 Download and extract +tk-devel gives the tkinter module, which the window programs use. -Download the Python source code and extract it as a sibling of the project directory. From the top of the project: +* 2. Download and Extract +Download the Python source code and extract it as a sibling to the main project directory. Assuming you are at the top level of your project workspace: #+begin_src bash cd ../ @@ -88,9 +28,8 @@ tar -xzf Python-3.12.3.tgz cd Python-3.12.3 #+end_src -** 3.3 Build - -Compile in place. Setting the installation prefix to the current directory isolates the build. +* 3. Build the Environment +Compile the Python binaries in place. Setting the installation prefix to the current directory isolates the build. #+begin_src bash CONFIG_SITE= ./configure --prefix="$(pwd)" --libdir="$(pwd)/lib" --enable-optimizations @@ -106,14 +45,14 @@ The build is static: the interpreter is one program, with no separate libpython3 Near the end of build.log, make lists any optional modules it could not build, under "The necessary bits to build these optional modules were not found". nis, an obsolete network directory service, and _curses_panel may appear there; the project uses neither. For any other module listed, install the missing development library and build again. -** 3.4 Check the build +* 4. Check the build #+begin_src bash bin/python3 -c 'import sys ,ssl ,sqlite3 ,ctypes ,lzma ,bz2 ,readline ,tkinter; print(sys.prefix)' ldd bin/python3 | grep libpython # prints nothing: no shared libpython #+end_src -** 3.5 Use pip through the interpreter +* 5. Use pip through the interpreter The scripts make install writes into bin/, such as pip3, start with a #! line naming the absolute path of the interpreter, so they break when the directory moves. Run pip as a module instead, which uses whichever interpreter runs it: @@ -121,8 +60,12 @@ The scripts make install writes into bin/, such as pip3, start with a #! line na python3 -m pip install #+end_src -** 3.6 Link the project +* 6. Link the Project + +Python-3.12.3 is linked to a Harmony derived project by default. + +See administrator/document/how-to_add-project-dependency.html for details about linking a project, if you want to check the setup. -The project reaches the build through shared/project-dependency/Python-3.12.3, a link to project/Python-3.12.3, where project is the link to the directory holding this project and its siblings. A build extracted as a sibling of the project, as in 3.2, is found without further steps. +If you are installing a different version, then at least the Python-3.12.3 version project linking is there as a good example. -See administrator/document/how-to_add-project-dependency.html for how dependencies are linked, or to link a different version. +Project linking is not difficult, make a sym link to `project/`, and then add an init file, as described in administrator/document/how-to_add-project-dependency.html. The 'project' in the link is literal. diff --git a/shared/tool/env_to_emacs b/shared/tool/env_to_emacs index 947efc0..8a65dba 100755 --- a/shared/tool/env_to_emacs +++ b/shared/tool/env_to_emacs @@ -1,30 +1,78 @@ #!/usr/bin/env bash +# env_to_emacs - give the Emacs shell buffer this setup runs in the role's environment +# +# Emacs starts programs (compile, grep, and the like) with its own copy of the +# environment, which a shell running inside it cannot change. This tool sets +# REPO_HOME, PROJECT, ROLE and PATH in Emacs, and exec-path to match, for the +# one shell buffer whose shell this setup runs under, and for no other. +# +# The values are buffer-local. Killing the buffer discards them, and other +# buffers, including shells started later, keep Emacs's own environment. (An +# earlier version set them globally, so every shell started afterward began +# in the last role set up, even after that role's buffer was gone.) +# +# The buffer is found by process: Emacs knows the process id of the shell in +# each shell buffer, and that shell is an ancestor of this script, perhaps +# several generations up, since setup starts a new bash under the buffer's own. + script_afp=$(realpath "${BASH_SOURCE[0]}") set -euo pipefail if [ -z "${REPO_HOME:-}" ]; then - echo "emacs-sync:: error: REPO_HOME is not set. Source a setup environment first." + echo "env_to_emacs: error: REPO_HOME is not set. Source a setup environment first." exit 1 fi if ! command -v emacsclient >/dev/null 2>&1; then - echo "emacs-sync:: error: emacsclient is not installed or not in PATH." + echo "env_to_emacs: error: emacsclient is not installed or not in PATH." exit 1 fi -# Push variables to the Emacs global environment -emacsclient --eval "(setenv \"REPO_HOME\" \"$REPO_HOME\")" > /dev/null -emacsclient --eval "(setenv \"PROJECT\" \"$PROJECT\")" > /dev/null +# quote a value as an Emacs Lisp string +elisp_string(){ + local s="$1" + s=${s//\\/\\\\} + s=${s//\"/\\\"} + printf '"%s"' "$s" +} -if [ -n "${ROLE:-}" ]; then - emacsclient --eval "(setenv \"ROLE\" \"$ROLE\")" > /dev/null -fi +form=$(cat < pid 1) (not buffer)) + (setq buffer + (seq-find (lambda (b) + (let ((p (get-buffer-process b))) + (and p (eql (process-id p) pid)))) + (buffer-list))) + (unless buffer + (setq pid (alist-get 'ppid (process-attributes pid))))) + (if (not buffer) + "no shell buffer found" + (with-current-buffer buffer + (setq-local process-environment (copy-sequence process-environment)) + (setenv "REPO_HOME" $(elisp_string "$REPO_HOME")) + (setenv "PROJECT" $(elisp_string "${PROJECT:-}")) + (setenv "ROLE" $(elisp_string "${ROLE:-}")) + (setenv "PATH" $(elisp_string "$PATH")) + (setq-local exec-path (append (parse-colon-path $(elisp_string "$PATH")) (list exec-directory))) + (buffer-name)))) +EOF +) -# Sync the PATH so Emacs subprocesses find the local tools -emacsclient --eval "(setenv \"PATH\" \"$PATH\")" > /dev/null +if ! result=$(emacsclient --eval "$form" 2>&1); then + echo "env_to_emacs: cannot reach the Emacs server; Emacs environment left unchanged" + echo " $result" + exit 0 +fi +result=${result#\"} +result=${result%\"} -# Update exec-path for native Emacs executable resolution -emacsclient --eval "(setq exec-path (append (parse-colon-path \"$PATH\") (list exec-directory)))" > /dev/null +if [[ "$result" == "no shell buffer found" ]]; then + echo "env_to_emacs: no Emacs shell buffer runs this shell; Emacs environment left unchanged" + exit 0 +fi -echo "Emacs environment synchronized for project: $PROJECT" +echo "Emacs buffer $result has the environment of $PROJECT ${ROLE:-}" diff --git a/shared/tool/makefile/Harmony.mk b/shared/tool/makefile/Harmony.mk index eaf1ce8..b168ac5 100644 --- a/shared/tool/makefile/Harmony.mk +++ b/shared/tool/makefile/Harmony.mk @@ -30,7 +30,9 @@ MACHINE_DIR ?= scratchpad/made LIBRARY_FILE ?= $(LIBRARY_DIR)/lib$(LIBRARY_NAME).a -LN_FLAGS ?= -L$(LIBRARY_DIR) -L/lib64 -L/lib -l$(LIBRARY_NAME) +# the project library is linked only when the namespace makes one +LN_FLAGS ?= -L$(LIBRARY_DIR) -L/lib64 -L/lib $(if $(object_lib),-l$(LIBRARY_NAME)) +# LN_FLAGS ?= -L$(LIBRARY_DIR) -L/lib64 -L/lib -l$(LIBRARY_NAME) KMOD_SOURCE_DIR ?= authored KMOD_CCFLAGS ?= -I $(KMOD_SOURCE_DIR) diff --git a/shared/tool/setup b/shared/tool/setup index 5ecb239..7dc24d3 100644 --- a/shared/tool/setup +++ b/shared/tool/setup @@ -74,6 +74,8 @@ umask 0077 PATH="$REPO_HOME/shared/authored:$PATH" # Remove duplicates, keeping the first occurrence + # Done with bash built-ins only. A pipeline through awk and sed left PATH + # empty whenever either program failed to run. clean_path(){ local entry result="" local -A seen=() diff --git a/shared/tool/version b/shared/tool/version index b264028..68524a3 100755 --- a/shared/tool/version +++ b/shared/tool/version @@ -1,2 +1,4 @@ echo "Harmony v3.5 2026-10-14 04:51:11 Z" + + echo "SubU v4.0 2026-10-14 04:51:11 Z" -- 2.20.1