From 715a75f5cd5fbb4d584b4653ea03bd4292c1431d Mon Sep 17 00:00:00 2001 From: Thomas Walker Lynch Date: Thu, 22 Oct 2026 10:16:09 +0000 Subject: [PATCH] . --- administrator/tool/skeleton_update | 621 ++++++++++++++++++++++++ shared/project-dependency/Python-3.12.3 | 1 - shared/project-dependency/Python.org | 10 +- shared/tool/env_to_emacs | 74 ++- shared/tool/makefile/Harmony.mk | 4 +- shared/tool/replace_from_tar | 182 +++++++ 6 files changed, 875 insertions(+), 17 deletions(-) create mode 100755 administrator/tool/skeleton_update delete mode 120000 shared/project-dependency/Python-3.12.3 create mode 100755 shared/tool/replace_from_tar diff --git a/administrator/tool/skeleton_update b/administrator/tool/skeleton_update new file mode 100755 index 0000000..716e342 --- /dev/null +++ b/administrator/tool/skeleton_update @@ -0,0 +1,621 @@ +#!/usr/bin/env python3 +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- +""" +skeleton_update + +Brings a project onto the current Harmony skeleton. Skeleton files are +copied in from a Harmony checkout, and .gitignore and shared/tool/version are +merged with the project's own lines. The project's files are then sorted: + + a file in a place Harmony has, such as developer/authored/ or a directory + of the skeleton, stays where it is; + a file the plan moves is renamed to its new place; + a file under a plan's duplicate tree is deleted when the project holds a + file with the same content elsewhere, and is otherwise deprecated; + every other file is deprecated: moved, with its path kept, under + developer/authored/deprecated/, so nothing is lost and the original place + of each file can be read from its new path. + +The plan is a Python file giving what is particular to one project. A +project already laid out after Harmony needs no plan, and the tool then +updates its skeleton files only. The plan's names are: + + MOVE {old path: new path}; a path ending in / moves a directory tree + DEPRECATE paths that are deprecated even though Harmony has their place + DUPLICATE trees whose files are copies of files kept elsewhere + KEEP paths kept as they are, including skeleton files the project + has its own version of + EXECUTABLE paths, after any move, given execute permission where read + permission is given + +The files sorted are those git tracks or would track: the work tree less what +the project's .gitignore ignores. Ignored files, such as a local Python +environment, are left where they are and listed at the end of the log. +Compiled Python caches are deleted. Directories left empty are removed. + +Commit the project first; run refuses a work tree with uncommitted changes. +Afterward, 'git add -A' records the moves, and git recognizes each moved file +as a rename by its content, so 'git log --follow' follows its history. Run +again after committing, and the log reports nothing left to do. + +Usage: skeleton_update run [] + skeleton_update preview run [] + help | -h | --help print this message (also printed when given no arguments) + run bring onto the skeleton of + preview run print the log of what run would do, change nothing +""" + +import sys ,os ,shutil ,filecmp ,subprocess ,runpy ,hashlib +from pathlib import Path + + +# -------------------------------------------------------------------------------- +# what the tool knows about Harmony + +DEPRECATED_DIR = "developer/authored/deprecated" + +# Skeleton files a project does not take: it has its own. +SKELETON_SKIP_FILE = {"README.md" ,"LICENSE" ,"0pus_Harmony"} +SKELETON_SKIP_TREE = ("developer/authored/ExampleGreet.make/",) + +# Skeleton files merged with the project's version rather than replaced. +MERGED_FILE = (".gitignore" ,"shared/tool/version") + +# Files at the top of a project that belong to the project. +PROJECT_ROOT_FILE = {"LICENSE" ,"README.md" ,".gitattributes" ,".gitmodules"} + +# Trees in which every file belongs to the project. +PROJECT_TREE = ( + "administrator/authored/", + "administrator/document/", + "developer/authored/", + "developer/document/", + "developer/experiment/", + "document/", + "shared/authored/", + "tester/authored/", + "tester/document/", +) + +# Placeholder files of the old skeleton; Harmony uses .gitkeep. +PLACEHOLDER_NAME = {".githolder"} + +CACHE_DIR_NAME = {"__pycache__"} + +PLAN_NAME = ("MOVE" ,"DEPRECATE" ,"DUPLICATE" ,"KEEP" ,"EXECUTABLE") + + +# -------------------------------------------------------------------------------- +# the log + +TAG_WIDTH = 18 + +# the tag a preview prints in place of each past-tense tag +PREVIEW_TAG = { + "added": "add" ,"deleted": "delete" ,"deprecated": "deprecate" ,"kept": "keep", + "left alone": "leave alone" ,"made executable": "make executable" ,"merged": "merge", + "moved": "move" ,"removed": "remove", + "replaced": "replace", +} + +class Log: + def __init__(self ,preview): + self.preview = preview + self.count = {} + self.error_count = 0 + + def line(self ,tag ,rel ,reason=None): + if self.preview: tag = "would " + PREVIEW_TAG.get(tag ,tag) + text = f" {tag:<{TAG_WIDTH}}{rel}" + if reason: text += f" ({reason})" + print(text) + + def tally(self ,key ,n=1): + self.count[key] = self.count.get(key ,0) + n + + def error(self ,rel ,message): + self.error_count += 1 + print(f" {'error':<{TAG_WIDTH}}{rel} ({message})") + + def summary(self): + print("summary:") + order = [ + ("moved" ,"files moved to their place by the plan"), + ("deprecated" ,"files moved under " + DEPRECATED_DIR), + ("deleted duplicate" ,"duplicate files deleted"), + ("deleted placeholder" ,".githolder files deleted"), + ("deleted cache" ,"__pycache__ directories deleted"), + ("skeleton added" ,"skeleton files added"), + ("skeleton replaced" ,"skeleton files replaced"), + ("skeleton current" ,"skeleton files already current"), + ("skeleton kept" ,"skeleton files kept in the project's own version"), + ("merged" ,"files merged with the skeleton"), + ("merged current" ,"merged files already current"), + ("kept" ,"project files kept in place"), + ("made executable" ,"files made executable"), + ("left alone" ,"ignored files left in place, listed above"), + ("directory removed" ,"empty directories removed"), + ] + for key ,label in order: + print(f" {self.count.get(key ,0):>4} {label}") + if self.error_count: + print(f" {self.error_count:>4} errors") + if self.preview: + print("preview only: nothing was changed.") + + +# -------------------------------------------------------------------------------- +# reading git + +def git(directory ,*argument): + return subprocess.run( + ["git" ,"-C" ,str(directory) ,*argument], + capture_output=True ,text=True ,check=True + ).stdout + +def git_dirty(directory): + return git(directory ,"status" ,"--porcelain").strip() + +def project_file_list(project): + """ + The files git tracks or would track, that exist: tracked files, and + untracked files the project's .gitignore does not ignore. + """ + listing = git(project ,"ls-files" ,"-z" ,"--cached" ,"--others" ,"--exclude-standard").split("\0") + result = set() + for rel in listing: + if rel and ((project / rel).exists() or (project / rel).is_symlink()): + result.add(rel) + return sorted(result) + +def ignored_file_list(project): + listing = git(project ,"ls-files" ,"-z" ,"--others" ,"--ignored" ,"--exclude-standard").split("\0") + return sorted(rel for rel in listing if rel) + +def skeleton_file_list(harmony): + """ + The skeleton is the set of files in Harmony's last commit, as new-project + takes it. + """ + result = [] + for rel in git(harmony ,"ls-tree" ,"-r" ,"--name-only" ,"-z" ,"HEAD").split("\0"): + if not rel or rel in SKELETON_SKIP_FILE or in_tree(rel ,SKELETON_SKIP_TREE): + continue + result.append(rel) + return sorted(result) + + +# -------------------------------------------------------------------------------- +# paths + +def in_tree(rel ,tree_tuple): + return any(rel.startswith(t) for t in tree_tuple) + +def matches(rel ,path_tuple): + """A path names a file exactly, or, ending in /, the tree under it.""" + return any(rel == p or (p.endswith("/") and rel.startswith(p)) for p in path_tuple) + +def moved_path(rel ,move): + for old ,new in move.items(): + if rel == old: + return new + if old.endswith("/") and rel.startswith(old): + return new + rel[len(old):] + return None + +def directory_set(rel_list): + result = set() + for rel in rel_list: + parent = Path(rel).parent + while parent != Path("."): + result.add(parent.as_posix()) + parent = parent.parent + return result + +def content_hash(path): + if path.is_symlink(): + return "link:" + os.readlink(path) + h = hashlib.sha256() + with open(path ,"rb") as f: + for block in iter(lambda: f.read(1 << 16) ,b""): + h.update(block) + return h.hexdigest() + + +# -------------------------------------------------------------------------------- +# file operations; each tests preview immediately before acting + +def same_leaf(a ,b): + if a.is_symlink() or b.is_symlink(): + return a.is_symlink() and b.is_symlink() and os.readlink(a) == os.readlink(b) + return a.is_file() and b.is_file() and filecmp.cmp(a ,b ,shallow=False) + +def copy_leaf(src ,dst ,preview): + if preview: return + dst.parent.mkdir(parents=True ,exist_ok=True) + if dst.is_symlink() or dst.exists(): + dst.unlink() + if src.is_symlink(): + os.symlink(os.readlink(src) ,dst) + else: + shutil.copy2(src ,dst) + +def move_leaf(src ,dst ,preview): + if preview: return + dst.parent.mkdir(parents=True ,exist_ok=True) + os.rename(src ,dst) + +def delete_leaf(path ,preview): + if preview: return + path.unlink() + +def delete_tree(path ,preview): + if preview: return + shutil.rmtree(path) + +def write_text(path ,text ,preview): + if preview: return + path.parent.mkdir(parents=True ,exist_ok=True) + path.write_text(text ,encoding="utf-8") + + +# -------------------------------------------------------------------------------- +# merging + +def merged_gitignore(project_text ,harmony_text ,project_name): + """ + Harmony's lines, then a section holding the project's patterns that + Harmony lacks. The project's comments and blank lines are not carried. + """ + section = "# " + project_name + harmony_line = set(harmony_text.splitlines()) + extra = [] + for line in project_text.splitlines(): + pattern = line.strip() + if not pattern or pattern.startswith("#"): + continue + if pattern not in harmony_line and pattern not in extra: + extra.append(pattern) + text = harmony_text.rstrip("\n") + "\n" + if extra: + text += "\n" + section + "\n" + "\n".join(extra) + "\n" + return text + +def merged_version(project_text ,harmony_text ,project_name): + """Harmony's file, then the project's lines that do not name Harmony.""" + project_line = [l for l in project_text.splitlines() if l.strip() and "Harmony" not in l and l not in harmony_text.splitlines()] + text = harmony_text if harmony_text.endswith("\n") else harmony_text + "\n" + return text + "".join(l + "\n" for l in project_line) + +MERGE_FUNCTION = { + ".gitignore": merged_gitignore, + "shared/tool/version": merged_version, +} + + +# -------------------------------------------------------------------------------- +# the work + +def step_cache(project ,log ,preview): + for dirpath ,dirnames ,filenames in os.walk(project): + dirnames.sort() + if Path(dirpath) == project and ".git" in dirnames: + dirnames.remove(".git") + for name in list(dirnames): + if name in CACHE_DIR_NAME: + rel = Path(dirpath ,name).relative_to(project).as_posix() + delete_tree(project / rel ,preview) + log.line("deleted" ,rel + "/" ,"compiled Python cache") + log.tally("deleted cache") + dirnames.remove(name) + +def decide(rel ,plan ,skeleton_set ,skeleton_dir ,ignored_list): + action = decide_place(rel ,plan ,skeleton_set ,skeleton_dir) + if Path(rel).name == ".gitignore" and action[0] in ("move" ,"duplicate" ,"deprecate"): + prefix = Path(rel).parent.as_posix() + "/" + n = sum(1 for i in ignored_list if i.startswith(prefix)) + if n: + return ("keep" ,rel ,f"it hides {n} ignored file{'s' if n != 1 else ''}, left in place") + return action + +def decide_place(rel ,plan ,skeleton_set ,skeleton_dir): + """ + What becomes of one project file: (action ,new path ,reason). The action is + one of placeholder, keep, keep skeleton, move, duplicate, deprecate, or + skeleton, which the skeleton steps handle. + """ + name = Path(rel).name + if name in PLACEHOLDER_NAME: + return ("placeholder" ,None ,None) + if matches(rel ,plan["KEEP"]): + return ("keep skeleton" if rel in skeleton_set else "keep" ,rel ,None) + new = moved_path(rel ,plan["MOVE"]) + if new is not None: + return ("move" ,new ,None) + if matches(rel ,plan["DUPLICATE"]): + return ("duplicate" ,f"{DEPRECATED_DIR}/{rel}" ,None) + if matches(rel ,plan["DEPRECATE"]): + return ("deprecate" ,f"{DEPRECATED_DIR}/{rel}" ,"by the plan") + if rel in skeleton_set or rel in MERGED_FILE: + return ("skeleton" ,rel ,None) + parent = Path(rel).parent.as_posix() + if (parent == "." and (name in PROJECT_ROOT_FILE or name.startswith("0pus_"))) \ + or in_tree(rel ,PROJECT_TREE) or parent in skeleton_dir: + return ("keep" ,rel ,None) + return ("deprecate" ,f"{DEPRECATED_DIR}/{rel}" ,None) + +def step_sort(project ,plan ,skeleton_set ,file_list ,ignored_list ,log ,preview): + """ + Place every project file that is not a skeleton file. Every decision is + made before anything moves, so a duplicate can name where its twin ends up. + """ + skeleton_dir = directory_set(skeleton_set) + decision = {rel: decide(rel ,plan ,skeleton_set ,skeleton_dir ,ignored_list) for rel in file_list} + + # where the content of each file that stays in the project ends up + kept_at = {} + for rel ,(action ,new ,reason) in decision.items(): + if action in ("keep" ,"keep skeleton" ,"move" ,"deprecate" ,"skeleton"): + kept_at.setdefault(content_hash(project / rel) ,new) + + for rel in file_list: + action ,new ,reason = decision[rel] + if action == "placeholder": + delete_leaf(project / rel ,preview) + log.line("deleted" ,rel ,"placeholder; Harmony uses .gitkeep") + log.tally("deleted placeholder") + elif action == "keep skeleton": + log.line("kept" ,rel ,"the project's own version, by the plan") + log.tally("skeleton kept") + elif action == "keep": + if reason: log.line("kept" ,rel ,reason) + log.tally("kept") + elif action == "move": + place(project ,rel ,new ,"moved" ,f"to {new}" ,log ,preview) + elif action == "duplicate": + twin = kept_at.get(content_hash(project / rel)) + if twin: + delete_leaf(project / rel ,preview) + log.line("deleted" ,rel ,f"the same as {twin}") + log.tally("deleted duplicate") + else: + place(project ,rel ,new ,"deprecated" ,"no copy of it is kept" ,log ,preview) + elif action == "deprecate": + place(project ,rel ,new ,"deprecated" ,reason ,log ,preview) + +def place(project ,rel ,new ,tag ,reason ,log ,preview): + dst = project / new + if dst.exists() or dst.is_symlink(): + log.error(rel ,f"{new} already exists; nothing was moved") + return + move_leaf(project / rel ,dst ,preview) + log.line(tag ,rel ,reason) + log.tally(tag) + +def step_skeleton(project ,harmony ,skeleton_list ,plan ,log ,preview): + for rel in skeleton_list: + if rel in MERGED_FILE or matches(rel ,plan["KEEP"]): + continue + src = harmony / rel + dst = project / rel + if dst.exists() or dst.is_symlink(): + if same_leaf(src ,dst): + log.tally("skeleton current") + continue + copy_leaf(src ,dst ,preview) + log.line("replaced" ,rel ,"from the skeleton") + log.tally("skeleton replaced") + else: + copy_leaf(src ,dst ,preview) + log.line("added" ,rel ,"from the skeleton") + log.tally("skeleton added") + +def step_merge(project ,harmony ,project_name ,plan ,log ,preview): + for rel in MERGED_FILE: + if matches(rel ,plan["KEEP"]): + continue + project_path = project / rel + harmony_text = (harmony / rel).read_text(encoding="utf-8") + project_text = project_path.read_text(encoding="utf-8") if project_path.is_file() else "" + text = MERGE_FUNCTION[rel](project_text ,harmony_text ,project_name) + if text == project_text: + log.tally("merged current") + continue + if not project_path.exists(): + copy_leaf(harmony / rel ,project_path ,preview) + write_text(project_path ,text ,preview) + log.line("merged" ,rel ,"the skeleton's lines with the project's own") + log.tally("merged") + +def step_opus(project ,harmony ,project_name ,log ,preview): + rel = "0pus_" + project_name + if (project / rel).exists(): + return + write_text(project / rel ,"" ,preview) + if not preview: shutil.copymode(harmony / "0pus_Harmony" ,project / rel) + log.line("added" ,rel ,"the file that marks the project root") + log.tally("skeleton added") + +def step_executable(project ,plan ,file_list ,log ,preview): + """ + Give each EXECUTABLE path execute permission wherever it has read + permission. In a preview the file is found at its place before the move. + """ + before = {moved_path(rel ,plan["MOVE"]) or rel: rel for rel in file_list} + for rel in plan["EXECUTABLE"]: + path = project / rel + if not path.is_file(): + path = project / before.get(rel ,rel) + if not path.is_file(): + log.error(rel ,"EXECUTABLE names a file that does not exist") + continue + mode = path.stat().st_mode + wanted = mode | ((mode & 0o444) >> 2) + if wanted == mode: + continue + if not preview: os.chmod(path ,wanted) + log.line("made executable" ,rel) + log.tally("made executable") + +def step_ignored(project ,file_list ,ignored_list ,log): + """ + List ignored files outside scratchpad and consumer, by the highest + directory holding nothing git tracks. + """ + tracked_dir = directory_set(file_list) + group = {} + for rel in ignored_list: + part = rel.split("/") + if "scratchpad" in part[:-1] or part[0] == "consumer" or any(p in CACHE_DIR_NAME for p in part): + continue + top = rel + parent = Path(rel).parent + while parent != Path(".") and parent.as_posix() not in tracked_dir: + top = parent.as_posix() + "/" + parent = parent.parent + group[top] = group.get(top ,0) + 1 + for top ,n in sorted(group.items()): + log.line("left alone" ,top ,f"ignored by git; {n} file{'s' if n != 1 else ''}") + log.tally("left alone" ,n) + +def step_newly_ignored(project ,ignored_list ,log): + """ + A file the run placed where the merged .gitignore ignores it would not be + recorded by 'git add -A'. Such files are reported as errors. + """ + before = set(ignored_list) + for rel in ignored_file_list(project): + part = rel.split("/") + if rel in before or "scratchpad" in part[:-1] or part[0] == "consumer" or any(p in CACHE_DIR_NAME for p in part): + continue + log.error(rel ,"git ignores it now, so 'git add -A' will not record it; add it with 'git add -f'") + +def step_empty_dir(project ,log ,preview): + for dirpath ,dirnames ,filenames in os.walk(project ,topdown=False): + path = Path(dirpath) + if path == project or ".git" in path.relative_to(project).parts: + continue + try: + remaining = list(path.iterdir()) + except FileNotFoundError: + continue + if not remaining: + rel = path.relative_to(project).as_posix() + if not preview: path.rmdir() + log.line("removed" ,rel + "/" ,"empty directory") + log.tally("directory removed") + +def work(harmony ,project ,plan ,preview): + print(f"skeleton_update {'preview ' if preview else ''}run in {project}") + print(f" skeleton from {harmony}") + log = Log(preview) + project_name = project.name + + skeleton_list = skeleton_file_list(harmony) + skeleton_set = set(skeleton_list) + file_list = [r for r in project_file_list(project) if not any(p in CACHE_DIR_NAME for p in r.split("/"))] + ignored_list = [r for r in ignored_file_list(project) if not any(p in CACHE_DIR_NAME for p in r.split("/"))] + + step_cache(project ,log ,preview) + step_sort(project ,plan ,skeleton_set ,file_list ,ignored_list ,log ,preview) + step_skeleton(project ,harmony ,skeleton_list ,plan ,log ,preview) + step_merge(project ,harmony ,project_name ,plan ,log ,preview) + step_opus(project ,harmony ,project_name ,log ,preview) + step_executable(project ,plan ,file_list ,log ,preview) + step_ignored(project ,file_list ,ignored_list ,log) + if preview: + print(" (directories left empty are found and removed by run, not listed in a preview)") + else: + step_empty_dir(project ,log ,preview) + step_newly_ignored(project ,ignored_list ,log) + + log.summary() + return 1 if log.error_count else 0 + + +# -------------------------------------------------------------------------------- +# command line + +def read_plan(path): + plan = {"MOVE": {} ,"DEPRECATE": () ,"DUPLICATE": () ,"KEEP": () ,"EXECUTABLE": ()} + if path is None: + return plan + given = runpy.run_path(str(path)) + for name in PLAN_NAME: + if name in given: + plan[name] = given[name] + if not isinstance(plan["MOVE"] ,dict): + raise ValueError("MOVE must be a dictionary of old path to new path") + for name in ("DEPRECATE" ,"DUPLICATE" ,"KEEP" ,"EXECUTABLE"): + plan[name] = tuple(plan[name]) + for old ,new in plan["MOVE"].items(): + if old.endswith("/") != new.endswith("/"): + raise ValueError(f"MOVE '{old}' to '{new}': a directory moves to a directory, both ending in /") + return plan + +def print_help(): + print(__doc__.strip()) + return 0 + +def CLI(argv=None): + if argv is None: argv = sys.argv[1:] + if not argv or argv[0] in ("help" ,"-h" ,"--help"): return print_help() + + preview = argv[0] == "preview" + if preview: argv = argv[1:] + if not argv: + print("Error: 'preview' needs a command after it." ,file=sys.stderr) + return 1 + + command ,rest = argv[0] ,argv[1:] + if command != "run": + print(f"Error: unknown command '{command}'." ,file=sys.stderr) + print_help() + return 1 + if len(rest) not in (2 ,3): + print("Error: run is given []." ,file=sys.stderr) + return 1 + + harmony = Path(rest[0]).resolve() + project = Path(rest[1]).resolve() + plan_path = Path(rest[2]).resolve() if len(rest) == 3 else None + + if not (harmony / "0pus_Harmony").is_file() or not (harmony / ".git").exists(): + print(f"Error: {harmony} is not a Harmony checkout: it lacks 0pus_Harmony or .git." ,file=sys.stderr) + return 1 + if not (project / ".git").exists(): + print(f"Error: {project} is not a git work tree; the tool sorts the files git tracks." ,file=sys.stderr) + return 1 + if harmony == project: + print("Error: the project is the Harmony checkout itself." ,file=sys.stderr) + return 1 + if plan_path and not plan_path.is_file(): + print(f"Error: plan file {plan_path} does not exist." ,file=sys.stderr) + return 1 + try: + plan = read_plan(plan_path) + except Exception as error: + print(f"Error: plan file {plan_path}: {error}" ,file=sys.stderr) + return 1 + + dirty = git_dirty(project) + if dirty and not preview: + print("Error: the project work tree has uncommitted changes. Commit them first:" ,file=sys.stderr) + print(dirty ,file=sys.stderr) + return 1 + dirty = git_dirty(harmony) + if dirty: + print("Error: the Harmony checkout has uncommitted changes; the skeleton is taken" ,file=sys.stderr) + print("from its last commit, and its files would not match it:" ,file=sys.stderr) + print(dirty ,file=sys.stderr) + return 1 + + return work(harmony ,project ,plan ,preview) + + +if __name__ == "__main__": + try: + sys.exit(CLI()) + except BrokenPipeError: + os.dup2(os.open(os.devnull ,os.O_WRONLY) ,sys.stdout.fileno()) + sys.exit(1) diff --git a/shared/project-dependency/Python-3.12.3 b/shared/project-dependency/Python-3.12.3 deleted file mode 120000 index c2895cf..0000000 --- a/shared/project-dependency/Python-3.12.3 +++ /dev/null @@ -1 +0,0 @@ -project/Python-3.12.3 \ No newline at end of file diff --git a/shared/project-dependency/Python.org b/shared/project-dependency/Python.org index 7d2b4e7..8e67c79 100644 --- a/shared/project-dependency/Python.org +++ b/shared/project-dependency/Python.org @@ -14,6 +14,8 @@ sudo zypper install gcc make sqlite3-devel libopenssl-devel libffi-devel zlib-de readline-devel ncurses-devel xz-devel libbz2-devel gdbm-devel tk-devel libuuid-devel #+end_src +zlib-ng-compat-devel serves in place of zlib-devel, and the two conflict; if zypper reports that conflict, keep the one already installed. + tk-devel gives the tkinter module, which the window programs use. * 2. Download and Extract @@ -30,14 +32,18 @@ cd Python-3.12.3 Compile the Python binaries in place. Setting the installation prefix to the current directory isolates the build. #+begin_src bash -./configure --prefix="$(pwd)" --enable-optimizations +CONFIG_SITE= ./configure --prefix="$(pwd)" --libdir="$(pwd)/lib" --enable-optimizations make -j8 2>&1 | tee build.log make install #+end_src +Everything installs inside this directory; nothing is written to the system. + +CONFIG_SITE= and --libdir keep the machine's own defaults out of the build, so the recipe gives the same layout on every Linux. Some distributions, openSUSE and Fedora among them, name a site file in CONFIG_SITE that sets the library directory to lib64 for every configure script. Python then installs its compiled modules under lib64 while the interpreter looks for them under lib, and every import of a compiled module, such as ssl, fails. + The build is static: the interpreter is one program, with no separate libpython3.12.so to find when it starts. A build with --enable-shared records the absolute path of its library directory in the program, and stops working when the directory moves. C extension modules, such as those pip installs, work with the static build. -Near the end of build.log, make lists any optional modules it could not build, under "The necessary bits to build these optional modules were not found". The list should be empty; if it is not, install the missing development library and build again. +Near the end of build.log, make lists any optional modules it could not build, under "The necessary bits to build these optional modules were not found". nis, an obsolete network directory service, and _curses_panel may appear there; the project uses neither. For any other module listed, install the missing development library and build again. * 4. Check the build diff --git a/shared/tool/env_to_emacs b/shared/tool/env_to_emacs index 947efc0..8a65dba 100755 --- a/shared/tool/env_to_emacs +++ b/shared/tool/env_to_emacs @@ -1,30 +1,78 @@ #!/usr/bin/env bash +# env_to_emacs - give the Emacs shell buffer this setup runs in the role's environment +# +# Emacs starts programs (compile, grep, and the like) with its own copy of the +# environment, which a shell running inside it cannot change. This tool sets +# REPO_HOME, PROJECT, ROLE and PATH in Emacs, and exec-path to match, for the +# one shell buffer whose shell this setup runs under, and for no other. +# +# The values are buffer-local. Killing the buffer discards them, and other +# buffers, including shells started later, keep Emacs's own environment. (An +# earlier version set them globally, so every shell started afterward began +# in the last role set up, even after that role's buffer was gone.) +# +# The buffer is found by process: Emacs knows the process id of the shell in +# each shell buffer, and that shell is an ancestor of this script, perhaps +# several generations up, since setup starts a new bash under the buffer's own. + script_afp=$(realpath "${BASH_SOURCE[0]}") set -euo pipefail if [ -z "${REPO_HOME:-}" ]; then - echo "emacs-sync:: error: REPO_HOME is not set. Source a setup environment first." + echo "env_to_emacs: error: REPO_HOME is not set. Source a setup environment first." exit 1 fi if ! command -v emacsclient >/dev/null 2>&1; then - echo "emacs-sync:: error: emacsclient is not installed or not in PATH." + echo "env_to_emacs: error: emacsclient is not installed or not in PATH." exit 1 fi -# Push variables to the Emacs global environment -emacsclient --eval "(setenv \"REPO_HOME\" \"$REPO_HOME\")" > /dev/null -emacsclient --eval "(setenv \"PROJECT\" \"$PROJECT\")" > /dev/null +# quote a value as an Emacs Lisp string +elisp_string(){ + local s="$1" + s=${s//\\/\\\\} + s=${s//\"/\\\"} + printf '"%s"' "$s" +} -if [ -n "${ROLE:-}" ]; then - emacsclient --eval "(setenv \"ROLE\" \"$ROLE\")" > /dev/null -fi +form=$(cat < pid 1) (not buffer)) + (setq buffer + (seq-find (lambda (b) + (let ((p (get-buffer-process b))) + (and p (eql (process-id p) pid)))) + (buffer-list))) + (unless buffer + (setq pid (alist-get 'ppid (process-attributes pid))))) + (if (not buffer) + "no shell buffer found" + (with-current-buffer buffer + (setq-local process-environment (copy-sequence process-environment)) + (setenv "REPO_HOME" $(elisp_string "$REPO_HOME")) + (setenv "PROJECT" $(elisp_string "${PROJECT:-}")) + (setenv "ROLE" $(elisp_string "${ROLE:-}")) + (setenv "PATH" $(elisp_string "$PATH")) + (setq-local exec-path (append (parse-colon-path $(elisp_string "$PATH")) (list exec-directory))) + (buffer-name)))) +EOF +) -# Sync the PATH so Emacs subprocesses find the local tools -emacsclient --eval "(setenv \"PATH\" \"$PATH\")" > /dev/null +if ! result=$(emacsclient --eval "$form" 2>&1); then + echo "env_to_emacs: cannot reach the Emacs server; Emacs environment left unchanged" + echo " $result" + exit 0 +fi +result=${result#\"} +result=${result%\"} -# Update exec-path for native Emacs executable resolution -emacsclient --eval "(setq exec-path (append (parse-colon-path \"$PATH\") (list exec-directory)))" > /dev/null +if [[ "$result" == "no shell buffer found" ]]; then + echo "env_to_emacs: no Emacs shell buffer runs this shell; Emacs environment left unchanged" + exit 0 +fi -echo "Emacs environment synchronized for project: $PROJECT" +echo "Emacs buffer $result has the environment of $PROJECT ${ROLE:-}" diff --git a/shared/tool/makefile/Harmony.mk b/shared/tool/makefile/Harmony.mk index eaf1ce8..b168ac5 100644 --- a/shared/tool/makefile/Harmony.mk +++ b/shared/tool/makefile/Harmony.mk @@ -30,7 +30,9 @@ MACHINE_DIR ?= scratchpad/made LIBRARY_FILE ?= $(LIBRARY_DIR)/lib$(LIBRARY_NAME).a -LN_FLAGS ?= -L$(LIBRARY_DIR) -L/lib64 -L/lib -l$(LIBRARY_NAME) +# the project library is linked only when the namespace makes one +LN_FLAGS ?= -L$(LIBRARY_DIR) -L/lib64 -L/lib $(if $(object_lib),-l$(LIBRARY_NAME)) +# LN_FLAGS ?= -L$(LIBRARY_DIR) -L/lib64 -L/lib -l$(LIBRARY_NAME) KMOD_SOURCE_DIR ?= authored KMOD_CCFLAGS ?= -I $(KMOD_SOURCE_DIR) diff --git a/shared/tool/replace_from_tar b/shared/tool/replace_from_tar new file mode 100755 index 0000000..6da5c0f --- /dev/null +++ b/shared/tool/replace_from_tar @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- +""" +replace_from_tar + +Makes directories hold exactly what a tar file holds. For each directory at +the top of the tar file, such as SubU.copy/, the directory of the same name in +the extract dir is brought to match: files the tar file has are written when +they are new or differ, and files and directories it does not have are +deleted. The extract dir is the directory 'tar -x' would be run in: the one +that holds SubU.copy, not SubU.copy itself. It is the current directory unless +given. Extracting a tar file over a directory only adds and overwrites, so +a file that was removed from the tar file would otherwise stay behind. + +The tool deletes files. Checkpoint the project with git first, and run +preview to read what it would do. + +Usage: replace_from_tar [argument ...] + replace_from_tar preview [argument ...] + help | -h | --help print this message (also printed when given no arguments) + replace [] + bring each directory at the top of the tar file, + in the extract dir, to match the tar file + preview print the log of what would do, change nothing + +Example, run in developer/authored, which holds SubU.copy: + replace_from_tar preview replace SubU.copy.tar + replace_from_tar replace SubU.copy.tar +""" + +import sys ,os ,tarfile +from pathlib import Path, PurePosixPath + +TAG_WIDTH = 12 + + +class Log: + PREVIEW_TAG = {"made": "would make" ,"replaced": "would replace" ,"deleted": "would delete"} + + def __init__(self ,preview): + self.preview = preview + self.count = {} + self.error_count = 0 + + def line(self ,tag ,subject ,reason=None): + self.count[tag] = self.count.get(tag ,0) + 1 + if tag == "unchanged": return + shown = self.PREVIEW_TAG.get(tag ,tag) if self.preview else tag + print(f" {shown:<{TAG_WIDTH + 2}}{subject}" + (f" ({reason})" if reason else "")) + + def error(self ,subject ,reason): + self.error_count += 1 + print(f" {'error':<{TAG_WIDTH + 2}}{subject} ({reason})") + + def summary(self): + print("summary:") + for tag ,what in (("made" ,"files") ,("replaced" ,"files") ,("deleted" ,"files and directories") ,("unchanged" ,"files")): + n = self.count.get(tag ,0) + verb = f"that would be {tag}" if self.preview and tag != "unchanged" else tag + print(f" {n:>4} {what} {verb}") + if self.error_count: + print(f" {self.error_count:>4} errors") + if self.preview: + print("preview only: nothing was changed.") + return 1 if self.error_count else 0 + + +def member_problem(member): + path = PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts: + return "an absolute path or '..'" + if len(path.parts) < 1: + return "an empty name" + if not (member.isfile() or member.isdir() or member.issym()): + return "neither a file, a directory nor a symbolic link" + if member.issym() and (PurePosixPath(member.linkname).is_absolute() or ".." in PurePosixPath(member.linkname).parts): + return "a symbolic link leaving its directory" + return None + +def replace(tar_path ,extract_dir ,log ,preview): + extract_dir = Path(extract_dir) + if not extract_dir.is_dir(): + log.error(str(extract_dir) ,"not a directory"); return + with tarfile.open(tar_path) as tar: + member_list = tar.getmembers() + for m in member_list: + problem = member_problem(m) + if problem: + log.error(m.name ,f"refused: {problem}; nothing was changed"); return + top_set = sorted({PurePosixPath(m.name).parts[0] for m in member_list}) + for top in top_set: + if any(PurePosixPath(m.name) == PurePosixPath(top) and not m.isdir() for m in member_list): + log.error(top ,"refused: the top of the tar file must hold directories; nothing was changed"); return + target = extract_dir / top + if target.is_symlink(): + log.error(str(target) ,"refused: it is a symbolic link; nothing was changed"); return + + wanted = {str(PurePosixPath(m.name)): m for m in member_list} + + # ---- delete what the tar file does not have, deepest first + for top in top_set: + target = extract_dir / top + if not target.exists(): continue + for dirpath ,dirname_list ,filename_list in os.walk(target ,topdown=False ,followlinks=False): + for name in sorted(filename_list) + sorted(d for d in dirname_list if (Path(dirpath) / d).is_symlink()): + path = Path(dirpath) / name + key = str(PurePosixPath(path.relative_to(extract_dir))) + if key not in wanted or wanted[key].isdir(): + if not preview: path.unlink() + log.line("deleted" ,key) + for name in sorted(d for d in dirname_list if not (Path(dirpath) / d).is_symlink()): + path = Path(dirpath) / name + key = str(PurePosixPath(path.relative_to(extract_dir))) + if key not in wanted: + if not preview: + if any(path.iterdir()): + log.error(key ,"not empty after its files were deleted"); continue + path.rmdir() + log.line("deleted" ,key + "/") + + # ---- write what is new or differs + for m in sorted(member_list ,key=lambda m: m.name): + key = str(PurePosixPath(m.name)) + path = extract_dir / key + if m.isdir(): + if not path.is_dir(): + if not preview: path.mkdir(parents=True ,exist_ok=True) + log.line("made" ,key + "/") + continue + if m.issym(): + if path.is_symlink() and os.readlink(path) == m.linkname: + log.line("unchanged" ,key); continue + if not preview: + if path.is_symlink() or path.exists(): path.unlink() + path.parent.mkdir(parents=True ,exist_ok=True) + os.symlink(m.linkname ,path) + log.line("replaced" if path.is_symlink() else "made" ,key); continue + data = tar.extractfile(m).read() + mode = m.mode & 0o777 + if path.is_file() and not path.is_symlink(): + if path.read_bytes() == data and (path.stat().st_mode & 0o777) == mode: + log.line("unchanged" ,key); continue + tag = "replaced" + else: + tag = "made" + if not preview: + path.parent.mkdir(parents=True ,exist_ok=True) + if path.is_symlink(): path.unlink() + path.write_bytes(data) + os.chmod(path ,mode) + log.line(tag ,key) + + +def print_help(): + print(__doc__.strip()) + return 0 + +def CLI(argv=None): + if argv is None: argv = sys.argv[1:] + if not argv or argv[0] in ("help" ,"-h" ,"--help"): return print_help() + preview = argv[0] == "preview" + if preview: argv = argv[1:] + if not argv: + print("Error: 'preview' needs a command after it." ,file=sys.stderr) + return 1 + command ,rest = argv[0] ,argv[1:] + if command == "replace" and len(rest) in (1 ,2): + if len(rest) == 1: rest = [rest[0] ,"."] + if not Path(rest[0]).is_file(): + print(f"Error: no tar file at {rest[0]}." ,file=sys.stderr) + return 1 + log = Log(preview) + print(f"replace_from_tar {'preview ' if preview else ''}replace {rest[0]} in {os.path.abspath(rest[1])}") + replace(rest[0] ,rest[1] ,log ,preview) + return log.summary() + print(f"Error: unknown command or wrong arguments: '{command}'." ,file=sys.stderr) + print_help() + return 1 + + +if __name__ == "__main__": + sys.exit(CLI()) -- 2.20.1