From 4d81d1121da65904853e7a11dc8d99cc3853c552 Mon Sep 17 00:00:00 2001 From: Thomas Walker Lynch Date: Thu, 22 Oct 2026 10:27:20 +0000 Subject: [PATCH] syncing to current Harmony version --- .gitignore | 37 +- .../document/.githolder => 0pus_Man-In-Grey | 0 .../authored/.gitkeep | 0 .../how-to_add-project-dependency.html | 119 ++++ administrator/document/how-to_release.html | 114 ++++ administrator/tool/RT-Style_header_v5_to_v6 | 306 +++++++++ administrator/tool/archive | 284 ++++++++ administrator/tool/html_to_v31 | 63 ++ administrator/tool/new-project | 177 +++++ administrator/tool/setup | 3 + administrator/tool/skeleton_diff | 216 ++++++ administrator/tool/skeleton_update | 621 ++++++++++++++++++ consumer/.gitignore | 5 + .../.githolder => authored/.gitkeep} | 0 .../Man_In_Grey.copy}/Man_In_Grey | 0 .../Man_In_Grey.copy}/Man_In_Grey.py | 0 .../Man_In_Grey.copy}/Planner.py | 0 .../Man_In_Grey.copy}/diag_perm | 0 .../Man_In_Grey.copy}/executor_inner.py | 0 .../Man_In_Grey.copy}/executor_outer.py | 0 .../man_in_grey_apply.CLI.c} | 0 .../developer}/machine/man_in_grey_apply | Bin .../deprecated/developer}/tool/clean_release | 0 .../deprecated/developer}/tool/compile | 0 .../deprecated/developer}/tool/env | 0 .../deprecated/developer}/tool/release | 0 .../deprecated/document}/Workflow.org | 0 .../document}/environment_guide.org | 0 .../deprecated/document}/general_info.org | 0 .../authored/deprecated/env_developer | 0 .../authored/deprecated/env_tester | 0 .../authored/deprecated/env_toolsmith | 0 .../authored/deprecated/tester}/tool/env | 0 .../authored/deprecated/tool}/bless | 0 .../authored/deprecated/tool}/unbless | 0 .../deprecated/tool_shared}/bespoke/env | 0 .../tool_shared}/bespoke/print_project_env.py | 0 .../deprecated/tool_shared}/bespoke/version | 0 .../deprecated/tool_shared}/bespoke/vl | 0 .../tool_shared}/document/install_Python.org | 0 .../document/naming_file-and-directory.html | 101 +++ .../single-file_C-module-and-namespace.html | 184 ++++++ developer/{source => document}/todo.txt | 0 .../experiment/.gitkeep | 0 developer/scratchpad/.gitignore | 2 + developer/tool/build | 123 ++++ developer/tool/build_component/copy | 59 ++ developer/tool/build_component/make | 15 + developer/tool/build_component/makefile | 55 ++ developer/tool/build_component/sync_engine.py | 80 +++ developer/tool/promote | 261 ++++++++ developer/tool/setup | 3 + document/introduction_Harmony.html | 249 +++++++ ...role-and-workflow_product-development.html | 134 ++++ ...role-and-workflow_product-maintenance.html | 111 ++++ document/writing_tool-script.html | 287 ++++++++ release/python3/Man_In_Grey.py | 346 ---------- release/python3/Planner.py | 515 --------------- release/python3/executor_inner.py | 473 ------------- release/python3/executor_outer.py | 487 -------------- release/shell/Man_In_Grey | 62 -- release/shell/diag_perm | 194 ------ release/x86_64/man_in_grey_apply | Bin 17232 -> 0 bytes setup | 82 +++ .../.githolder => shared/authored/.gitkeep | 0 shared/project-dependency/.gitignore | 11 + shared/project-dependency/Python.init | 25 + shared/project-dependency/Python.org | 71 ++ shared/project-dependency/project | 1 + .../shell/.githolder => shared/tool/PATH.sh | 0 shared/tool/env_to_emacs | 78 +++ shared/tool/makefile/Harmony.mk | 137 ++++ shared/tool/makefile/RT_global.h | 11 + shared/tool/makefile/target_kernel-module.mk | 130 ++++ shared/tool/replace_from_tar | 182 +++++ shared/tool/scratchpad | 225 +++++++ shared/tool/setup | 100 +++ shared/tool/version | 3 + release/x86_64/.githolder => tester/.gitkeep | 0 tester/authored/.gitkeep | 0 .../test_0/Man_In_Grey_input_acceptance.py | 0 tester/{ => authored}/test_0/cat_stage | 0 tester/{ => authored}/test_0/clean_out | 0 tester/{ => authored}/test_0/ls_stage | 0 tester/{ => authored}/test_0/run | 0 tester/{ => authored}/test_0/run_check | 0 .../test_0/run_check.transcript | 0 .../test_0/stage_test_0/DNS/unbound.conf.py | 0 .../test_0/stage_test_0/unbound_conf.py | 0 .../test_0/stage_test_0/web/site_conf.py | 0 .../test_0/stage_test_0_out/net/unbound.conf | 0 .../test_0/stage_test_0_out/unbound_conf | 0 .../unbound_conf.20250920T161232Z | 0 .../test_0/stage_test_0_out/web/nginx.conf | 0 .../web/nginx.conf.20250920T161233Z | 0 .../net/unbound.conf | 0 .../stage_test_0_out_expected_0/unbound_conf | 0 .../web/nginx.conf | 0 .../net/unbound.conf | 0 .../stage_test_0_out_expected_1/unbound_conf | 0 .../unbound_conf.20250920T160648Z | 0 .../web/nginx.conf | 0 .../web/nginx.conf.20250920T160648Z | 0 tester/tool/setup | 3 + 104 files changed, 4656 insertions(+), 2089 deletions(-) rename developer/document/.githolder => 0pus_Man-In-Grey (100%) rename developer/source/deprecated/.githolder => administrator/authored/.gitkeep (100%) create mode 100644 administrator/document/how-to_add-project-dependency.html create mode 100644 administrator/document/how-to_release.html create mode 100755 administrator/tool/RT-Style_header_v5_to_v6 create mode 100755 administrator/tool/archive create mode 100755 administrator/tool/html_to_v31 create mode 100755 administrator/tool/new-project create mode 100644 administrator/tool/setup create mode 100755 administrator/tool/skeleton_diff create mode 100755 administrator/tool/skeleton_update create mode 100644 consumer/.gitignore rename developer/{source/scratchpad/.githolder => authored/.gitkeep} (100%) rename developer/{source => authored/Man_In_Grey.copy}/Man_In_Grey (100%) mode change 100644 => 100755 rename developer/{source => authored/Man_In_Grey.copy}/Man_In_Grey.py (100%) mode change 100644 => 100755 rename developer/{source => authored/Man_In_Grey.copy}/Planner.py (100%) rename developer/{source => authored/Man_In_Grey.copy}/diag_perm (100%) mode change 100644 => 100755 rename developer/{source => authored/Man_In_Grey.copy}/executor_inner.py (100%) mode change 100644 => 100755 rename developer/{source => authored/Man_In_Grey.copy}/executor_outer.py (100%) rename developer/{source/Man_In_Grey_apply.c => authored/Man_In_Grey_apply.make/man_in_grey_apply.CLI.c} (100%) rename developer/{ => authored/deprecated/developer}/machine/man_in_grey_apply (100%) rename developer/{ => authored/deprecated/developer}/tool/clean_release (100%) rename developer/{ => authored/deprecated/developer}/tool/compile (100%) rename developer/{ => authored/deprecated/developer}/tool/env (100%) rename developer/{ => authored/deprecated/developer}/tool/release (100%) rename {document => developer/authored/deprecated/document}/Workflow.org (100%) rename {document => developer/authored/deprecated/document}/environment_guide.org (100%) rename {document => developer/authored/deprecated/document}/general_info.org (100%) rename env_developer => developer/authored/deprecated/env_developer (100%) rename env_tester => developer/authored/deprecated/env_tester (100%) rename env_toolsmith => developer/authored/deprecated/env_toolsmith (100%) rename {tester => developer/authored/deprecated/tester}/tool/env (100%) rename {tool => developer/authored/deprecated/tool}/bless (100%) rename {tool => developer/authored/deprecated/tool}/unbless (100%) rename {tool_shared => developer/authored/deprecated/tool_shared}/bespoke/env (100%) rename {tool_shared => developer/authored/deprecated/tool_shared}/bespoke/print_project_env.py (100%) rename {tool_shared => developer/authored/deprecated/tool_shared}/bespoke/version (100%) rename {tool_shared => developer/authored/deprecated/tool_shared}/bespoke/vl (100%) rename {tool_shared => developer/authored/deprecated/tool_shared}/document/install_Python.org (100%) create mode 100644 developer/document/naming_file-and-directory.html create mode 100644 developer/document/single-file_C-module-and-namespace.html rename developer/{source => document}/todo.txt (100%) rename document/.githolder => developer/experiment/.gitkeep (100%) create mode 100644 developer/scratchpad/.gitignore create mode 100755 developer/tool/build create mode 100755 developer/tool/build_component/copy create mode 100755 developer/tool/build_component/make create mode 100644 developer/tool/build_component/makefile create mode 100644 developer/tool/build_component/sync_engine.py create mode 100755 developer/tool/promote create mode 100644 developer/tool/setup create mode 100644 document/introduction_Harmony.html create mode 100644 document/role-and-workflow_product-development.html create mode 100644 document/role-and-workflow_product-maintenance.html create mode 100644 document/writing_tool-script.html delete mode 100755 release/python3/Man_In_Grey.py delete mode 100644 release/python3/Planner.py delete mode 100755 release/python3/executor_inner.py delete mode 100755 release/python3/executor_outer.py delete mode 100755 release/shell/Man_In_Grey delete mode 100755 release/shell/diag_perm delete mode 100755 release/x86_64/man_in_grey_apply create mode 100755 setup rename release/python3/.githolder => shared/authored/.gitkeep (100%) create mode 100644 shared/project-dependency/.gitignore create mode 100644 shared/project-dependency/Python.init create mode 100644 shared/project-dependency/Python.org create mode 120000 shared/project-dependency/project rename release/shell/.githolder => shared/tool/PATH.sh (100%) create mode 100755 shared/tool/env_to_emacs create mode 100644 shared/tool/makefile/Harmony.mk create mode 100644 shared/tool/makefile/RT_global.h create mode 100644 shared/tool/makefile/target_kernel-module.mk create mode 100755 shared/tool/replace_from_tar create mode 100755 shared/tool/scratchpad create mode 100644 shared/tool/setup create mode 100755 shared/tool/version rename release/x86_64/.githolder => tester/.gitkeep (100%) create mode 100644 tester/authored/.gitkeep rename tester/{ => authored}/test_0/Man_In_Grey_input_acceptance.py (100%) rename tester/{ => authored}/test_0/cat_stage (100%) rename tester/{ => authored}/test_0/clean_out (100%) rename tester/{ => authored}/test_0/ls_stage (100%) rename tester/{ => authored}/test_0/run (100%) rename tester/{ => authored}/test_0/run_check (100%) rename tester/{ => authored}/test_0/run_check.transcript (100%) rename tester/{ => authored}/test_0/stage_test_0/DNS/unbound.conf.py (100%) rename tester/{ => authored}/test_0/stage_test_0/unbound_conf.py (100%) rename tester/{ => authored}/test_0/stage_test_0/web/site_conf.py (100%) rename tester/{ => authored}/test_0/stage_test_0_out/net/unbound.conf (100%) rename tester/{ => authored}/test_0/stage_test_0_out/unbound_conf (100%) rename tester/{ => authored}/test_0/stage_test_0_out/unbound_conf.20250920T161232Z (100%) rename tester/{ => authored}/test_0/stage_test_0_out/web/nginx.conf (100%) rename tester/{ => authored}/test_0/stage_test_0_out/web/nginx.conf.20250920T161233Z (100%) rename tester/{ => authored}/test_0/stage_test_0_out_expected_0/net/unbound.conf (100%) rename tester/{ => authored}/test_0/stage_test_0_out_expected_0/unbound_conf (100%) rename tester/{ => authored}/test_0/stage_test_0_out_expected_0/web/nginx.conf (100%) rename tester/{ => authored}/test_0/stage_test_0_out_expected_1/net/unbound.conf (100%) rename tester/{ => authored}/test_0/stage_test_0_out_expected_1/unbound_conf (100%) rename tester/{ => authored}/test_0/stage_test_0_out_expected_1/unbound_conf.20250920T160648Z (100%) rename tester/{ => authored}/test_0/stage_test_0_out_expected_1/web/nginx.conf (100%) rename tester/{ => authored}/test_0/stage_test_0_out_expected_1/web/nginx.conf.20250920T160648Z (100%) create mode 100644 tester/tool/setup diff --git a/.gitignore b/.gitignore index 7c79eb8..fd15d33 100644 --- a/.gitignore +++ b/.gitignore @@ -1,25 +1,38 @@ -# ─── Python Detritus ────────────────────────────────────────────── + +# no archive or temporary files +*.tar +*.tgz +*.zip +*. +temp.* +**/scratchpad/* +!**/scratchpad/.gitignore + + +# Python __pycache__/ *.py[cod] *.pyo *.pyd - -# pytest cache +.ipynb_checkpoints/ .pytest_cache/ -# Cython / coverage / profiling +# C / compiled artifacts +*.o +*.a +*.out *.so -*.gcov -*.gcda -*.gcno - -# IPython junk -.ipynb_checkpoints/ -# editor backup files (optional) +# OS and editor backup files *~ +*.bak +*.swp +.DS_Store -# ─── Virtualenv (just in case someone makes one elsewhere) ──────── +# Man-In-Grey +*.gcov +*.gcda +*.gcno venv/ .venv/ env/ diff --git a/developer/document/.githolder b/0pus_Man-In-Grey similarity index 100% rename from developer/document/.githolder rename to 0pus_Man-In-Grey diff --git a/developer/source/deprecated/.githolder b/administrator/authored/.gitkeep similarity index 100% rename from developer/source/deprecated/.githolder rename to administrator/authored/.gitkeep diff --git a/administrator/document/how-to_add-project-dependency.html b/administrator/document/how-to_add-project-dependency.html new file mode 100644 index 0000000..327a28c --- /dev/null +++ b/administrator/document/how-to_add-project-dependency.html @@ -0,0 +1,119 @@ + + + + + How to add a project dependency + + + + + + + + + + + +

Project dependencies

+ +

+ A project dependency is software the project uses but did not author: a language toolchain, a library, a tool, or another project. A dependency is installed outside the project, as a sibling directory of it, and is linked into the project under shared/project-dependency. +

+ +

+ Getting an arbitrary piece of software working is different every time, so the administrator is free to do whatever a dependency needs inside shared/project-dependency. The skeleton relies on only the few conventions given in the next section. +

+ +

What the skeleton relies on

+ + + +

+ Files for one dependency share a base name, so that Python.init, Python.org, and the link they describe sit together in a listing. +

+ +

Steps

+ +

Install the dependency beside the project

+

+ Download, clone, or extract the dependency into the directory that holds the project, so that it sits as a sibling of the project directory. Build it there if it needs building. +

+ +

Link it

+

+ Make the link through the project link. The word project in the link target is literal. +

+ + > cd $REPO_HOME/shared/project-dependency + > ln -s project/<dependency directory> <name> + + +

Write the init file

+

+ Write <name>.init in the same directory. Because it runs on every setup for every role, keep it to setting the environment: +

+ +

+ Python.init, which comes with the skeleton, is an example: +

+ + if [[ -d "$REPO_HOME/shared/project-dependency/Python-3.12.3/bin" ]]; then + PATH="$REPO_HOME/shared/project-dependency/Python-3.12.3/bin:$PATH" + else + if [[ "$ROLE" == "administrator" ]]; then + echo "Notice: Local Python-3.12.3 bin not found. Using system Python." + fi + fi + + +

Write the installation notes

+

+ Record what was installed, where it came from, and the commands used, in <name>.org, <name>.html, or comments in the init file. A team member who has cloned the project follows these notes to install the dependency, and then recreates the link if it is not tracked. +

+ +

Decide what is tracked

+

+ The skeleton's .gitignore tracks itself, the project link, and files ending in .init, .install, .org, and .html. Everything else in the directory is ignored. To track a dependency's link, so that it arrives with every clone, add a line naming it: +

+ + !<name> + +

+ A tracked link points at nothing until the dependency is installed beside the project, which is why init files test for the dependency before using it. +

+ +

Commit

+

+ Add the init file, the notes, the .gitignore change, and any tracked link, then commit. +

+ + + + diff --git a/administrator/document/how-to_release.html b/administrator/document/how-to_release.html new file mode 100644 index 0000000..0f33c4a --- /dev/null +++ b/administrator/document/how-to_release.html @@ -0,0 +1,114 @@ + + + + + + + Release howto + + + + + + + + + + + +

Prerequisites

+

+ A developer works in the developer branch of the project, and makes use of the build and promote tools to stage release candidates into the $REPO_HOME/consumer directory. The tester then validates the release candidate. This constitutes a cycle of work, developing, promoting candidates, testing, and then further developing. Once the project manager agrees this cycle is complete, the release flow moves to the next step. +

+ +

Make the release branch

+

+ The administrator executes the release. Open a terminal. Enter the project workspace. +

+ + > . setup administrator + + +

+ Switch to the core development branch. Ensure the local repository is current. +

+ + > git checkout core-developer_branch + > git pull + + +

+ Make a new branch for the release. Use the major version number. +

+ + > git checkout -b release_v<major> + + +

Issue management and patching

+

+ Only critical issues receive patches on a release branch. The product manager defines what is critical. If approved, the release team applies the fix directly to the existing release_v<major> branch. +

+

+ The branch name release_v<major> remains static. The code on the branch is updated. The administrator advances the minor release number in the shared/tool/version file. +

+

+ The administrator tags the new commit with the full version number, such as v<major>.<minor>. Minor versions are edits on the major release branch. They are visible to a person by running the version command. +

+

+ The developer must ensure the fix is also ported to the core-developer_branch. This prevents the defect from reappearing in future major releases. +

+ +

Tag and push

+

+ Tag the release with the exact version number. +

+ + > git tag -a v<major>.<minor> -m "Release v<major>.<minor>" + + +

+ Push the branch to the remote repository. Push the tags. +

+ + > git push origin release_v<major> + > git push origin --tags + + +

Repository default branch

+

+ Set the repository default branch to the new release_v<major> branch on the hosting platform. This ensures a person cloning or pulling the repository receives the most recent major release code by default. +

+ +

Verification

+

+ Check the remote repository. Confirm the branch exists. Confirm the tag is visible. Confirm the default branch is updated. +

+ +

Role responsibilities

+ + + + + diff --git a/administrator/tool/RT-Style_header_v5_to_v6 b/administrator/tool/RT-Style_header_v5_to_v6 new file mode 100755 index 0000000..3ca4169 --- /dev/null +++ b/administrator/tool/RT-Style_header_v5_to_v6 @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- + +""" +RT-Style_header_v5_to_v6 + +Converts RT-Style version 5 document headers to version 6. + +A version 5 document loads the engine through a locator file beside it: + + + +A version 6 document carries an inline header block instead. The block takes +the library from the record made by opening a Manuscript directory's +setup.html, and falls back to the Reasoning Technology server. + +For each version 5 .html file found, the tool replaces the locator line in + with the version 6 block, keeping its indentation. It then deletes +RT-Manuscript_locator.js from that file's directory, unless another version 5 +document there, not converted in this run, still uses it. Any other mention of +RT-Manuscript_locator in the file (an escaped example, prose) is logged for a +person to edit. + +Usage: RT-Style_header_v5_to_v6 [file ...] + RT-Style_header_v5_to_v6 preview [file ...] + + help | -h | --help print this message (also printed when given no arguments) + project walk the directory tree rooted at $REPO_HOME + tree walk the directory tree rooted at the current directory + local only the current directory, no subdirectories + file ... only the named .html files + preview print the log of what would do, change nothing + +Every directory is walked, whatever its name. Symbolic links to directories +are not followed, so a link back up the tree cannot cause a loop or reach +outside it. +""" + +import sys ,os ,re +from pathlib import Path + +LOCATOR_FILE = "RT-Manuscript_locator.js" +V6_MARK = 'localStorage.getItem("RT.dirpr_library")' + +V6_BLOCK = [ + '' +] + +LOCATOR_LINE = re.compile( + r'^(?P[ \t]*)\s*\s*$' + ,re.IGNORECASE +) +HEAD_END = re.compile(r'' ,re.IGNORECASE) + + +# ---------------------------------------------------------------- log + +class Log: + def __init__(self ,preview: bool ,base: Path): + self.preview = preview + self.base = base + self.count = {"converted": 0 ,"deleted": 0 ,"already": 0 ,"hand": 0 + ,"untouched": 0 ,"stray": 0 ,"error": 0} + + def rel(self ,p: Path) -> str: + try: return str(p.resolve().relative_to(self.base)) + except ValueError: return str(p) + + def line(self ,tag: str ,text: str): + print(f" {tag:<15} {text}") + + def act(self ,done: str ,would: str ,p: Path ,key: str): + self.line(would if self.preview else done ,self.rel(p)) + self.count[key] += 1 + + def summary(self): + c = self.count + v = "would be " if self.preview else "" + print() + print("summary:") + print(f" {c['converted']:>4} .html files {v}converted to version 6") + print(f" {c['deleted']:>4} {LOCATOR_FILE} files {v}deleted") + print(f" {c['already']:>4} .html files already version 6") + print(f" {c['hand']:>4} .html files with mentions to edit by hand") + print(f" {c['untouched']:>4} other .html files left alone") + if c['stray']: + print(f" {c['stray']:>4} {LOCATOR_FILE} files kept (see 'kept' lines)") + if c['error']: + print(f" {c['error']:>4} errors") + if self.preview: + print("preview only: nothing was changed") + + +# ---------------------------------------------------------------- finding files + +def walk(root: Path ,recursive: bool): + """Yields (directory ,[.html files] ,has_locator) for each directory.""" + if not recursive: + entries = sorted(root.iterdir()) + yield root ,[e for e in entries if e.is_file() and e.suffix == ".html"] ,(root / LOCATOR_FILE).is_file() + return + for d ,dirs ,files in os.walk(root): + dirs.sort() + files = sorted(files) + yield Path(d) ,[Path(d) / f for f in files if f.endswith(".html")] ,LOCATOR_FILE in files + + +# ---------------------------------------------------------------- conversion + +def convert(text: str): + """Returns (new_text ,replaced_count ,leftover). leftover lists + (line_number ,line) for locator mentions that were not replaced.""" + lines = text.split("\n") + out = [] + replaced = 0 + leftover = [] + in_head = True + for n ,line in enumerate(lines ,1): + m = LOCATOR_LINE.match(line) if in_head else None + if m: + eol = "\r" if line.endswith("\r") else "" + out.extend(m.group("indent") + b + eol for b in V6_BLOCK) + replaced += 1 + else: + out.append(line) + if LOCATOR_FILE.removesuffix(".js") in line: + leftover.append((n ,line.strip())) + if in_head and HEAD_END.search(line): + in_head = False + return "\n".join(out) ,replaced ,leftover + +def do_file(f: Path ,log: Log) -> bool: + """Converts one .html file. Returns True when it was version 5.""" + try: + text = f.read_text(encoding="utf-8") + except Exception as e: + log.line("error" ,f"{log.rel(f)}: cannot read: {e}") + log.count["error"] += 1 + return False + + if V6_MARK in text and LOCATOR_FILE.removesuffix(".js") not in text: + log.line("already v6" ,log.rel(f)) + log.count["already"] += 1 + return False + + new_text ,replaced ,leftover = convert(text) + + if replaced: + if not log.preview: + try: + mode = f.stat().st_mode + f.write_text(new_text ,encoding="utf-8") + os.chmod(f ,mode) + except Exception as e: + log.line("error" ,f"{log.rel(f)}: cannot write: {e}") + log.count["error"] += 1 + return False + log.act("converted" ,"would convert" ,f ,"converted") + if replaced > 1: + log.line("note" ,f"{log.rel(f)}: {replaced} locator lines in , all replaced") + elif not leftover: + log.count["untouched"] += 1 + + if leftover: + log.count["hand"] += 1 + for n ,line in leftover: + log.line("edit by hand" ,f"{log.rel(f)}:{n}: {line[:100]}") + + return replaced > 0 + +def uses_locator(f: Path) -> bool: + """True when f still loads the engine through a locator line in .""" + try: + text = f.read_text(encoding="utf-8") + except Exception: + return False + for line in text.split("\n"): + if LOCATOR_LINE.match(line): return True + if HEAD_END.search(line): return False + return False + +def do_locator(d: Path ,converted: set ,log: Log): + """Deletes d's locator unless a version 5 document not converted in this + run still uses it.""" + loc = d / LOCATOR_FILE + users = sorted(f.name for f in d.iterdir() + if f.is_file() and f.suffix == ".html" + and f.resolve() not in converted and uses_locator(f)) + if users: + shown = ", ".join(users[:3]) + (" ..." if len(users) > 3 else "") + log.line("kept" ,f"{log.rel(loc)} (still used by {shown})") + log.count["stray"] += 1 + return + if not log.preview: + try: + loc.unlink() + except Exception as e: + log.line("error" ,f"{log.rel(loc)}: cannot delete: {e}") + log.count["error"] += 1 + return + log.act("deleted" ,"would delete" ,loc ,"deleted") + +def run_dirs(root: Path ,recursive: bool ,log: Log): + for d ,html ,has_locator in walk(root ,recursive): + converted = set() + for f in html: + if do_file(f ,log): converted.add(f.resolve()) + if has_locator: + if converted: + do_locator(d ,converted ,log) + else: + log.line("kept" ,f"{log.rel(d / LOCATOR_FILE)} (no version 5 document beside it)") + log.count["stray"] += 1 + +def run_files(files: list[Path] ,log: Log): + converted = set() + dirs = [] + for f in files: + if do_file(f ,log): + converted.add(f.resolve()) + d = f.parent.resolve() + if d not in dirs: dirs.append(d) + for d in dirs: + if (d / LOCATOR_FILE).is_file(): + do_locator(d ,converted ,log) + + +# ---------------------------------------------------------------- command line + +def print_help() -> int: + print(__doc__.strip()) + return 0 + +def CLI(argv=None) -> int: + if argv is None: + argv = sys.argv[1:] + if not argv or argv[0] in ("help" ,"-h" ,"--help"): + return print_help() + + preview = False + if argv[0] == "preview": + preview = True + argv = argv[1:] + if not argv: + print("Error: 'preview' needs a command after it.\n" ,file=sys.stderr) + print_help() + return 1 + + command ,rest = argv[0] ,argv[1:] + + if command == "file": + if not rest: + print("Error: 'file' needs at least one file name." ,file=sys.stderr) + return 1 + files = [Path(a) for a in rest] + for f in files: + if not f.is_file(): + print(f"Error: {f} is not a file." ,file=sys.stderr) + return 1 + base = Path.cwd().resolve() + log = Log(preview ,base) + print(f"RT-Style_header_v5_to_v6 {'preview ' if preview else ''}file in {base}") + run_files(files ,log) + log.summary() + return 1 if log.count["error"] else 0 + + if rest: + print(f"Error: '{command}' takes no arguments." ,file=sys.stderr) + return 1 + + if command == "project": + repo_home = os.environ.get("REPO_HOME") + if not repo_home: + print("Error: 'project' needs REPO_HOME to be set." ,file=sys.stderr) + return 1 + root ,recursive = Path(repo_home) ,True + elif command == "tree": + root ,recursive = Path.cwd() ,True + elif command == "local": + root ,recursive = Path.cwd() ,False + else: + print(f"Error: unknown command '{command}'.\n" ,file=sys.stderr) + print_help() + return 1 + + if not root.is_dir(): + print(f"Error: {root} is not a directory." ,file=sys.stderr) + return 1 + + base = root.resolve() + log = Log(preview ,base) + print(f"RT-Style_header_v5_to_v6 {'preview ' if preview else ''}{command} in {base}") + run_dirs(base ,recursive ,log) + log.summary() + return 1 if log.count["error"] else 0 + +if __name__ == "__main__": + sys.exit(CLI()) diff --git a/administrator/tool/archive b/administrator/tool/archive new file mode 100755 index 0000000..a60a0a0 --- /dev/null +++ b/administrator/tool/archive @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- + +""" +archive - Create an archive of the current Git repo's ref into ./scratchpad + +Commands (order-insensitive): + archive # default: zip (HEAD, ./scratchpad, Z-stamp if importable) + archive help # show help + archive version # show version + archive ref- # choose ref (tag/branch/commit), default HEAD + archive out- # choose output directory (default: /scratchpad) + archive no-stamp # force omit timestamp even if Z is importable + archive z-format- # override timestamp format used with Z (optional) + archive zip # write .zip + archive tgz # write .tgz (compressed tar) + archive tar # write .tar (uncompressed tar) + +Output names: + __[__].zip + __[__].tgz + __[__].tar +""" + +from __future__ import annotations +import gzip ,os ,pathlib ,subprocess ,sys +from typing import Optional +import importlib ,importlib.util +from importlib.machinery import SourceFileLoader + +VERSION = "2.0" + +# ---------------------------------------------------------------------- +# Editable timestamp format (used when calling Z) +# ---------------------------------------------------------------------- +Z_FORMAT = "%year-%month-%day_%hour%minute%secondZ" + +USAGE = f"""archive {VERSION} + +Usage: + archive [commands...] + +Commands (order-insensitive): + help + version + ref- + out- + no-stamp + z-format- + zip + tgz + tar + +Examples: + archive + archive tgz + archive ref-main out-/tmp + archive z-format-%year-%month-%dayT%hour:%minute:%second.%scintillaZ +""".rstrip() + +# ---------------------------------------------------------------------- +# git helpers +# ---------------------------------------------------------------------- +def _run(*args: str ,check: bool = True ,cwd: Optional[pathlib.Path] = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + args + ,check=check + ,cwd=(str(cwd) if cwd else None) + ,text=True + ,stdout=subprocess.PIPE + ,stderr=subprocess.PIPE + ) + +def _in_git_repo() -> bool: + try: + return _run("git" ,"rev-parse" ,"--is-inside-work-tree").stdout.strip().lower() == "true" + except subprocess.CalledProcessError: + return False + +def _git_top() -> pathlib.Path: + return pathlib.Path(_run("git" ,"rev-parse" ,"--show-toplevel").stdout.strip()) + +def _git_ref_label(repo_top: pathlib.Path ,ref: str) -> str: + try: + return _run("git" ,"-C" ,str(repo_top) ,"describe" ,"--tags" ,"--always" ,"--dirty" ,ref).stdout.strip() + except subprocess.CalledProcessError: + return _run("git" ,"-C" ,str(repo_top) ,"rev-parse" ,"--short" ,ref).stdout.strip() + +# ---------------------------------------------------------------------- +# Z module discovery (supports extension-less file named "Z") +# ---------------------------------------------------------------------- +def _import_Z_module(repo_top: pathlib.Path) -> Optional[object]: + try: + return importlib.import_module("Z") + except Exception: + pass + + candidates: list[pathlib.Path] = [] + here = pathlib.Path(__file__).resolve().parent + candidates += [here / "Z" ,here / "Z.py"] + candidates += [ + repo_top / "shared" / "third_party" / "RT-project-share" / "release" / "python" / "Z" + ,repo_top / "shared" / "third_party" / "RT-project-share" / "release" / "python" / "Z.py" + ,repo_top / "shared" / "third_party" / "RT-project-share" / "release" / "bash" / "Z" + ] + for d in (pathlib.Path(p) for p in (os.getenv("PATH") or "").split(os.pathsep) if p): + p = d / "Z" + if p.exists() and p.is_file(): + candidates.append(p) + + for path in candidates: + try: + if not path.exists() or not path.is_file(): continue + spec = importlib.util.spec_from_loader("Z" ,SourceFileLoader("Z" ,str(path))) + if not spec or not spec.loader: continue + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + if hasattr(mod ,"make_timestamp") or (hasattr(mod ,"get_utc_dict") and hasattr(mod ,"format_timestamp")): + return mod + except Exception: + continue + return None + +# ---------------------------------------------------------------------- +# Z stamp helper +# ---------------------------------------------------------------------- +def make_z_stamp(zmod: object ,z_format: str) -> Optional[str]: + try: + if hasattr(zmod ,"make_timestamp"): + s = zmod.make_timestamp(fmt=z_format) + return (str(s).strip().replace("\n" ,"") or None) + if hasattr(zmod ,"get_utc_dict") and hasattr(zmod ,"format_timestamp"): + td = zmod.get_utc_dict() + s = zmod.format_timestamp(td ,z_format) + return (str(s).strip().replace("\n" ,"") or None) + except Exception: + return None + return None + +# ---------------------------------------------------------------------- +# archiving +# ---------------------------------------------------------------------- +def _stream_git_archive_tar(repo_top: pathlib.Path ,prefix: str ,ref: str ,out_path: pathlib.Path ,compress: bool) -> None: + proc = subprocess.Popen( + ["git" ,"-C" ,str(repo_top) ,"archive" ,"--format=tar" ,f"--prefix={prefix}/" ,ref] + ,stdout=subprocess.PIPE + ) + try: + if compress: + with gzip.open(out_path ,"wb") as f: + while True: + chunk = proc.stdout.read(1024 * 1024) + if not chunk: break + f.write(chunk) + else: + with open(out_path ,"wb") as f: + while True: + chunk = proc.stdout.read(1024 * 1024) + if not chunk: break + f.write(chunk) + finally: + if proc.stdout: proc.stdout.close() + rc = proc.wait() + if rc != 0: + try: + out_path.unlink(missing_ok=True) + finally: + raise subprocess.CalledProcessError(rc ,proc.args) + +def _stream_git_archive_zip(repo_top: pathlib.Path ,prefix: str ,ref: str ,out_zip_path: pathlib.Path) -> None: + proc = subprocess.Popen( + ["git" ,"-C" ,str(repo_top) ,"archive" ,"--format=zip" ,f"--prefix={prefix}/" ,ref] + ,stdout=subprocess.PIPE + ) + try: + with open(out_zip_path ,"wb") as f: + while True: + chunk = proc.stdout.read(1024 * 1024) + if not chunk: break + f.write(chunk) + finally: + if proc.stdout: proc.stdout.close() + rc = proc.wait() + if rc != 0: + try: + out_zip_path.unlink(missing_ok=True) + finally: + raise subprocess.CalledProcessError(rc ,proc.args) + +# ---------------------------------------------------------------------- +# work function +# ---------------------------------------------------------------------- +def work( + ref: str = "HEAD" + ,outdir: Optional[pathlib.Path] = None + ,force_no_stamp: bool = False + ,z_format: Optional[str] = None + ,archive_kind: str = "zip" +) -> pathlib.Path: + + if archive_kind not in ("zip" ,"tgz" ,"tar"): + raise RuntimeError("archive_kind must be 'zip', 'tgz', or 'tar'") + + if not _in_git_repo(): + raise RuntimeError("not inside a git repository") + + repo_top = _git_top() + repo_name = repo_top.name + ref_label = _git_ref_label(repo_top ,ref) + + stamp: Optional[str] = None + if not force_no_stamp: + zmod = _import_Z_module(repo_top) + if zmod is not None: + stamp = make_z_stamp(zmod ,z_format or Z_FORMAT) + + target_dir = (outdir or (repo_top / "scratchpad")) + target_dir.mkdir(parents=True ,exist_ok=True) + + if archive_kind == "zip": + suffix = ".zip" + elif archive_kind == "tgz": + suffix = ".tgz" + else: + suffix = ".tar" + + out_name = f"{repo_name}__{ref_label}{('__' + stamp) if stamp else ''}{suffix}" + out_path = target_dir / out_name + + if archive_kind == "zip": + _stream_git_archive_zip(repo_top ,repo_name ,ref ,out_path) + else: + compress = (archive_kind == "tgz") + _stream_git_archive_tar(repo_top ,repo_name ,ref ,out_path ,compress) + + return out_path + +# ---------------------------------------------------------------------- +# CLI with command tokens +# ---------------------------------------------------------------------- +def CLI(argv: Optional[list[str]] = None) -> int: + if argv is None: argv = sys.argv[1:] + + ref = "HEAD" + outdir: Optional[pathlib.Path] = None + force_no_stamp = False + z_format: Optional[str] = None + archive_kind = "zip" + + if not argv: + try: + print(f"Wrote {work(ref=ref ,outdir=outdir ,force_no_stamp=force_no_stamp ,z_format=z_format ,archive_kind=archive_kind)}") + return 0 + except Exception as e: + print(f"archive: {e}" ,file=sys.stderr); return 1 + + for arg in argv: + if arg in ("help" ,"-h" ,"--help"): print(USAGE); return 0 + if arg == "version": print(f"archive {VERSION}"); return 0 + if arg == "no-stamp": force_no_stamp = True; continue + if arg == "zip": archive_kind = "zip"; continue + if arg == "tgz": archive_kind = "tgz"; continue + if arg == "tar": archive_kind = "tar"; continue + if arg.startswith("ref-"): ref = arg[4:] or ref; continue + if arg.startswith("out-"): + od = arg[4:] + outdir = pathlib.Path(od).resolve() if od else None + continue + if arg.startswith("z-format-"): + z_format = arg[len("z-format-"):] or None + continue + print(f"archive: unknown command '{arg}'" ,file=sys.stderr); return 1 + + try: + out_path = work(ref=ref ,outdir=outdir ,force_no_stamp=force_no_stamp ,z_format=z_format ,archive_kind=archive_kind) + except Exception as e: + print(f"archive: {e}" ,file=sys.stderr); return 1 + + print(f"Wrote {out_path}") + return 0 + +# ---------------------------------------------------------------------- +if __name__ == "__main__": + raise SystemExit(CLI()) diff --git a/administrator/tool/html_to_v31 b/administrator/tool/html_to_v31 new file mode 100755 index 0000000..b1aba38 --- /dev/null +++ b/administrator/tool/html_to_v31 @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- +""" +html_update_to_3-1 + +Replaces specific StyleRT script blocks with RT 3.1 loader blocks using +literal string replacement. +""" + +import sys ,os + +OLD_BLOCK = \ +""" + """ + +NEW_BLOCK = \ +""" + """ + +def process_file(file_path: str) -> None: + try: + with open(file_path ,"r" ,encoding="utf-8") as f: + content = f.read() + except Exception as e: + print(f"Error reading {file_path}: {e}" ,file=sys.stderr) + return + + # Normalize line endings to ensure a match across environments + normalized_content = content.replace("\r\n" ,"\n") + normalized_old = OLD_BLOCK.replace("\r\n" ,"\n") + + if( normalized_old in normalized_content ): + new_content = normalized_content.replace(normalized_old ,NEW_BLOCK) + with open(file_path ,"w" ,encoding="utf-8") as f: + f.write(new_content) + print(f"Updated: {file_path}") + else: + print(f"Target lines not found in: {file_path}") + +def CLI() -> None: + args = sys.argv[1:] + + if( not args or "-h" in args or "--help" in args ): + print("Usage: html_update_to_3-1 ") + sys.exit(0) + + for TM_f in args: + process_file(TM_f) + +if __name__ == "__main__": + CLI() diff --git a/administrator/tool/new-project b/administrator/tool/new-project new file mode 100755 index 0000000..c949ddb --- /dev/null +++ b/administrator/tool/new-project @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- + +""" +new-project + +This administrative tool instantiates a fresh project workspace modeled after +the Harmony skeleton architecture. + +The script executes the following sequential lifecycle: + 1. Validates the execution environment (requires an active role session). + 2. Checks the git status of the skeleton. The new project is made from the + last commit, so uncommitted changes and untracked files would be left + out. When there are any, lists them and asks whether to continue. + 3. Safely extracts a clean copy of all Git-tracked files using an archive stream. + 4. Renames the boilerplate 0pus_Harmony anchor file to reflect the new project. + 5. Injects an initial version tracking entry into the shared configuration space. + 6. Initializes a fresh Git repository bound to the core-developer_branch. + 7. Stages and commits the pristine skeleton infrastructure as a baseline. + 8. Maps any upstream remote network targets passed via CLI parameters. +""" + +import sys ,os ,subprocess +from pathlib import Path + +def get_Z_date() -> str: + """Retrieves the current date utilizing the system Z timestamp tool.""" + try: + res = subprocess.run(["Z" ,"format-%year-%month-%day"] ,capture_output=True ,text=True ,check=True) + return res.stdout.strip() + except Exception: + return "1970-01-01" + +def uncommitted(repo_path: Path) -> list[str] | None: + """Returns the lines of 'git status --porcelain' for the skeleton: one per + file with uncommitted changes or untracked. None when git cannot report.""" + try: + res = subprocess.run( + ["git" ,"status" ,"--porcelain" ,"--untracked-files=normal"] + ,cwd=str(repo_path) ,capture_output=True ,text=True ,check=True + ) + except Exception: + return None + return [line for line in res.stdout.splitlines() if line.strip()] + +def confirm(question: str) -> bool: + """Asks a yes or no question at the terminal. Without a terminal to ask + at, the answer is no.""" + if not sys.stdin.isatty(): + return False + try: + answer = input(question + " [y/N] ") + except EOFError: + return False + return answer.strip().lower() in ("y" ,"yes") + +def work(repo_path: Path ,project_name: str ,remotes: list[str]) -> int: + """Orchestrates the assembly of the new skeleton instance.""" + + # Resolve destination relative to the parent space of the current repository + dest_path = Path(project_name) + if not dest_path.is_absolute(): + dest_path = (repo_path.parent / project_name).resolve() + + if dest_path.exists(): + print(f"Error: Destination {dest_path} already exists." ,file=sys.stderr) + return 1 + + print(f"Making new project at: {dest_path}") + dest_path.mkdir(parents=True) + + # Export tracked files without duplicating untracked scratchpads or git histories + archive_cmd = "git archive HEAD | tar -x -C " + str(dest_path) + res = subprocess.run(archive_cmd ,shell=True ,cwd=str(repo_path)) + if res.returncode != 0: + print("Error extracting archive." ,file=sys.stderr) + return 1 + + # Personalize the project baseline anchor file + opus_src = dest_path / "0pus_Harmony" + opus_dst = dest_path / f"0pus_{dest_path.name}" + if opus_src.exists(): + opus_src.rename(opus_dst) + + # Record the lineage in the shared version history + version_file = dest_path / "shared" / "tool" / "version" + date_str = get_Z_date() + version_line = f"echo \"{dest_path.name} v0.1 {date_str}\"\n" + + if version_file.exists(): + with open(version_file ,"a" ,encoding="utf-8") as vf: + vf.write(version_line) + else: + with open(version_file ,"w" ,encoding="utf-8") as vf: + vf.write(version_line) + + # Initialize the pristine Git environment tracking system + print("Initializing git repository...") + subprocess.run(["git" ,"init" ,"-b" ,"core-developer_branch"] ,cwd=str(dest_path) ,check=True) + subprocess.run(["git" ,"add" ,"."] ,cwd=str(dest_path) ,check=True) + subprocess.run(["git" ,"commit" ,"-m" ,f"Initial commit of {dest_path.name} from Harmony skeleton"] ,cwd=str(dest_path) ,check=True) + + # Configure tracking addresses for upstreams + for idx ,remote_url in enumerate(remotes): + remote_name = f"remote_{idx}" + if "github.com" in remote_url: + remote_name = "github" + elif "RT" in remote_url: + remote_name = "RT" + elif idx == 0: + remote_name = "origin" + + print(f"Adding remote '{remote_name}': {remote_url}") + subprocess.run(["git" ,"remote" ,"add" ,remote_name ,remote_url] ,cwd=str(dest_path) ,check=True) + + print(f"Project '{dest_path.name}' successfully made.") + return 0 + +def CLI(argv=None) -> int: + """Parses user input tokens from the shell execution frame.""" + if argv is None: + argv = sys.argv[1:] + + if not argv or "-h" in argv or "--help" in argv: + print("Usage: new-project [-remote ]...") + return 0 + + project_name = None + remotes = [] + + idx = 0 + while idx < len(argv): + arg = argv[idx] + if arg == "-remote": + if idx + 1 < len(argv): + remotes.append(argv[idx+1]) + idx += 1 + else: + print("Error: -remote requires a URL." ,file=sys.stderr) + return 1 + elif project_name is None: + project_name = arg + else: + print(f"Error: unexpected argument '{arg}'" ,file=sys.stderr) + return 1 + idx += 1 + + if not project_name: + print("Error: project name is required." ,file=sys.stderr) + return 1 + + repo_home = os.environ.get("REPO_HOME") + if not repo_home: + print("Error: REPO_HOME is not set. A person must run this from within a sourced role environment." ,file=sys.stderr) + return 1 + repo_path = Path(repo_home).resolve() + + status = uncommitted(repo_path) + if status is None: + print(f"Error: could not read the git status of {repo_path}." ,file=sys.stderr) + return 1 + if status: + shown = status[:20] + print(f"Warning: {repo_path} has {len(status)} uncommitted or untracked file(s):") + for line in shown: + print(" " + line) + if len(status) > len(shown): + print(f" ... and {len(status) - len(shown)} more (see 'git status')") + print("The new project is made from the last commit, so these will not be in it.") + if not confirm("Continue anyway?"): + print("Stopped. Nothing was made. Commit or stash the changes, then run new-project again.") + return 1 + + return work(repo_path ,project_name ,remotes) + +if __name__ == "__main__": + sys.exit(CLI()) diff --git a/administrator/tool/setup b/administrator/tool/setup new file mode 100644 index 0000000..0b993ad --- /dev/null +++ b/administrator/tool/setup @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +script_afp=$(realpath "${BASH_SOURCE[0]}") + diff --git a/administrator/tool/skeleton_diff b/administrator/tool/skeleton_diff new file mode 100755 index 0000000..973c4ae --- /dev/null +++ b/administrator/tool/skeleton_diff @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- + +import sys ,os ,filecmp ,stat ,subprocess +from pathlib import Path + +def get_project_name(repo_path: Path) -> str: + return repo_path.resolve().name + +def get_git_branch(repo_path: Path) -> str: + try: + result = subprocess.run( + ["git" ,"-C" ,str(repo_path) ,"branch" ,"--show-current"], + capture_output=True ,text=True ,check=True + ) + branch = result.stdout.strip() + return branch if branch else "unknown_branch" + except (subprocess.CalledProcessError ,FileNotFoundError): + return "unknown_branch" + +def format_perms(mode: int) -> str: + # Extract the 9-character permission string (e.g., rwxr-xr-x), dropping the leading file type char + return stat.filemode(mode)[1:] + +def CLI(argv=None) -> int: + if argv is None: + argv = sys.argv[1:] + + if len(argv) != 2: + print("Usage: skeleton_diff ") + return 1 + + harmony_dir = Path(argv[0]).resolve() + project_dir = Path(argv[1]).resolve() + + if not harmony_dir.is_dir() or not project_dir.is_dir(): + print("Error: Both arguments must be valid directories." ,file=sys.stderr) + return 1 + + project_name = get_project_name(project_dir) + harmony_branch = get_git_branch(harmony_dir) + project_branch = get_git_branch(project_dir) + + # Data structures for the output sections + opus_warnings = [] + missing_files = [] + + # Time-based modification buckets + mod_harmony_newer = [] + mod_same_time = [] + mod_project_newer = [] + + perms_only_changes = [] + added_files = [] + + # --- Section 1: Opus File Structural Audit --- + project_opus = f"0pus_{project_name}" + harmony_opus = "0pus_Harmony" + found_opus = False + + for TM_item in os.listdir(project_dir): + if TM_item.startswith("0pus_") and os.path.isfile(project_dir / TM_item): + if TM_item == project_opus: + found_opus = True + else: + opus_warnings.append(f"Incorrectly named Opus file found: '{TM_item}'. Expected '{project_opus}'.") + + if not found_opus: + opus_warnings.append(f"Expected Opus file '{project_opus}' is MISSING from project root.") + + # Semantic boundaries: isolate the skeleton by ignoring user zones + IGNORED_DIRS = {".git" ,"scratchpad" ,"consumer" ,"authored" ,"__pycache__"} + IGNORED_EXTS = {".pyc" ,".pyo" ,".swp" ,".bak"} + + def is_ignored(path_obj: Path) -> bool: + if path_obj.name in IGNORED_DIRS: return True + if path_obj.suffix in IGNORED_EXTS: return True + if path_obj.name.endswith("~"): return True + return False + + harmony_files = set() + for TM_root ,TM_dirs ,TM_files in os.walk(harmony_dir): + TM_dirs[:] = [TM_d for TM_d in TM_dirs if not is_ignored(Path(TM_d))] + for TM_f in TM_files: + f_path = Path(TM_root) / TM_f + if not is_ignored(f_path): + rel_path = f_path.relative_to(harmony_dir) + harmony_files.add(str(rel_path)) + + project_files = set() + for TM_root ,TM_dirs ,TM_files in os.walk(project_dir): + TM_dirs[:] = [TM_d for TM_d in TM_dirs if not is_ignored(Path(TM_d))] + for TM_f in TM_files: + f_path = Path(TM_root) / TM_f + if not is_ignored(f_path): + rel_path = f_path.relative_to(project_dir) + project_files.add(str(rel_path)) + + # Align the Opus file names for comparison + if project_opus in project_files: + project_files.remove(project_opus) + project_files.add(harmony_opus) + else: + for TM_item in list(project_files): + if TM_item.startswith("0pus_") and "/" not in TM_item: + project_files.remove(TM_item) + + all_files = sorted(list(harmony_files | project_files)) + + for TM_rel in all_files: + if TM_rel not in project_files: + if TM_rel == harmony_opus and not found_opus: + continue + missing_files.append(TM_rel) + elif TM_rel not in harmony_files: + added_files.append(TM_rel) + else: + h_abs = harmony_dir / TM_rel + p_abs = project_dir / TM_rel + + if TM_rel == harmony_opus: + p_abs = project_dir / project_opus + + if p_abs.exists(): + h_stat = h_abs.stat() + p_stat = p_abs.stat() + + h_perms = format_perms(h_stat.st_mode) + p_perms = format_perms(p_stat.st_mode) + perms_differ = (h_perms != p_perms) + + display_str = f"({h_perms}, {p_perms}) {TM_rel}" if perms_differ else TM_rel + content_differs = not filecmp.cmp(h_abs ,p_abs ,shallow=False) + + if content_differs: + h_time = h_stat.st_mtime + p_time = p_stat.st_mtime + + if h_time > p_time: + mod_harmony_newer.append(display_str) + elif h_time < p_time: + mod_project_newer.append(display_str) + else: + mod_same_time.append(display_str) + elif perms_differ: + perms_only_changes.append(display_str) + + # --- Output Generation --- + print("Skeleton Audit:") + print(f" Harmony {harmony_branch}") + print(" vs") + print(f" {project_name} {project_branch}") + print("=" * 60) + + print("\n1. 0pus File Audit") + print("-" * 20) + if not opus_warnings: + print(" [OK] Opus file is correctly named and present.") + else: + for TM_w in opus_warnings: + print(f" [WARNING] {TM_w}") + + print("\n2. Missing from Project (In Harmony, missing locally)") + print("-" * 20) + if not missing_files: + print(" [OK] No Harmony skeleton files are missing.") + else: + for TM_f in missing_files: + print(f" - {TM_f}") + + print("\n3. Modified Skeleton Files (Diverged from Harmony)") + print("-" * 20) + if not (mod_harmony_newer or mod_same_time or mod_project_newer): + print(" [OK] No shared skeleton files have been modified.") + else: + if mod_harmony_newer: + print(" [Harmony newer]") + for TM_f in mod_harmony_newer: + print(f" ~ {TM_f}") + if mod_same_time: + print(" [Same modification time]") + for TM_f in mod_same_time: + print(f" ~ {TM_f}") + if mod_project_newer: + print(" [Project newer]") + for TM_f in mod_project_newer: + print(f" ~ {TM_f}") + + print("\n4. Permissions Only Changes") + print("-" * 20) + if not perms_only_changes: + print(" [OK] No permission-only changes detected.") + else: + for TM_f in perms_only_changes: + print(f" * {TM_f}") + + print("\n5. Project-Specific Files (Not in Harmony)") + print("-" * 20) + if not added_files: + print(" [INFO] No additional files outside the standard skeleton.") + else: + for TM_f in added_files: + print(f" + {TM_f}") + + print("\n" + "=" * 60) + drift_count = len(opus_warnings) + len(missing_files) + len(mod_harmony_newer) + len(mod_same_time) + len(mod_project_newer) + len(perms_only_changes) + if drift_count == 0: + print("Result: Skeleton is perfectly synchronized with Harmony.") + else: + print(f"Total Structural Drift (Sections 1-4): {drift_count} issues.") + + return 0 + +if __name__ == "__main__": + sys.exit(CLI()) + diff --git a/administrator/tool/skeleton_update b/administrator/tool/skeleton_update new file mode 100755 index 0000000..716e342 --- /dev/null +++ b/administrator/tool/skeleton_update @@ -0,0 +1,621 @@ +#!/usr/bin/env python3 +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- +""" +skeleton_update + +Brings a project onto the current Harmony skeleton. Skeleton files are +copied in from a Harmony checkout, and .gitignore and shared/tool/version are +merged with the project's own lines. The project's files are then sorted: + + a file in a place Harmony has, such as developer/authored/ or a directory + of the skeleton, stays where it is; + a file the plan moves is renamed to its new place; + a file under a plan's duplicate tree is deleted when the project holds a + file with the same content elsewhere, and is otherwise deprecated; + every other file is deprecated: moved, with its path kept, under + developer/authored/deprecated/, so nothing is lost and the original place + of each file can be read from its new path. + +The plan is a Python file giving what is particular to one project. A +project already laid out after Harmony needs no plan, and the tool then +updates its skeleton files only. The plan's names are: + + MOVE {old path: new path}; a path ending in / moves a directory tree + DEPRECATE paths that are deprecated even though Harmony has their place + DUPLICATE trees whose files are copies of files kept elsewhere + KEEP paths kept as they are, including skeleton files the project + has its own version of + EXECUTABLE paths, after any move, given execute permission where read + permission is given + +The files sorted are those git tracks or would track: the work tree less what +the project's .gitignore ignores. Ignored files, such as a local Python +environment, are left where they are and listed at the end of the log. +Compiled Python caches are deleted. Directories left empty are removed. + +Commit the project first; run refuses a work tree with uncommitted changes. +Afterward, 'git add -A' records the moves, and git recognizes each moved file +as a rename by its content, so 'git log --follow' follows its history. Run +again after committing, and the log reports nothing left to do. + +Usage: skeleton_update run [] + skeleton_update preview run [] + help | -h | --help print this message (also printed when given no arguments) + run bring onto the skeleton of + preview run print the log of what run would do, change nothing +""" + +import sys ,os ,shutil ,filecmp ,subprocess ,runpy ,hashlib +from pathlib import Path + + +# -------------------------------------------------------------------------------- +# what the tool knows about Harmony + +DEPRECATED_DIR = "developer/authored/deprecated" + +# Skeleton files a project does not take: it has its own. +SKELETON_SKIP_FILE = {"README.md" ,"LICENSE" ,"0pus_Harmony"} +SKELETON_SKIP_TREE = ("developer/authored/ExampleGreet.make/",) + +# Skeleton files merged with the project's version rather than replaced. +MERGED_FILE = (".gitignore" ,"shared/tool/version") + +# Files at the top of a project that belong to the project. +PROJECT_ROOT_FILE = {"LICENSE" ,"README.md" ,".gitattributes" ,".gitmodules"} + +# Trees in which every file belongs to the project. +PROJECT_TREE = ( + "administrator/authored/", + "administrator/document/", + "developer/authored/", + "developer/document/", + "developer/experiment/", + "document/", + "shared/authored/", + "tester/authored/", + "tester/document/", +) + +# Placeholder files of the old skeleton; Harmony uses .gitkeep. +PLACEHOLDER_NAME = {".githolder"} + +CACHE_DIR_NAME = {"__pycache__"} + +PLAN_NAME = ("MOVE" ,"DEPRECATE" ,"DUPLICATE" ,"KEEP" ,"EXECUTABLE") + + +# -------------------------------------------------------------------------------- +# the log + +TAG_WIDTH = 18 + +# the tag a preview prints in place of each past-tense tag +PREVIEW_TAG = { + "added": "add" ,"deleted": "delete" ,"deprecated": "deprecate" ,"kept": "keep", + "left alone": "leave alone" ,"made executable": "make executable" ,"merged": "merge", + "moved": "move" ,"removed": "remove", + "replaced": "replace", +} + +class Log: + def __init__(self ,preview): + self.preview = preview + self.count = {} + self.error_count = 0 + + def line(self ,tag ,rel ,reason=None): + if self.preview: tag = "would " + PREVIEW_TAG.get(tag ,tag) + text = f" {tag:<{TAG_WIDTH}}{rel}" + if reason: text += f" ({reason})" + print(text) + + def tally(self ,key ,n=1): + self.count[key] = self.count.get(key ,0) + n + + def error(self ,rel ,message): + self.error_count += 1 + print(f" {'error':<{TAG_WIDTH}}{rel} ({message})") + + def summary(self): + print("summary:") + order = [ + ("moved" ,"files moved to their place by the plan"), + ("deprecated" ,"files moved under " + DEPRECATED_DIR), + ("deleted duplicate" ,"duplicate files deleted"), + ("deleted placeholder" ,".githolder files deleted"), + ("deleted cache" ,"__pycache__ directories deleted"), + ("skeleton added" ,"skeleton files added"), + ("skeleton replaced" ,"skeleton files replaced"), + ("skeleton current" ,"skeleton files already current"), + ("skeleton kept" ,"skeleton files kept in the project's own version"), + ("merged" ,"files merged with the skeleton"), + ("merged current" ,"merged files already current"), + ("kept" ,"project files kept in place"), + ("made executable" ,"files made executable"), + ("left alone" ,"ignored files left in place, listed above"), + ("directory removed" ,"empty directories removed"), + ] + for key ,label in order: + print(f" {self.count.get(key ,0):>4} {label}") + if self.error_count: + print(f" {self.error_count:>4} errors") + if self.preview: + print("preview only: nothing was changed.") + + +# -------------------------------------------------------------------------------- +# reading git + +def git(directory ,*argument): + return subprocess.run( + ["git" ,"-C" ,str(directory) ,*argument], + capture_output=True ,text=True ,check=True + ).stdout + +def git_dirty(directory): + return git(directory ,"status" ,"--porcelain").strip() + +def project_file_list(project): + """ + The files git tracks or would track, that exist: tracked files, and + untracked files the project's .gitignore does not ignore. + """ + listing = git(project ,"ls-files" ,"-z" ,"--cached" ,"--others" ,"--exclude-standard").split("\0") + result = set() + for rel in listing: + if rel and ((project / rel).exists() or (project / rel).is_symlink()): + result.add(rel) + return sorted(result) + +def ignored_file_list(project): + listing = git(project ,"ls-files" ,"-z" ,"--others" ,"--ignored" ,"--exclude-standard").split("\0") + return sorted(rel for rel in listing if rel) + +def skeleton_file_list(harmony): + """ + The skeleton is the set of files in Harmony's last commit, as new-project + takes it. + """ + result = [] + for rel in git(harmony ,"ls-tree" ,"-r" ,"--name-only" ,"-z" ,"HEAD").split("\0"): + if not rel or rel in SKELETON_SKIP_FILE or in_tree(rel ,SKELETON_SKIP_TREE): + continue + result.append(rel) + return sorted(result) + + +# -------------------------------------------------------------------------------- +# paths + +def in_tree(rel ,tree_tuple): + return any(rel.startswith(t) for t in tree_tuple) + +def matches(rel ,path_tuple): + """A path names a file exactly, or, ending in /, the tree under it.""" + return any(rel == p or (p.endswith("/") and rel.startswith(p)) for p in path_tuple) + +def moved_path(rel ,move): + for old ,new in move.items(): + if rel == old: + return new + if old.endswith("/") and rel.startswith(old): + return new + rel[len(old):] + return None + +def directory_set(rel_list): + result = set() + for rel in rel_list: + parent = Path(rel).parent + while parent != Path("."): + result.add(parent.as_posix()) + parent = parent.parent + return result + +def content_hash(path): + if path.is_symlink(): + return "link:" + os.readlink(path) + h = hashlib.sha256() + with open(path ,"rb") as f: + for block in iter(lambda: f.read(1 << 16) ,b""): + h.update(block) + return h.hexdigest() + + +# -------------------------------------------------------------------------------- +# file operations; each tests preview immediately before acting + +def same_leaf(a ,b): + if a.is_symlink() or b.is_symlink(): + return a.is_symlink() and b.is_symlink() and os.readlink(a) == os.readlink(b) + return a.is_file() and b.is_file() and filecmp.cmp(a ,b ,shallow=False) + +def copy_leaf(src ,dst ,preview): + if preview: return + dst.parent.mkdir(parents=True ,exist_ok=True) + if dst.is_symlink() or dst.exists(): + dst.unlink() + if src.is_symlink(): + os.symlink(os.readlink(src) ,dst) + else: + shutil.copy2(src ,dst) + +def move_leaf(src ,dst ,preview): + if preview: return + dst.parent.mkdir(parents=True ,exist_ok=True) + os.rename(src ,dst) + +def delete_leaf(path ,preview): + if preview: return + path.unlink() + +def delete_tree(path ,preview): + if preview: return + shutil.rmtree(path) + +def write_text(path ,text ,preview): + if preview: return + path.parent.mkdir(parents=True ,exist_ok=True) + path.write_text(text ,encoding="utf-8") + + +# -------------------------------------------------------------------------------- +# merging + +def merged_gitignore(project_text ,harmony_text ,project_name): + """ + Harmony's lines, then a section holding the project's patterns that + Harmony lacks. The project's comments and blank lines are not carried. + """ + section = "# " + project_name + harmony_line = set(harmony_text.splitlines()) + extra = [] + for line in project_text.splitlines(): + pattern = line.strip() + if not pattern or pattern.startswith("#"): + continue + if pattern not in harmony_line and pattern not in extra: + extra.append(pattern) + text = harmony_text.rstrip("\n") + "\n" + if extra: + text += "\n" + section + "\n" + "\n".join(extra) + "\n" + return text + +def merged_version(project_text ,harmony_text ,project_name): + """Harmony's file, then the project's lines that do not name Harmony.""" + project_line = [l for l in project_text.splitlines() if l.strip() and "Harmony" not in l and l not in harmony_text.splitlines()] + text = harmony_text if harmony_text.endswith("\n") else harmony_text + "\n" + return text + "".join(l + "\n" for l in project_line) + +MERGE_FUNCTION = { + ".gitignore": merged_gitignore, + "shared/tool/version": merged_version, +} + + +# -------------------------------------------------------------------------------- +# the work + +def step_cache(project ,log ,preview): + for dirpath ,dirnames ,filenames in os.walk(project): + dirnames.sort() + if Path(dirpath) == project and ".git" in dirnames: + dirnames.remove(".git") + for name in list(dirnames): + if name in CACHE_DIR_NAME: + rel = Path(dirpath ,name).relative_to(project).as_posix() + delete_tree(project / rel ,preview) + log.line("deleted" ,rel + "/" ,"compiled Python cache") + log.tally("deleted cache") + dirnames.remove(name) + +def decide(rel ,plan ,skeleton_set ,skeleton_dir ,ignored_list): + action = decide_place(rel ,plan ,skeleton_set ,skeleton_dir) + if Path(rel).name == ".gitignore" and action[0] in ("move" ,"duplicate" ,"deprecate"): + prefix = Path(rel).parent.as_posix() + "/" + n = sum(1 for i in ignored_list if i.startswith(prefix)) + if n: + return ("keep" ,rel ,f"it hides {n} ignored file{'s' if n != 1 else ''}, left in place") + return action + +def decide_place(rel ,plan ,skeleton_set ,skeleton_dir): + """ + What becomes of one project file: (action ,new path ,reason). The action is + one of placeholder, keep, keep skeleton, move, duplicate, deprecate, or + skeleton, which the skeleton steps handle. + """ + name = Path(rel).name + if name in PLACEHOLDER_NAME: + return ("placeholder" ,None ,None) + if matches(rel ,plan["KEEP"]): + return ("keep skeleton" if rel in skeleton_set else "keep" ,rel ,None) + new = moved_path(rel ,plan["MOVE"]) + if new is not None: + return ("move" ,new ,None) + if matches(rel ,plan["DUPLICATE"]): + return ("duplicate" ,f"{DEPRECATED_DIR}/{rel}" ,None) + if matches(rel ,plan["DEPRECATE"]): + return ("deprecate" ,f"{DEPRECATED_DIR}/{rel}" ,"by the plan") + if rel in skeleton_set or rel in MERGED_FILE: + return ("skeleton" ,rel ,None) + parent = Path(rel).parent.as_posix() + if (parent == "." and (name in PROJECT_ROOT_FILE or name.startswith("0pus_"))) \ + or in_tree(rel ,PROJECT_TREE) or parent in skeleton_dir: + return ("keep" ,rel ,None) + return ("deprecate" ,f"{DEPRECATED_DIR}/{rel}" ,None) + +def step_sort(project ,plan ,skeleton_set ,file_list ,ignored_list ,log ,preview): + """ + Place every project file that is not a skeleton file. Every decision is + made before anything moves, so a duplicate can name where its twin ends up. + """ + skeleton_dir = directory_set(skeleton_set) + decision = {rel: decide(rel ,plan ,skeleton_set ,skeleton_dir ,ignored_list) for rel in file_list} + + # where the content of each file that stays in the project ends up + kept_at = {} + for rel ,(action ,new ,reason) in decision.items(): + if action in ("keep" ,"keep skeleton" ,"move" ,"deprecate" ,"skeleton"): + kept_at.setdefault(content_hash(project / rel) ,new) + + for rel in file_list: + action ,new ,reason = decision[rel] + if action == "placeholder": + delete_leaf(project / rel ,preview) + log.line("deleted" ,rel ,"placeholder; Harmony uses .gitkeep") + log.tally("deleted placeholder") + elif action == "keep skeleton": + log.line("kept" ,rel ,"the project's own version, by the plan") + log.tally("skeleton kept") + elif action == "keep": + if reason: log.line("kept" ,rel ,reason) + log.tally("kept") + elif action == "move": + place(project ,rel ,new ,"moved" ,f"to {new}" ,log ,preview) + elif action == "duplicate": + twin = kept_at.get(content_hash(project / rel)) + if twin: + delete_leaf(project / rel ,preview) + log.line("deleted" ,rel ,f"the same as {twin}") + log.tally("deleted duplicate") + else: + place(project ,rel ,new ,"deprecated" ,"no copy of it is kept" ,log ,preview) + elif action == "deprecate": + place(project ,rel ,new ,"deprecated" ,reason ,log ,preview) + +def place(project ,rel ,new ,tag ,reason ,log ,preview): + dst = project / new + if dst.exists() or dst.is_symlink(): + log.error(rel ,f"{new} already exists; nothing was moved") + return + move_leaf(project / rel ,dst ,preview) + log.line(tag ,rel ,reason) + log.tally(tag) + +def step_skeleton(project ,harmony ,skeleton_list ,plan ,log ,preview): + for rel in skeleton_list: + if rel in MERGED_FILE or matches(rel ,plan["KEEP"]): + continue + src = harmony / rel + dst = project / rel + if dst.exists() or dst.is_symlink(): + if same_leaf(src ,dst): + log.tally("skeleton current") + continue + copy_leaf(src ,dst ,preview) + log.line("replaced" ,rel ,"from the skeleton") + log.tally("skeleton replaced") + else: + copy_leaf(src ,dst ,preview) + log.line("added" ,rel ,"from the skeleton") + log.tally("skeleton added") + +def step_merge(project ,harmony ,project_name ,plan ,log ,preview): + for rel in MERGED_FILE: + if matches(rel ,plan["KEEP"]): + continue + project_path = project / rel + harmony_text = (harmony / rel).read_text(encoding="utf-8") + project_text = project_path.read_text(encoding="utf-8") if project_path.is_file() else "" + text = MERGE_FUNCTION[rel](project_text ,harmony_text ,project_name) + if text == project_text: + log.tally("merged current") + continue + if not project_path.exists(): + copy_leaf(harmony / rel ,project_path ,preview) + write_text(project_path ,text ,preview) + log.line("merged" ,rel ,"the skeleton's lines with the project's own") + log.tally("merged") + +def step_opus(project ,harmony ,project_name ,log ,preview): + rel = "0pus_" + project_name + if (project / rel).exists(): + return + write_text(project / rel ,"" ,preview) + if not preview: shutil.copymode(harmony / "0pus_Harmony" ,project / rel) + log.line("added" ,rel ,"the file that marks the project root") + log.tally("skeleton added") + +def step_executable(project ,plan ,file_list ,log ,preview): + """ + Give each EXECUTABLE path execute permission wherever it has read + permission. In a preview the file is found at its place before the move. + """ + before = {moved_path(rel ,plan["MOVE"]) or rel: rel for rel in file_list} + for rel in plan["EXECUTABLE"]: + path = project / rel + if not path.is_file(): + path = project / before.get(rel ,rel) + if not path.is_file(): + log.error(rel ,"EXECUTABLE names a file that does not exist") + continue + mode = path.stat().st_mode + wanted = mode | ((mode & 0o444) >> 2) + if wanted == mode: + continue + if not preview: os.chmod(path ,wanted) + log.line("made executable" ,rel) + log.tally("made executable") + +def step_ignored(project ,file_list ,ignored_list ,log): + """ + List ignored files outside scratchpad and consumer, by the highest + directory holding nothing git tracks. + """ + tracked_dir = directory_set(file_list) + group = {} + for rel in ignored_list: + part = rel.split("/") + if "scratchpad" in part[:-1] or part[0] == "consumer" or any(p in CACHE_DIR_NAME for p in part): + continue + top = rel + parent = Path(rel).parent + while parent != Path(".") and parent.as_posix() not in tracked_dir: + top = parent.as_posix() + "/" + parent = parent.parent + group[top] = group.get(top ,0) + 1 + for top ,n in sorted(group.items()): + log.line("left alone" ,top ,f"ignored by git; {n} file{'s' if n != 1 else ''}") + log.tally("left alone" ,n) + +def step_newly_ignored(project ,ignored_list ,log): + """ + A file the run placed where the merged .gitignore ignores it would not be + recorded by 'git add -A'. Such files are reported as errors. + """ + before = set(ignored_list) + for rel in ignored_file_list(project): + part = rel.split("/") + if rel in before or "scratchpad" in part[:-1] or part[0] == "consumer" or any(p in CACHE_DIR_NAME for p in part): + continue + log.error(rel ,"git ignores it now, so 'git add -A' will not record it; add it with 'git add -f'") + +def step_empty_dir(project ,log ,preview): + for dirpath ,dirnames ,filenames in os.walk(project ,topdown=False): + path = Path(dirpath) + if path == project or ".git" in path.relative_to(project).parts: + continue + try: + remaining = list(path.iterdir()) + except FileNotFoundError: + continue + if not remaining: + rel = path.relative_to(project).as_posix() + if not preview: path.rmdir() + log.line("removed" ,rel + "/" ,"empty directory") + log.tally("directory removed") + +def work(harmony ,project ,plan ,preview): + print(f"skeleton_update {'preview ' if preview else ''}run in {project}") + print(f" skeleton from {harmony}") + log = Log(preview) + project_name = project.name + + skeleton_list = skeleton_file_list(harmony) + skeleton_set = set(skeleton_list) + file_list = [r for r in project_file_list(project) if not any(p in CACHE_DIR_NAME for p in r.split("/"))] + ignored_list = [r for r in ignored_file_list(project) if not any(p in CACHE_DIR_NAME for p in r.split("/"))] + + step_cache(project ,log ,preview) + step_sort(project ,plan ,skeleton_set ,file_list ,ignored_list ,log ,preview) + step_skeleton(project ,harmony ,skeleton_list ,plan ,log ,preview) + step_merge(project ,harmony ,project_name ,plan ,log ,preview) + step_opus(project ,harmony ,project_name ,log ,preview) + step_executable(project ,plan ,file_list ,log ,preview) + step_ignored(project ,file_list ,ignored_list ,log) + if preview: + print(" (directories left empty are found and removed by run, not listed in a preview)") + else: + step_empty_dir(project ,log ,preview) + step_newly_ignored(project ,ignored_list ,log) + + log.summary() + return 1 if log.error_count else 0 + + +# -------------------------------------------------------------------------------- +# command line + +def read_plan(path): + plan = {"MOVE": {} ,"DEPRECATE": () ,"DUPLICATE": () ,"KEEP": () ,"EXECUTABLE": ()} + if path is None: + return plan + given = runpy.run_path(str(path)) + for name in PLAN_NAME: + if name in given: + plan[name] = given[name] + if not isinstance(plan["MOVE"] ,dict): + raise ValueError("MOVE must be a dictionary of old path to new path") + for name in ("DEPRECATE" ,"DUPLICATE" ,"KEEP" ,"EXECUTABLE"): + plan[name] = tuple(plan[name]) + for old ,new in plan["MOVE"].items(): + if old.endswith("/") != new.endswith("/"): + raise ValueError(f"MOVE '{old}' to '{new}': a directory moves to a directory, both ending in /") + return plan + +def print_help(): + print(__doc__.strip()) + return 0 + +def CLI(argv=None): + if argv is None: argv = sys.argv[1:] + if not argv or argv[0] in ("help" ,"-h" ,"--help"): return print_help() + + preview = argv[0] == "preview" + if preview: argv = argv[1:] + if not argv: + print("Error: 'preview' needs a command after it." ,file=sys.stderr) + return 1 + + command ,rest = argv[0] ,argv[1:] + if command != "run": + print(f"Error: unknown command '{command}'." ,file=sys.stderr) + print_help() + return 1 + if len(rest) not in (2 ,3): + print("Error: run is given []." ,file=sys.stderr) + return 1 + + harmony = Path(rest[0]).resolve() + project = Path(rest[1]).resolve() + plan_path = Path(rest[2]).resolve() if len(rest) == 3 else None + + if not (harmony / "0pus_Harmony").is_file() or not (harmony / ".git").exists(): + print(f"Error: {harmony} is not a Harmony checkout: it lacks 0pus_Harmony or .git." ,file=sys.stderr) + return 1 + if not (project / ".git").exists(): + print(f"Error: {project} is not a git work tree; the tool sorts the files git tracks." ,file=sys.stderr) + return 1 + if harmony == project: + print("Error: the project is the Harmony checkout itself." ,file=sys.stderr) + return 1 + if plan_path and not plan_path.is_file(): + print(f"Error: plan file {plan_path} does not exist." ,file=sys.stderr) + return 1 + try: + plan = read_plan(plan_path) + except Exception as error: + print(f"Error: plan file {plan_path}: {error}" ,file=sys.stderr) + return 1 + + dirty = git_dirty(project) + if dirty and not preview: + print("Error: the project work tree has uncommitted changes. Commit them first:" ,file=sys.stderr) + print(dirty ,file=sys.stderr) + return 1 + dirty = git_dirty(harmony) + if dirty: + print("Error: the Harmony checkout has uncommitted changes; the skeleton is taken" ,file=sys.stderr) + print("from its last commit, and its files would not match it:" ,file=sys.stderr) + print(dirty ,file=sys.stderr) + return 1 + + return work(harmony ,project ,plan ,preview) + + +if __name__ == "__main__": + try: + sys.exit(CLI()) + except BrokenPipeError: + os.dup2(os.open(os.devnull ,os.O_WRONLY) ,sys.stdout.fileno()) + sys.exit(1) diff --git a/consumer/.gitignore b/consumer/.gitignore new file mode 100644 index 0000000..76bedae --- /dev/null +++ b/consumer/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore + diff --git a/developer/source/scratchpad/.githolder b/developer/authored/.gitkeep similarity index 100% rename from developer/source/scratchpad/.githolder rename to developer/authored/.gitkeep diff --git a/developer/source/Man_In_Grey b/developer/authored/Man_In_Grey.copy/Man_In_Grey old mode 100644 new mode 100755 similarity index 100% rename from developer/source/Man_In_Grey rename to developer/authored/Man_In_Grey.copy/Man_In_Grey diff --git a/developer/source/Man_In_Grey.py b/developer/authored/Man_In_Grey.copy/Man_In_Grey.py old mode 100644 new mode 100755 similarity index 100% rename from developer/source/Man_In_Grey.py rename to developer/authored/Man_In_Grey.copy/Man_In_Grey.py diff --git a/developer/source/Planner.py b/developer/authored/Man_In_Grey.copy/Planner.py similarity index 100% rename from developer/source/Planner.py rename to developer/authored/Man_In_Grey.copy/Planner.py diff --git a/developer/source/diag_perm b/developer/authored/Man_In_Grey.copy/diag_perm old mode 100644 new mode 100755 similarity index 100% rename from developer/source/diag_perm rename to developer/authored/Man_In_Grey.copy/diag_perm diff --git a/developer/source/executor_inner.py b/developer/authored/Man_In_Grey.copy/executor_inner.py old mode 100644 new mode 100755 similarity index 100% rename from developer/source/executor_inner.py rename to developer/authored/Man_In_Grey.copy/executor_inner.py diff --git a/developer/source/executor_outer.py b/developer/authored/Man_In_Grey.copy/executor_outer.py similarity index 100% rename from developer/source/executor_outer.py rename to developer/authored/Man_In_Grey.copy/executor_outer.py diff --git a/developer/source/Man_In_Grey_apply.c b/developer/authored/Man_In_Grey_apply.make/man_in_grey_apply.CLI.c similarity index 100% rename from developer/source/Man_In_Grey_apply.c rename to developer/authored/Man_In_Grey_apply.make/man_in_grey_apply.CLI.c diff --git a/developer/machine/man_in_grey_apply b/developer/authored/deprecated/developer/machine/man_in_grey_apply similarity index 100% rename from developer/machine/man_in_grey_apply rename to developer/authored/deprecated/developer/machine/man_in_grey_apply diff --git a/developer/tool/clean_release b/developer/authored/deprecated/developer/tool/clean_release similarity index 100% rename from developer/tool/clean_release rename to developer/authored/deprecated/developer/tool/clean_release diff --git a/developer/tool/compile b/developer/authored/deprecated/developer/tool/compile similarity index 100% rename from developer/tool/compile rename to developer/authored/deprecated/developer/tool/compile diff --git a/developer/tool/env b/developer/authored/deprecated/developer/tool/env similarity index 100% rename from developer/tool/env rename to developer/authored/deprecated/developer/tool/env diff --git a/developer/tool/release b/developer/authored/deprecated/developer/tool/release similarity index 100% rename from developer/tool/release rename to developer/authored/deprecated/developer/tool/release diff --git a/document/Workflow.org b/developer/authored/deprecated/document/Workflow.org similarity index 100% rename from document/Workflow.org rename to developer/authored/deprecated/document/Workflow.org diff --git a/document/environment_guide.org b/developer/authored/deprecated/document/environment_guide.org similarity index 100% rename from document/environment_guide.org rename to developer/authored/deprecated/document/environment_guide.org diff --git a/document/general_info.org b/developer/authored/deprecated/document/general_info.org similarity index 100% rename from document/general_info.org rename to developer/authored/deprecated/document/general_info.org diff --git a/env_developer b/developer/authored/deprecated/env_developer similarity index 100% rename from env_developer rename to developer/authored/deprecated/env_developer diff --git a/env_tester b/developer/authored/deprecated/env_tester similarity index 100% rename from env_tester rename to developer/authored/deprecated/env_tester diff --git a/env_toolsmith b/developer/authored/deprecated/env_toolsmith similarity index 100% rename from env_toolsmith rename to developer/authored/deprecated/env_toolsmith diff --git a/tester/tool/env b/developer/authored/deprecated/tester/tool/env similarity index 100% rename from tester/tool/env rename to developer/authored/deprecated/tester/tool/env diff --git a/tool/bless b/developer/authored/deprecated/tool/bless similarity index 100% rename from tool/bless rename to developer/authored/deprecated/tool/bless diff --git a/tool/unbless b/developer/authored/deprecated/tool/unbless similarity index 100% rename from tool/unbless rename to developer/authored/deprecated/tool/unbless diff --git a/tool_shared/bespoke/env b/developer/authored/deprecated/tool_shared/bespoke/env similarity index 100% rename from tool_shared/bespoke/env rename to developer/authored/deprecated/tool_shared/bespoke/env diff --git a/tool_shared/bespoke/print_project_env.py b/developer/authored/deprecated/tool_shared/bespoke/print_project_env.py similarity index 100% rename from tool_shared/bespoke/print_project_env.py rename to developer/authored/deprecated/tool_shared/bespoke/print_project_env.py diff --git a/tool_shared/bespoke/version b/developer/authored/deprecated/tool_shared/bespoke/version similarity index 100% rename from tool_shared/bespoke/version rename to developer/authored/deprecated/tool_shared/bespoke/version diff --git a/tool_shared/bespoke/vl b/developer/authored/deprecated/tool_shared/bespoke/vl similarity index 100% rename from tool_shared/bespoke/vl rename to developer/authored/deprecated/tool_shared/bespoke/vl diff --git a/tool_shared/document/install_Python.org b/developer/authored/deprecated/tool_shared/document/install_Python.org similarity index 100% rename from tool_shared/document/install_Python.org rename to developer/authored/deprecated/tool_shared/document/install_Python.org diff --git a/developer/document/naming_file-and-directory.html b/developer/document/naming_file-and-directory.html new file mode 100644 index 0000000..c597855 --- /dev/null +++ b/developer/document/naming_file-and-directory.html @@ -0,0 +1,101 @@ + + + + + File and directory naming conventions + + + + + + + + + + + +

Program file system objects

+ +

+ Harmony files and directories are considered to be program file systems objects. +

+ +

Files

+

+ Files that hold a single class, have he same base name as that class. If a file holds multiple classes, but there is a class of primary importance and auxiliary classes, the file carries the name of the primary class. +

+ +

+ If a file represents a module or name space, follow the same naming conventions specified for classes and name spaces in the format_RT·code.html document. +

+ +

+ Other program files follow the same naming conventions specified in the format_RT·code.html document for an identifier of the + contained or related language. +

+ +

Hyphens and spaces

+ +

In a C language culture, file names use underscores between words. In a Lisp language culture use snake-kebab_case. Spaces are generally not used in program file names. Though the Harmony skeleton is language agnostic, it was first used with C, hence file names use primarily underscores. +

+ + +

Program Directories

+

+ A directory that is a package, follows the convention for name spaces. +

+ +

+ A directory that is used to give the contained files a property is an identifier. +

+ +

+ What is a reasonable primary property for a set of files? Perhaps: +

+
    +
  • Who uses each file with this property. Home directories are named like this.
    • +
  • The role of the people using the file. The developer and tester directories were named in this manner.
    • +
  • What program are the files for.
    • +
  • The generic category of program said files are for. We prefer the names authored and made. authored files are those written by humans or AI, while made files are products of tools.
    • +
+ +

Property indicators on files

+

+ We add a second property to a file using dot extensions to the file's name. We can extend the dot suffix model by using multiple dot suffixes. File name extensions are used to signal to the build tools how the file is to be processed: +

+
    +
  • .cli.c : Compiler fodder made into a stand-alone executable.
    • +
  • .lib.c : Library code source, compiled as an object file and added to the project archive.
    • +
  • .mod.c : Kernel module sources.
    • +
+ +

Exercises

+

Exercise 1: Program file names

+

+ Rename the following poorly named files to strictly adhere to the RT naming conventions. Pay close attention to the language culture, the capitalization of acronyms, and the rule for spelling out abbreviations in outer scope identifiers. +

+
    +
  • A C-culture makefile that builds a target library and command line interface: target-lib-cli.mk
    • +
  • A Lisp-culture source file for an HTML parsing module: html_parser_mod.el
    • +
  • A C language source file that initializes an HTTP server: init-http-srv.c
    • +
+ + + + diff --git a/developer/document/single-file_C-module-and-namespace.html b/developer/document/single-file_C-module-and-namespace.html new file mode 100644 index 0000000..0e6cd17 --- /dev/null +++ b/developer/document/single-file_C-module-and-namespace.html @@ -0,0 +1,184 @@ + + + + + + + C modules, namespaces, and the build lifecycle + + + + + + + + + + + +

Single-file C source

+

+ The RT C language culture abandons the traditional separation of declarations into .h header files and definitions into .c source files. A developer integrates the interface and the implementation into a single file. +

+

+ When a person includes the file using an #include directive, the file exposes only its interface. When the build system compiles the file into an object, a preprocessor macro acts as an implementation gate to compile the definitions. This keeps the source tree clean and ensures the interface and implementation never fall out of synchronization. +

+ +

Ad hoc namespaces

+

+ The C language lacks native namespaces. To prevent symbol collisions in large projects, RT code format uses a center dot (·) to denote ad hoc namespaces within identifiers. +

+

+ A programmer maps the directory structure directly to these namespaces. For example, a file located at authored/ExampleGreet/Math.lib.c belongs to the ExampleGreet namespace and defines the Math module. +

+
    +
  • Types and modules use PascalCase.
    • +
  • Functions and variables use snake_case.
    • +
+

+ An exported function from this module carries the full namespace and module prefix, such as ExampleGreet·Math·add. +

+ +

The implementation gate

+

+ To achieve the single-file source pattern, the code relies on two preprocessor constructs: +

+
    +
  • Include Guard: The entire file is wrapped in an include guard using the ·ONCE suffix. This prevents redeclaration errors if the file is included multiple times.
    • +
  • Implementation Block: The definitions are wrapped in a single #ifdef block using the exact namespace and module name. The build system dynamically injects this macro via a -D compiler flag when building the module's specific object file.
    • +
+ +

Build system mechanics

+

+ When a consumer file, such as hello.CLI.c, contains #include "Math.lib.c", the compiler processes the file without the ExampleGreet·Math macro defined. It skips the implementation and reads only the function prototype. +

+

+ When the orchestrator compiles the library object, it evaluates the target name and explicitly passes -DExampleGreet·Math to the compiler. This unlocks the gate, compiling the machine code for the definitions into Math.lib.o. +

+ +

Example: Math.lib.c

+

+ The following example demonstrates the complete structure. +

+ + + #ifndef ExampleGreet·Math·ONCE + #define ExampleGreet·Math·ONCE + + int ExampleGreet·Math·add(int a ,int b); + + #ifdef ExampleGreet·Math + + int ExampleGreet·Math·add(int a ,int b){ + return a + b; + } + + #endif // ExampleGreet·Math + #endif // ExampleGreet·Math·ONCE + + +

Cross-module dependencies

+

+ When one module depends on another, the developer directly includes the library source file. For example, if the Greeter module requires the Math module, the file Greeter.lib.c will contain: +

+ + #include "Math.lib.c" + +

+ Because every file is protected by a ·ONCE include guard, it is safe for multiple modules to include the same dependency. The preprocessor will expand the interface once per translation unit. +

+ +

Information hiding

+

+ To define internal helper functions or private data that should not be exposed in the module's public interface, a programmer places them strictly inside the #ifdef implementation block. +

+

+ To prevent these internal symbols from leaking into the global namespace during linking, they must be given internal linkage. The RT skeleton provides the Local macro (defined as static in RT_global.h) for this exact purpose. +

+ + #ifdef ExampleGreet·Math + + Local int internal_helper(int val){ + return val * 2; + } + + int ExampleGreet·Math·add(int a ,int b){ + return internal_helper(a) + b; + } + + #endif // ExampleGreet·Math + + +

Executable entry points

+

+ Programs intended to be compiled into standalone executables use the .CLI.c suffix. These files consume the library modules but do not define their own namespaces or include guards, as they are never included by other files. +

+

+ A .CLI.c file includes the necessary .lib.c files, parses command-line arguments in the main function, and passes native data types to a dedicated CLI() function to orchestrate the core logic. +

+ +

Directory structure and namespaces

+

+ The physical layout of the authored directory dictates how the build orchestrator finds the source code. +

+
    +
  • Flat Structure: If a project does not utilize namespaces, the .lib.c and .CLI.c files reside directly inside the developer/authored/ directory.
    • +
  • Namespaced Structure: If a project utilizes namespaces, the files are grouped into subdirectories matching the namespace name, such as developer/authored/ExampleGreet/.
    • +
+ +

Making the code

+

+ The developer compiles the code using the make wrapper script located in developer/tool/make. This script accepts two primary arguments: the build command and the target namespace. +

+ + + > make <command> [namespace] + + +

+ To build the ExampleGreet code, a person issues the following command from the developer workspace: +

+ + + > make all ExampleGreet + + +

+ Behind the scenes, the make wrapper assigns the namespace to an environment variable and calls the master orchestrator makefile. The orchestrator detects the namespace, updates the source directory search path to authored/ExampleGreet, and injects the appropriate namespace macros during compilation. +

+

+ The orchestrator compiles the .lib.c files into object files under scratchpad/build/object/, archives them into a library file, and links the .CLI.c objects into standalone executables. The final artifacts are placed in the scratchpad/made/ directory. +

+ +

Promoting the artifacts

+

+ Artifacts resting in the scratchpad are volatile and private to the developer. To share the executables and libraries with the rest of the project (such as the tester or consumer roles), the developer must promote them. +

+ + + > promote write + + +

+ The promote tool compares the contents of developer/scratchpad/made/ against consumer/made/. It atomically copies any new or updated files to the consumer workspace, stripping write permissions to enforce the immutability of the deployment target. Once promoted, the hello executable is ready for testing. +

+ + + + diff --git a/developer/source/todo.txt b/developer/document/todo.txt similarity index 100% rename from developer/source/todo.txt rename to developer/document/todo.txt diff --git a/document/.githolder b/developer/experiment/.gitkeep similarity index 100% rename from document/.githolder rename to developer/experiment/.gitkeep diff --git a/developer/scratchpad/.gitignore b/developer/scratchpad/.gitignore new file mode 100644 index 0000000..120f485 --- /dev/null +++ b/developer/scratchpad/.gitignore @@ -0,0 +1,2 @@ +* +!/.gitignore diff --git a/developer/tool/build b/developer/tool/build new file mode 100755 index 0000000..547fc4c --- /dev/null +++ b/developer/tool/build @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- +""" +developer/tool/build - Build Orchestrator Wrapper + +This script acts as the primary entry point for staging the project. +It parses colon-separated arguments mapping specific namespaces to their +intended build arguments. + +It enforces the structural convention where authored directories must +be named .. The script delegates execution to +developer/tool/build_component/. +""" + +import sys, os, subprocess, fnmatch +from typing import Dict, Tuple + +def check_environment() -> None: + setup_must_be = "developer/tool/setup" + current_setup = os.environ.get("SETUP", "") + if current_setup != setup_must_be: + print(f"developer/tool/build:: error: must be run in the {setup_must_be} environment", file=sys.stderr) + sys.exit(1) + +def get_namespaces() -> Dict[str, Tuple[str, str]]: + authored_dir = os.path.join(os.environ.get("REPO_HOME", "."), "developer", "authored") + if not os.path.isdir(authored_dir): + return {} + + namespaces = {} + for TM_item in os.listdir(authored_dir): + path = os.path.join(authored_dir, TM_item) + if os.path.isdir(path) and not TM_item.startswith("."): + parts = TM_item.rsplit(".", 1) + if len(parts) == 2: + ns = parts[0] + tool = parts[1] + namespaces[ns] = (tool, TM_item) + else: + print(f"(warning) Ignoring '{TM_item}': directory lacks the required . suffix.", file=sys.stderr) + + return dict(sorted(namespaces.items())) + +def print_usage(namespaces: Dict[str, Tuple[str, str]]) -> None: + print("Usage: build [[:]*]*\n") + print("Arguments are passed directly to the component specified by the directory suffix.\n") + + print("Available namespaces:") + if namespaces: + for TM_ns, (TM_tool, TM_raw_dir) in namespaces.items(): + print(f" {TM_ns:<20} (Component: {TM_tool}, Dir: {TM_raw_dir})") + else: + print(" (No conforming namespaces currently found in authored/)") + + print("\nExamples:") + print(" build (Displays this usage message)") + print(" build ExampleGreet (Executes 'all' via its assigned component)") + print(" build Ex*:library RT:all:information (Batch execution with pattern matching)") + print(" build \"*:clean\" (Cleans all namespaces. Quotes prevent Bash expansion)") + +def run_build(argument: str, namespace: str, tool: str, raw_dir: str) -> None: + tool_path = os.path.join("tool", "build_component", tool) + + if not os.path.isfile(tool_path) or not os.access(tool_path, os.X_OK): + print(f"error: Build component '{tool}' for namespace '{namespace}' is missing or not executable at {tool_path}", file=sys.stderr) + sys.exit(1) + + env = os.environ.copy() + env["NAMESPACE"] = namespace + env["NAMESPACE_DIR"] = raw_dir + + tool_cmd = [tool_path, argument] + + try: + subprocess.run(tool_cmd, env=env, check=True) + except subprocess.CalledProcessError as e: + print(f"Build component '{tool}' failed for namespace '{namespace}' with exit code {e.returncode}", file=sys.stderr) + sys.exit(e.returncode) + +def CLI() -> None: + check_environment() + + repo_home = os.environ.get("REPO_HOME", ".") + dev_dir = os.path.join(repo_home, "developer") + os.chdir(dev_dir) + + args = sys.argv[1:] + namespaces = get_namespaces() + + if not args or args[0] in ["usage", "help", "-h", "--help"]: + print_usage(namespaces) + sys.exit(0) + + matched_any = False + ns_keys = list(namespaces.keys()) + + for TM_arg in args: + parts = TM_arg.split(":") + pattern = parts[0] + arguments = parts[1:] + + if not arguments: + arguments = ["all"] + + matches = fnmatch.filter(ns_keys, pattern) + + if not matches: + print(f"(warning) Pattern '{pattern}' matched no namespaces.", file=sys.stderr) + continue + + matched_any = True + for TM_ns in matches: + tool, raw_dir = namespaces[TM_ns] + for TM_argument in arguments: + run_build(TM_argument, TM_ns, tool, raw_dir) + + if not matched_any: + sys.exit(1) + + print("build done.") + +if __name__ == "__main__": + CLI() diff --git a/developer/tool/build_component/copy b/developer/tool/build_component/copy new file mode 100755 index 0000000..0a1998c --- /dev/null +++ b/developer/tool/build_component/copy @@ -0,0 +1,59 @@ +#!/usr/bin/env -S python3 -B +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- +# developer/tool/build_component/copy + +import sys +import os +import shutil + +sys.path.insert(0, os.path.join(os.path.abspath("tool"), "build_component")) +import sync_engine + +HELP = """usage: copy {all|clean|help} + all Stages files from authored to scratchpad/made/. + clean Removes scratchpad/made/. + help Shows this message. +""" + +def cmd_all(): + namespace = os.environ.get("NAMESPACE") + namespace_dir = os.environ.get("NAMESPACE_DIR") + + if not namespace or not namespace_dir: + print("copy: error: NAMESPACE and NAMESPACE_DIR environment variables must be set.", file=sys.stderr) + sys.exit(1) + + src_root = os.path.join("authored", namespace_dir) + dst_root = os.path.join("scratchpad", "made", namespace) + + sync_engine.execute_sync(src_root, dst_root, prefix="copy", apply_modes=False) + +def cmd_clean(): + namespace = os.environ.get("NAMESPACE") + if not namespace: + print("copy: error: NAMESPACE environment variable must be set.", file=sys.stderr) + sys.exit(1) + + dst_root = os.path.join("scratchpad", "made", namespace) + if os.path.exists(dst_root): + shutil.rmtree(dst_root) + print(f"copy: Cleaned {dst_root}") + +def CLI(): + if len(sys.argv) < 2: + cmd = "all" + else: + cmd = sys.argv[1] + + if cmd == "all": + cmd_all() + elif cmd == "clean": + cmd_clean() + elif cmd in ["help", "-h", "--help"]: + print(HELP) + else: + print(f"copy: Unrecognized command '{cmd}'", file=sys.stderr) + sys.exit(1) + +if __name__ == "__main__": + CLI() diff --git a/developer/tool/build_component/make b/developer/tool/build_component/make new file mode 100755 index 0000000..30fb82d --- /dev/null +++ b/developer/tool/build_component/make @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# developer/tool/build_component/make + +set -euo pipefail +CMD="${1:-all}" + +# The GNU Make orchestrator expects C_SOURCE_DIR to point to the raw directory +export C_SOURCE_DIR="authored/$NAMESPACE_DIR" +export KMOD_SOURCE_DIR="authored/$NAMESPACE_DIR" + +# Isolate build products into their specific namespace directory +export LIBRARY_DIR="scratchpad/made/$NAMESPACE" +export MACHINE_DIR="scratchpad/made/$NAMESPACE" + +/bin/make -f tool/build_component/makefile "$CMD" diff --git a/developer/tool/build_component/makefile b/developer/tool/build_component/makefile new file mode 100644 index 0000000..7914c0b --- /dev/null +++ b/developer/tool/build_component/makefile @@ -0,0 +1,55 @@ +# developer/tool/build_component/makefile +.SUFFIXES: +.EXPORT_ALL_VARIABLES: + +RT_MAKEFILE_DP := $(REPO_HOME)/shared/tool/makefile + +.PHONY: usage +usage: + @printf "Usage: make [usage|version|information|all|lib|CLI|kmod|clean]\n" + +.PHONY: version +version: + @printf "local ----------------------------------------\n" + @echo tool/makefile version 2.0 + @printf "Harmony.mk ----------------------------------------\n" + @$(MAKE) -f $(RT_MAKEFILE_DP)/Harmony.mk version + @printf "target_kernel-module.mk ----------------------------------------\n" + @$(MAKE) -f $(RT_MAKEFILE_DP)/target_kernel-module.mk version + +.PHONY: information +information: + @printf "local ----------------------------------------\n" + -@echo CURDIR='$(CURDIR)' + @echo REPO_HOME="$(REPO_HOME)" + @echo NAMESPACE="$(NAMESPACE)" + @echo C_SOURCE_DIR="$(C_SOURCE_DIR)" + @echo KMOD_BUILD_DIR="/lib/modules/$(shell uname -r)/build" + @echo CURDIR="$(CURDIR)" + @printf "Harmony.mk ----------------------------------------\n" + @$(MAKE) -f $(RT_MAKEFILE_DP)/Harmony.mk information + @printf "target_kernel-module.mk ----------------------------------------\n" + @$(MAKE) -f $(RT_MAKEFILE_DP)/target_kernel-module.mk information + +.PHONY: all +all: library CLI kmod + +.PHONY: library lib +library lib: + @$(MAKE) -f $(RT_MAKEFILE_DP)/Harmony.mk library + +.PHONY: CLI +CLI: + @$(MAKE) -f $(RT_MAKEFILE_DP)/Harmony.mk CLI + +.PHONY: kmod +kmod: + @$(MAKE) -f $(RT_MAKEFILE_DP)/target_kernel-module.mk kmod + +.PHONY: clean +clean: + @printf "local ----------------------------------------\n" + @printf "Harmony.mk ----------------------------------------\n" + @$(MAKE) -f $(RT_MAKEFILE_DP)/Harmony.mk clean + @printf "target_kernel-module.mk ----------------------------------------\n" + @$(MAKE) -f $(RT_MAKEFILE_DP)/target_kernel-module.mk clean diff --git a/developer/tool/build_component/sync_engine.py b/developer/tool/build_component/sync_engine.py new file mode 100644 index 0000000..cf6412b --- /dev/null +++ b/developer/tool/build_component/sync_engine.py @@ -0,0 +1,80 @@ +#!/usr/bin/env -S python3 -B +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- +# developer/tool/sync_engine.py + +import os +import sys +import shutil +import filecmp +import stat +import tempfile + +def copy_file_atomic(src_abs, dst_abs, mode, dry=False, prefix="sync"): + if dry: + print(f"(dry) {prefix}: install -m {oct(mode)[2:]} '{src_abs}' -> '{dst_abs}'") + return + + parent = os.path.dirname(dst_abs) + os.makedirs(parent, exist_ok=True) + + fd, tmp_path = tempfile.mkstemp(prefix=".tmp.", dir=parent) + try: + with os.fdopen(fd, "wb") as tmpf, open(src_abs, "rb") as sf: + shutil.copyfileobj(sf, tmpf) + tmpf.flush() + os.chmod(tmp_path, mode) + os.replace(tmp_path, dst_abs) + except Exception as e: + if os.path.exists(tmp_path): + os.unlink(tmp_path) + raise e + +def execute_sync(src_root, dst_root, prefix="sync", dry=False, apply_modes=False, ignore_list=None): + wrote = False + if ignore_list is None: + ignore_list = [] + + if not os.path.isdir(src_root): + print(f"{prefix}: error: Source directory '{src_root}' does not exist.", file=sys.stderr) + return False + + for root, dirs, files in os.walk(src_root): + dirs.sort() + files.sort() + for fn in files: + if fn in ignore_list: + continue + + src_abs = os.path.join(root, fn) + rel = os.path.relpath(src_abs, src_root) + dst_abs = os.path.join(dst_root, rel) + + st = os.stat(src_abs) + if apply_modes: + is_exec = st.st_mode & stat.S_IXUSR + mode = 0o500 if is_exec else 0o400 + else: + mode = stat.S_IMODE(st.st_mode) + + if not os.path.exists(dst_abs): + copy_file_atomic(src_abs, dst_abs, mode, dry, prefix) + if not dry: print(f"{prefix}: Added {rel}") + wrote = True + continue + + if filecmp.cmp(src_abs, dst_abs, shallow=False): + continue + + dst_st = os.stat(dst_abs) + if dst_st.st_mtime > st.st_mtime: + print(f"{prefix}: error: {rel} skipped. Destination version is newer.") + continue + + copy_file_atomic(src_abs, dst_abs, mode, dry, prefix) + if not dry: print(f"{prefix}: Updated {rel}") + wrote = True + + if not wrote and not dry: + print(f"{prefix}: (info) nothing new to synchronize from {src_root}") + + return wrote diff --git a/developer/tool/promote b/developer/tool/promote new file mode 100755 index 0000000..7292374 --- /dev/null +++ b/developer/tool/promote @@ -0,0 +1,261 @@ +#!/usr/bin/env -S python3 -B +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- +# developer/tool/promote + +import os +import sys +import shutil +import stat +import pwd +import grp +import filecmp + +# Calculate the path relative to this script's location +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +# Assumes this script is in developer/tool/ +# sync_engine is in developer/tool/build_component +sys.path.insert(0, os.path.join(SCRIPT_DIR, "build_component")) +import sync_engine + +HELP = """usage: promote {write|clean|ls|diff|help|dry write} + write Copies what is staged in scratchpad/made into consumer. Only updates newer files. + clean Remove all contents of the consumer directory (preserves .gitignore). + ls List consumer as an indented tree: PERMS OWNER NAME. + diff List missing, orphaned, or out of sync files between scratchpad/made and consumer. + help Show this message. + dry write Preview what write would do without modifying the filesystem. + +Promoted files are read-only: 400, or 500 when executable. Directories below +consumer/ are 500, so not even the owner can add, replace or remove a file in +them; write and clean open them (700) for the duration and lock them again. +consumer/ itself stays 700, as it holds the .gitignore git manages. +""" + +SETUP_MUST_BE = "developer/tool/setup" +DEFAULT_DIR_MODE = 0o700 +LOCKED_DIR_MODE = 0o500 + +def exit_with_status(msg, code=1): + print(f"promote: {msg}", file=sys.stderr) + sys.exit(code) + +def assert_setup(): + setup_val = os.environ.get("SETUP", "") + if setup_val != SETUP_MUST_BE: + hint = ( + "SETUP is not 'developer/tool/setup'.\n" + "Enter the developer role with: ./setup developer\n" + "That script exports: ROLE=developer; SETUP=$ROLE/tool/setup" + ) + exit_with_status(f"bad environment: SETUP='{setup_val}'. {hint}") + +def repo_home(): + rh = os.environ.get("REPO_HOME") + if not rh: + exit_with_status("REPO_HOME not set") + return rh + +def dpath(*parts): + return os.path.join(repo_home(), "developer", *parts) + +def cpath(*parts): + return os.path.join(repo_home(), "consumer", *parts) + +def ensure_mode(path_str, mode): + try: os.chmod(path_str, mode) + except Exception: pass + +def ensure_dir(path_str, mode=DEFAULT_DIR_MODE, dry=False): + if dry: + return + os.makedirs(path_str, exist_ok=True) + ensure_mode(path_str, mode) + +def consumer_subdirs(root_dp): + """Every directory below root_dp; root_dp itself is not included.""" + found = [] + for TM_root, TM_dirs, TM_files in os.walk(root_dp): + for TM_d in TM_dirs: + found.append(os.path.join(TM_root, TM_d)) + return found + +def set_subdir_modes(root_dp, mode): + for TM_d in consumer_subdirs(root_dp): + ensure_mode(TM_d, mode) + +def filemode(m): + try: return stat.filemode(m) + except Exception: return oct(m & 0o777) + +def owner_group(st): + try: return f"{pwd.getpwuid(st.st_uid).pw_name}:{grp.getgrgid(st.st_gid).gr_name}" + except Exception: return f"{st.st_uid}:{st.st_gid}" + +def list_tree(root_dp): + if not os.path.isdir(root_dp): + return + + entries = [] + def gather(path_str, depth, is_root): + try: + it = list(os.scandir(path_str)) + except FileNotFoundError: + return + dirs = [TM_e for TM_e in it if TM_e.is_dir(follow_symlinks=False)] + files = [TM_e for TM_e in it if not TM_e.is_dir(follow_symlinks=False)] + dirs.sort(key=lambda TM_e: TM_e.name) + files.sort(key=lambda TM_e: TM_e.name) + + if is_root: + for TM_f in (TM_e for TM_e in files if TM_e.name.startswith(".")): + st = os.lstat(TM_f.path) + entries.append((False, depth, filemode(st.st_mode), owner_group(st), TM_f.name)) + for TM_d in dirs: + st = os.lstat(TM_d.path) + entries.append((True, depth, filemode(st.st_mode), owner_group(st), TM_d.name + "/")) + gather(TM_d.path, depth + 1, False) + for TM_f in (TM_e for TM_e in files if not TM_e.name.startswith(".")): + st = os.lstat(TM_f.path) + entries.append((False, depth, filemode(st.st_mode), owner_group(st), TM_f.name)) + else: + for TM_d in dirs: + st = os.lstat(TM_d.path) + entries.append((True, depth, filemode(st.st_mode), owner_group(st), TM_d.name + "/")) + gather(TM_d.path, depth + 1, False) + for TM_f in files: + st = os.lstat(TM_f.path) + entries.append((False, depth, filemode(st.st_mode), owner_group(st), TM_f.name)) + + gather(root_dp, 1, True) + + ogw = 0 + for TM_isdir, TM_depth, TM_perms, TM_ownergrp, TM_name in entries: + if len(TM_ownergrp) > ogw: + ogw = len(TM_ownergrp) + + print("consumer/") + for TM_isdir, TM_depth, TM_perms, TM_ownergrp, TM_name in entries: + indent = " " * TM_depth + print(f"{TM_perms} {TM_ownergrp:<{ogw}} {indent}{TM_name}") + +def cmd_write(dry=False): + dst_root = cpath() + ensure_dir(dst_root, DEFAULT_DIR_MODE, dry=dry) + src_root = dpath("scratchpad", "made") + + # Files are replaced by writing a temporary file beside them and renaming + # it into place, which needs write permission on the directory. + if not dry: set_subdir_modes(dst_root, DEFAULT_DIR_MODE) + try: + sync_engine.execute_sync( + src_root, + dst_root, + prefix="promote", + dry=dry, + apply_modes=True, + ignore_list=[".gitignore"] + ) + finally: + if not dry: set_subdir_modes(dst_root, LOCKED_DIR_MODE) + +def cmd_diff(): + dst_root = cpath() + src_root = dpath("scratchpad", "made") + + src_files = set() + if os.path.isdir(src_root): + for TM_root, TM_dirs, TM_files in os.walk(src_root): + for TM_fn in TM_files: + if TM_fn == ".gitignore": + continue + src_files.add(os.path.relpath(os.path.join(TM_root, TM_fn), src_root)) + + dst_files = set() + if os.path.isdir(dst_root): + for TM_root, TM_dirs, TM_files in os.walk(dst_root): + for TM_fn in TM_files: + if TM_fn == ".gitignore": + continue + dst_files.add(os.path.relpath(os.path.join(TM_root, TM_fn), dst_root)) + + if not src_files and not dst_files: + print("No differences found. Both directories are empty or missing.") + return + + all_files = sorted(list(src_files | dst_files)) + found_diff = False + + for TM_rel in all_files: + if TM_rel not in dst_files: + print(f"Pending promotion (missing in consumer): {TM_rel}") + found_diff = True + elif TM_rel not in src_files: + print(f"Orphaned in consumer (missing in scratchpad/made): {TM_rel}") + found_diff = True + else: + src_abs = os.path.join(src_root, TM_rel) + dst_abs = os.path.join(dst_root, TM_rel) + + if not filecmp.cmp(src_abs, dst_abs, shallow=False): + src_mtime = os.stat(src_abs).st_mtime + dst_mtime = os.stat(dst_abs).st_mtime + + if src_mtime > dst_mtime: + print(f"Pending update (contents differ, newer in scratchpad/made): {TM_rel}") + else: + print(f"Contents differ (locally modified in consumer): {TM_rel}") + found_diff = True + + if not found_diff: + print("No differences found. Consumer matches developer scratchpad/made.") + +def cmd_clean(): + consumer_root_dir = cpath() + if not os.path.isdir(consumer_root_dir): + return + # Removing an entry needs write permission on the directory holding it. + set_subdir_modes(consumer_root_dir, DEFAULT_DIR_MODE) + for TM_name in os.listdir(consumer_root_dir): + if TM_name == ".gitignore": + continue + p = os.path.join(consumer_root_dir, TM_name) + if os.path.isdir(p) and not os.path.islink(p): + shutil.rmtree(p, ignore_errors=True) + else: + try: os.unlink(p) + except FileNotFoundError: pass + +def CLI(): + assert_setup() + + repo_home = os.environ.get("REPO_HOME", ".") + dev_dir = os.path.join(repo_home, "developer") + os.chdir(dev_dir) + + if len(sys.argv) < 2: + print(HELP) + return + + cmd = sys.argv[1] + args = sys.argv[2:] + + if cmd == "write": + cmd_write(dry=False) + elif cmd == "clean": + cmd_clean() + elif cmd == "ls": + list_tree(cpath()) + elif cmd == "diff": + cmd_diff() + elif cmd == "help": + print(HELP) + elif cmd == "dry": + if args and args[0] == "write": + cmd_write(dry=True) + else: + print(HELP) + else: + print(HELP) + +if __name__ == "__main__": + CLI() diff --git a/developer/tool/setup b/developer/tool/setup new file mode 100644 index 0000000..0b993ad --- /dev/null +++ b/developer/tool/setup @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +script_afp=$(realpath "${BASH_SOURCE[0]}") + diff --git a/document/introduction_Harmony.html b/document/introduction_Harmony.html new file mode 100644 index 0000000..432c63b --- /dev/null +++ b/document/introduction_Harmony.html @@ -0,0 +1,249 @@ + + + + + Introduction to Harmony + + + + + + + + + + + +

Purpose

+

+ Harmony is a project skeleton: a language-agnostic directory structure and a set of maintenance tools, kept in a git repository, from which new projects are made. It is designed for long-lived software developed by a team, and serves equally well for one person, who takes on each role in turn. The structure exists to give: +

+
    +
  1. Clarity about where things live. +
      +
    • Role-based work areas.
      • +
    • Separation of skeleton, team member authored, machine-made, and depended-upon software.
      • +
    +
    2. +
  2. A safe, predictable build and promotion workflow.
    3. +
+ +

Making a project

+

+ A project administrator makes a new project from a clone of Harmony, by taking on the administrator role and running new-project with the name of the new project: +

+ + > cd Harmony + > ./setup administrator + > new-project <name> + +

+ new-project makes the new project as a sibling directory of Harmony, then: +

+
    +
  1. Copies the files of Harmony's last commit, leaving out Harmony's git history, untracked files, and uncommitted changes.
    2. +
  2. Renames 0pus_Harmony to 0pus_<name>.
    3. +
  3. Adds a line for the new project to shared/tool/version.
    4. +
  4. Initializes a git repository on the core-developer_branch branch, and commits the copied skeleton.
    5. +
  5. Adds any remote repositories given as -remote <url>.
    6. +
+

+ Because the new project starts with its own git history, work done in it cannot be checked in to the Harmony skeleton by accident. +

+

+ Leave the Harmony line in the version file. It records which skeleton the project was made from, so that an administrator can later see what has changed in the skeleton since, and bring the project up to date. The administrator tool skeleton_diff compares a project with a Harmony checkout for this purpose. +

+ +

Taking on a role

+

+ A person works in a project by taking on a role. The roles are administrator, developer, tester, and consumer. From the top directory of the project, run setup with the role: +

+ + > ./setup developer + +

+ setup starts a new bash shell configured for the role, and leaves the person in the role's directory. The prompt shows the project and the role. exit leaves the role and returns to the shell setup was run from. It is common to have several terminal windows or Emacs shells open, each in a different role. +

+

+ Python programmers will recognize this as similar to a virtual environment. Harmony calls it the project setup, and the command is setup rather than activate. Unlike a Python virtual environment, there is one setup for each role. +

+

+ In the new shell, the setup: +

+
    +
  • Sets ROLE, ROLE_HOME and SETUP, and puts <role>/tool on the PATH.
    • +
  • Sources the project-wide shared/tool/setup, which sets REPO_HOME and PROJECT, puts shared/tool and shared/authored on the PATH, and sources each project dependency's .init file.
    • +
  • Sources shared/authored/setup, when the project has one, for project-specific additions to every role.
    • +
  • Changes into the role's directory and sources <role>/tool/setup, the setup customized for that role in this project.
    • +
  • When running inside Emacs, copies the environment to Emacs with env_to_emacs.
    • +
+

+ Tools rely on this environment. Absolute paths are formed from REPO_HOME, and some tools, such as build and promote, refuse to run outside the role they belong to. +

+ +

Roles and their directories

+

+ Each role has a directory at the top of the project. The shared directory holds what all roles use. +

+ +

Administrator

+

+ The administrator keeps the project in step with the Harmony skeleton, maintains the role environments, installs project dependencies, and makes releases. administrator/tool holds the tools for this, among them new-project, skeleton_diff, and archive, which writes an archive of the repository to the scratchpad. +

+ +

Developer

+

+ The developer writes the project's source and builds it. +

+
    +
  • authored/ : Source written by the team. Tracked.
    • +
  • document/ : Documents for developers, such as coding and naming conventions.
    • +
  • experiment/ : Informal, short-lived trials.
    • +
  • scratchpad/ : Untracked. Intermediate build files and the staged products of a build.
    • +
  • tool/ : The developer's tools, including build and promote.
    • +
+ +

Tester

+

+ The tester holds the formal tests, including regression suites, and runs them against the work product in consumer/. A developer may keep informal tests in developer/experiment. A test that is to be kept is moved to tester/, which keeps writing code and validating it apart. +

+ +

Consumer

+

+ consumer/ receives the work product when a developer promotes it. It is untracked, apart from its .gitignore, and is treated as a deployment target: files arrive there only through promote, and nothing is built in it. The consumer role uses the work product as its end users would. +

+ +

Shared

+
    +
  • shared/tool/ : Tools for all roles, and the project-wide setup. The version command prints the version lines.
    • +
  • shared/authored/ : Project-specific additions for all roles. It is on every role's PATH, and may hold a setup that every role sources.
    • +
  • shared/project-dependency/ : Software the project uses but did not author. See "Project dependencies" below.
    • +
+ +

Building and promoting

+

+ Building and promotion are separate steps. In the developer role, build makes the work product from developer/authored and stages it on the developer's scratchpad. promote write then copies what changed to consumer/. The result is a promotion candidate, which stays as it is until the next promotion. +

+

+ Source in developer/authored is grouped into directories named <Namespace>.<Tool>. The suffix names the build component in developer/tool/build_component that builds that directory, such as make for C, or copy for files used as they are. Given no arguments, build prints its usage and the namespaces it found. promote given no arguments prints its commands, among them dry write, which shows what write would copy. +

+

+ The tester runs tests against the promotion candidate. Tests read from consumer/ and never write to it, though a tester may copy files elsewhere to experiment with them. A developer often has a second window open in the tester role, and cycles through edit, build, promote and test. +

+ +

After cloning a project

+

+ Built products are not tracked, so a fresh clone has an empty consumer/. A team member who needs the work product builds it, taking on the developer role to do so: +

+ + > ./setup developer + > build <namespace> + > promote write + > exit + > ./setup <role> + +

+ The developer documents any project-specific build steps in developer/document/build.html. A fresh clone also lacks the project's dependencies, which each team member installs as described beside each dependency in shared/project-dependency. +

+ +

Project dependencies

+

+ Harmony is language agnostic. When a project uses C, Python, NodeJS, Java, or other software it did not author, the administrator installs that software as a sibling directory of the project and links it in under shared/project-dependency, with an .init file that every role's setup sources to put the dependency into the environment. Installation notes for each dependency sit beside its .init file, so that each team member can repeat the installation after cloning. +

+

+ The directory comes with a project link to the directory holding the project and its siblings, through which dependencies are linked, and with Python as an example dependency. The steps are given in administrator/document/how-to_add-project-dependency.html. +

+ +

Directory names

+

+ A directory name is a property of each file in the directory, so a full path reads as a description of the file. Because a name states a property of each file, it is rarely plural: when every file in a directory is a test, the directory is named test. +

+

+ A conventional file system can only approximate a property-based one. Each file sits in one directory, so it receives one property per level of the path. Files sharing several properties can be given a compound name with the properties separated by an underscore, but overlapping groupings cannot be expressed. +

+

+ When choosing a directory name, prefer properties in this order: +

+
    +
  • Role (administrator, developer, tester, consumer): the role, human or AI, that works with the files.
    • +
  • Provenance (authored, made): whether a person wrote the file or a tool produced it.
    • +
  • Capability (tool, document, experiment): what the file is for.
    • +
  • Lifecycle (scratchpad, stage): how long the file is expected to last, or where it is in promotion.
    • +
  • Tracking (tracked, untracked): what version control does with it.
    • +
+ +

Authored, made, scratchpad, inherited

+

+ Files in a directory named authored were written by the project's team. They did not come with the skeleton, nor with the installation of other software, and they typically constitute the project's intellectual property. Build tools treat authored directories as read-only. All source that is built into a promotion or a release is placed in developer/authored. +

+

+ Files that came with the skeleton are inherited. When the team edits one, such as developer/tool/build, it becomes a customization. New documents go in document directories and new tools in tool directories. Because the Harmony line stays in the version file, an administrator can tell which skeleton files a project has customized and which it has added. +

+

+ Files in a directory named scratchpad are untracked, so a clone always has empty scratchpads. Tools place intermediate and staged files there, each in its own subdirectory, so a person who knows those names can also use a scratchpad as a temporary directory. The scratchpad tool maintains the scratchpad of the current directory. Note that its clear command deletes everything on that scratchpad. +

+ +

Documents

+

+ Each directory named document holds documents for the role, or the scope, of its parent: +

+
    +
  • document/ : The project as a whole, such as this introduction.
    • +
  • administrator/document/ : Maintaining the project skeleton, adding dependencies, and making releases.
    • +
  • developer/document/ : Coding standards, conventions, and the build.
    • +
  • tester/document/ : Test plans and testing tools.
    • +
  • consumer/<namespace>/document/ : Documents released with the work product, such as manuals and API references.
    • +
+

+ Documents are written in HTML with the RT semantic tags, and formatted by RT-Style, a separate project at https://github.com/Thomas-Walker-Lynch/RT-Style. A document loads RT-Style through a short script block at the top of its head. To write a new document, copy an existing one, keep that block as it is, and type over the contents. +

+

+ Documents are read in Chrome. With nothing set up, a document loads RT-Style from style.ReasoningTechnology.com. To read without a network connection, or against a local copy of RT-Style, open that copy's Manuscript/setup.html once in Chrome. Chrome then loads RT-Style from it for every local document. +

+

+ document/writing_tool-script.html gives the conventions for writing tools. +

+ +

What git tracks

+

+ Everything is tracked except: +

+
    +
  • consumer/, apart from its .gitignore.
    • +
  • Every scratchpad/, apart from its .gitignore.
    • +
  • In shared/project-dependency/, whatever its .gitignore does not list as tracked.
    • +
  • Archives, editor backups, and compiled files, as listed in the top-level .gitignore.
    • +
+ +

Workflow and releases

+

+ The project moves forward through a cycle: the developer promotes a candidate, the tester tests it and files issues, and the developer fixes them and promotes again. The core-developer_branch is the branch the core development team works on. When the product manager judges the work product reliable and complete enough, the administrator makes a project release on a branch named release_v<major>, and tags it. +

+

+ These documents give the details: +

+
    +
  • document/role-and-workflow_product-development.html : The roles, and how their work interacts.
    • +
  • document/role-and-workflow_product-maintenance.html : Issues, triage, and patching releases.
    • +
  • administrator/document/how-to_release.html : Making a release.
    • +
+ + + + diff --git a/document/role-and-workflow_product-development.html b/document/role-and-workflow_product-development.html new file mode 100644 index 0000000..411b732 --- /dev/null +++ b/document/role-and-workflow_product-development.html @@ -0,0 +1,134 @@ + + + + + Product development roles and workflow + + + + + + + + + + + +

Roles as hats

+

+ In Harmony, a role is a hat a person wears. There can be multiple people sharing a single role, and a single person can wear many hats. Some roles interact directly with the project directory structure, while others guide the process from outside the codebase. +

+ +

Workspace roles

+

+ These roles interact directly with the repository. To enter a workspace, change directory to the top-level of the project and source the setup file for the desired role: +

+ + > ./setup administrator + > ./setup developer + > ./setup tester + > ./setup consumer + + +

This will run a new shell with the proper environment setting. It is common for a person to have multiple shells, terminal sessions, or IDEs open, each running under a different role environment. Development and tests is a common pair, though tests are intended to be production tests. The developer has an experiment directory for experiments. +

+ +

Administrator role

+

Responsibilities:

+
    +
  1. Set up the project directory and keep it in sync with the Harmony skeleton.
    2. +
  2. Maintain role environments (apart from role-specific tool/setup files).
    3. +
  3. Install and maintain shared and third_party tools, addressing issues with the project workflow. Note that the term "third_party" encompasses any software not authored within this specific project.
    4. +
+ +

Developer role

+

Responsibilities and Boundaries:

+
    +
  1. Write and modify authored/ source.
    2. +
  2. Run builds and place artifacts in scratchpad/made, then execute the promote write script to copy artifacts to consumer/made for testing.
    3. +
  3. Run experiments in experiment/. These experiments can sometimes be promoted to formal tests, but there is no requirement to do so. The developer role should not blur into the tester role; experiments are informal, whereas tests are formal and retained.
    4. +
  4. Strict Boundary: A developer never writes into the tester/ directory. Instead, a developer adds tests to developer/experiment/ and offers to share them.
    5. +
+ +

Tester role

+

Responsibilities and Boundaries:

+
    +
  1. Evaluate candidates under consumer/made/ and run regression suites to confirm: +
      +
    • That the code does not crash.
      • +
    • That consumers do not have a bad experience.
      • +
    • That the goals specified by the product manager are met.
      • +
    +
    2. +
  2. File issues and communicate feedback to the developers.
    3. +
  3. Strict Boundary: A tester never patches code in the developer/ directory. Instead, the tester files issues or proposes code fixes on a separate branch.
    4. +
+ +

Consumer role

+

Responsibilities:

+
    +
  1. Act as the end-user simulation environment.
    2. +
  2. Consume and deploy artifacts exclusively from the consumer/made/ target.
    3. +
  3. Never author or modify code; strictly run local builds or deployments for architecture-specific testing.
    4. +
  4. Report issues. (Anyone can report an issue, and consumers regularly do).
    5. +
+ +

External roles

+

+ These roles drive the project forward but do not have a dedicated setup <role> workspace, as they do not directly build or test the code in that capacity. If these individuals need to interface with the code, they simply put on a workspace role hat, such as administer, developer, tester, or consumer. +

+ +

Product manager

+

+ The product manager receives specifications from the architect and sets the specific goals for a release. When the project manager provides updates indicating readiness, the product manager makes the final decision to cut a project release. +

+ +

Project manager

+

+ The project manager owns the schedule (such as the Gantt chart) and monitors progress. They coordinate code reviews, read issues and announcements, reformulate the schedule, and instruct the product manager. When parts of the codebase are outsourced, the project manager serves as the primary point of contact with the external teams. +

+ +

Architect

+

+ The architect writes the original specification, sets the technical direction, and performs code reviews. The architect might be a contractor or an in-house team member. +

+ +

The four interacting loops

+

+ The project moves forward through the continuous interaction of four distinct operational cycles. +

+ +
    +
  1. Developer Loop: Write code, compile to the scratchpad made directory, promote to the consumer made directory, announce the promotion to the tester, read issues, and repeat.
    2. +
  2. Tester Loop: Read developer announcements, read the consumer made directory, run tests, and file issues.
    3. +
  3. Product Manager Loop: Provide specifications to the developer, receive progress updates from the project manager, and instruct the project administrator to make new git branch project releases and deploy.
    4. +
  4. Project Manager Loop: Own the Gantt chart, coordinate code reviews, read issues and announcements, reformulate the Gantt chart, and instruct the product manager.
    5. +
+ +

Promotion mechanics

+

+ Building and promotion are separate activities. The developer compiles and places files in developer/scratchpad/made. The developer then runs promote write to transfer those files to consumer/made. +

+

+ The consumer/made directory is strictly an untracked deployment target. No tools are permitted to rebuild during promotion, and no builds are run directly inside the consumer made directory. +

+ + + + diff --git a/document/role-and-workflow_product-maintenance.html b/document/role-and-workflow_product-maintenance.html new file mode 100644 index 0000000..e93f12d --- /dev/null +++ b/document/role-and-workflow_product-maintenance.html @@ -0,0 +1,111 @@ + + + + + Product maintenance roles and workflow + + + + + + + + + + + +

Maintenance philosophy

+

+ Deployed software must be a usable quality product for the customer, beyond that we favor addressing issues in new releases, and encourage customers who are having issues to upgrade. This approach is purely one of keeping the maintenance problem tractable. +

+ +

Maintenance structure

+

+ Teams mentioned in this document: +

+
    +
  1. tester team
    2. +
  2. core developer team
    3. +
  3. branch maintenance team
    4. +
  4. triage team
    5. +
+ +

+ Team membership is a role a person takes on. Multiple people can share a single role, and one person might take on multiple roles. +

+ +

+ The tester team develops and maintains the regression suite, the reliability suite, and additional tests. +

+ +

+ The core developer team writes the source code and compiles promotion candidates. +

+ +

+ The branch maintenance team is typically a subset of the core developer team. This group is dedicated to applying approved fixes to specific release branches. +

+ +

+ The triage team evaluates defects reported against active releases. The team usually includes a developer contact, the project manager, and the product manager. +

+ +

Issue queues

+

+ The project maintains two primary issue queues: +

+
    +
  1. released product
    2. +
  2. core developer
    3. +
+ +

+ There can be additional queues for multiple branches of development, and for experiments. +

+ +

Core developer queue

+

+ This queue serves the core_developer_branch. A tester writes a test for every issue reported in this queue. Doing so isolates the defect, proves the fix, and guards against future regressions. +

+ +

Released product queue

+

+ All issues found in release branches go onto this queue initially. It tracks defects reported against deployed software across any major version. +

+ +

Triage and patching

+

+ Guided by the project philosophy, the triage team reviews each release queue issue to determine its impact. The team assesses whether the defect affects the core_developer_branch, assuming by default that it probably does. The team also determines if the defect is critical enough to warrant a patch on one or more active release branches. +

+

+ Based on this assessment, the triage team files actionable tickets in the core developer queue. A ticket explicitly specifies its target branches. A ticket will be filed against either the core developer branch, specific release branches, or a combination of both. +

+

+ Members of the tester team also file tickets directly into the core developer queue when they discover defects in the core branch. +

+

+ Responsibility for resolving a ticket depends on its target. The core developer team addresses fixes required on the core_developer_branch. The branch maintenance team addresses fixes required on the release_v<major> branches. +

+

+ When a release is patched, the branch name remains static. The administrator advances the minor release number in the shared/tool/version file and tags the commit. +

+ + + + diff --git a/document/writing_tool-script.html b/document/writing_tool-script.html new file mode 100644 index 0000000..5942d3f --- /dev/null +++ b/document/writing_tool-script.html @@ -0,0 +1,287 @@ + + + + + Writing a tool script + + + + + + + + + + + +

Purpose

+ +

+ A tool is a script a person runs from a role setup to do work on files. This document gives the conventions for how an RT tool presents itself on the command line, reports what it does, and changes files safely. It complements RT-code-format.html, which covers identifier names, layout, and the separation of the CLI() function from the work functions. Those rules apply here and are not repeated. +

+ +

+ The conventions serve three needs of the person at the keyboard. They must be able to see what a tool will do before it does it, read afterward what it did, and run it again without harm. +

+ +

+ Examples are taken from RT-Style_header_v5_to_v6, the administrator tool that converts RT-Style version 5 document headers to version 6. +

+ +

The command line

+ +

No arguments prints the help

+ +

+ A tool given no arguments prints its help message and exits with status 0. A person who types the bare name of a tool is asking what it does. Nothing on the file system changes. +

+ +

The first argument is a command

+ +

+ The first argument names a command, written as a word, not a flag. The words help, -h and --help all print the help message. Further arguments, when a command accepts any, follow the command. A command that accepts no further arguments rejects them with an error. +

+ +

+ An unknown command prints an error naming it, then the help message, and exits with a nonzero status. +

+ +

The help message is the module docstring

+ +

+ In Python the help message is the module docstring, printed by print(__doc__.strip()). The documentation at the top of the file and the message the person sees are then the same text, and cannot drift apart. The docstring opens with the tool name, a paragraph on what the tool does, then the usage and one line per command: +

+ + + Usage: RT-Style_header_v5_to_v6 <command> [file ...] + RT-Style_header_v5_to_v6 preview <command> [file ...] + + help | -h | --help print this message (also printed when given no arguments) + project walk the directory tree rooted at $REPO_HOME + tree walk the directory tree rooted at the current directory + local only the current directory, no subdirectories + file <name> ... only the named .html files + preview <command> print the log of what <command> would do, change nothing + + +

Scope commands

+ +

+ A tool that acts on files found in a directory tree offers the same four commands for choosing which files: +

+
    +
  • project walks the tree rooted at $REPO_HOME. When REPO_HOME is not set, the command is an error. Otherwise it behaves as tree run from $REPO_HOME, from whatever directory the person is in.
    • +
  • tree walks the tree rooted at the current directory.
    • +
  • local acts on the current directory only, without descending.
    • +
  • file acts on the files named after it.
    • +
+ +

+ Only project depends on a Harmony project. The others work anywhere, because the files a tool serves are not always in a project. RT-Style documents, for example, appear in projects and outside them. +

+ +

Preview

+ +

+ A tool that changes files offers preview, placed before any other command. preview tree prints the log that tree would print and changes nothing. The person reads it, decides, then runs the command without preview. +

+ +

+ Preview is not a separate code path. The work functions are given a preview flag and test it only at the point of each action, immediately before writing or deleting. Everything else, finding files, deciding what to do, and logging, runs the same in both modes. A preview log that came from different code could not be trusted to describe the real run. +

+ +

The log

+ +

+ A tool that acts on files prints a log of each thing it did. The log is for a person reading a terminal, and for that same person reading it again in a scrollback or a saved file later. This is the log of converting the Harmony skeleton documents, with the path on the first line shortened to fit the page: +

+ + + RT-Style_header_v5_to_v6 project in /home/Thomas/.../project/Harmony + converted administrator/document/how-to_release.html + converted developer/document/naming_file-and-directory.html + converted developer/document/single-file_C-module-and-namespace.html + converted document/introduction_Harmony.html + converted document/role-and-workflow_product-development.html + converted document/role-and-workflow_product-maintenance.html + + summary: + 6 .html files converted to version 6 + 0 RT-Manuscript_locator.js files deleted + 0 .html files already version 6 + 0 .html files with mentions to edit by hand + 0 other .html files left alone + + +
    +
  • The first line names the tool, the command, and the absolute directory the work is rooted in, so a saved log says where it came from.
    • +
  • Each action is one line. A fixed-width tag in the past tense says what happened, followed by the path relative to the root. Relative paths keep lines short and make logs from different machines comparable.
    • +
  • In a preview the tags read would convert, would delete, and the summary ends with preview only: nothing was changed.
    • +
  • A decision not to act is logged too, with the reason in parentheses, as in kept shared/document/RT-Manuscript_locator.js (no version 5 document beside it). A file the tool left alone on purpose must be distinguishable from one it missed.
    • +
  • Work the tool cannot do safely is logged with the tag edit by hand, the file, the line number and the line, as in edit by hand .../Document/user.html:83: &lt;script src=..., so a person can go straight to it.
    • +
  • A summary of counts ends the log. A count line that would always be zero for a run, such as errors, may be left out when it is zero.
    • +
  • Walk directories and files in sorted order, so two runs over the same tree produce the same log.
    • +
+ +

+ Errors are logged on their own lines with the tag error, the tool carries on with the remaining files, and the exit status is nonzero when any error occurred. Errors in the command line itself are printed to standard error before any work starts. +

+ +

Walking directories

+ +

+ A tool walks every directory in its scope, whatever the directory is named. It does not skip directories by name, since a name that means nothing to the tool's author may hold files that matter to its user. +

+ +

+ A tool does not follow symbolic links to directories. A Harmony project has shared/project-dependency/project pointing to ../../../, the directory holding all the projects. Following it would send the walk into every neighboring project and around the loop back into this one. In Python, os.walk does not follow such links unless given followlinks=True, so leave that argument out. +

+ +

Paths

+ +

+ A tool forms an absolute path only by starting from an environment variable that the role setup provides, such as $REPO_HOME. Paths the tool writes into files, and symbolic links it creates, are relative. +

+ +

+ An absolute path written into a file is correct for only one view of the file system. Under the subu arrangement, the master user reaches a sub-user's files through a mapped mount at a different absolute path from the one the sub-user sees. A path recorded by one of them is then unreadable, or points somewhere else, for the other. A relative path is resolved from wherever the file is reached, so it is correct in every view. +

+ +

Changing files safely

+ +

Checkpoint first

+ +

+ A tool edits files in place and does not make backup copies. The person checkpoints the project with git before running it, and reviews the result with git diff. Run preview first. +

+ +

Change only what is recognized exactly

+ +

+ Match the precise form the tool was written to change, in the precise place it occurs. RT-Style_header_v5_to_v6 replaces a locator script line only when it stands alone on a line inside <head>. A mention of the locator anywhere else, such as an escaped example in a manual, is not touched. It is logged for a person to edit. +

+ +

Keep everything not being changed

+ +

+ Carry over what the edit is not about: the indentation of a replaced line, line endings, file permissions, and every other line of the file byte for byte. After a run, git diff should show the intended change and nothing else. +

+ +

Running twice does no harm

+ +

+ A second run of the same command finds nothing left to do, changes nothing, and says so in its summary. A tool achieves this by recognizing work already done. The header converter counts a file holding the version 6 block as already version 6 and leaves it alone. A person can therefore rerun a tool to confirm a job is complete, or to finish one that stopped part way. +

+ +

Before deleting, check nothing still uses it

+ +

+ A tool deletes a file only when nothing remaining depends on it. In testing, the header converter run with file on two of six test documents in a directory deleted the shared locator file while the other four still needed it. The rule now is that a locator is deleted only when no unconverted version 5 document beside it still loads it. Otherwise it is kept, and the log names the documents still using it. +

+ +

Leave alone what the tool cannot identify as its own

+ +

+ A file with the right name in an unexpected place is not evidence that the tool should remove it. The converter keeps a locator file that has no version 5 document beside it, and logs it as kept, leaving the decision to a person. +

+ +

Testing a tool

+ +

+ Test a new tool on a copy of real material before it runs on the original. An archive of the project, extracted into a scratch directory, gives realistic files that can be discarded. +

+
    +
  1. Run with no arguments, an unknown command, and each command's error cases, such as project with REPO_HOME unset or file naming a file that does not exist.
    2. +
  2. Run preview and confirm that nothing changed, for example with diff -r against an untouched copy.
    3. +
  3. Run each scope command from more than one directory, including from outside any project.
    4. +
  4. Run each command twice. The second run must report nothing left to do.
    5. +
  5. Run on part of a job, such as file on some of the documents in a directory, and confirm the rest still work.
    6. +
  6. Read the diff of a changed file in full.
    7. +
+ +

A skeleton

+ +

+ A Python tool laid out to these conventions begins with #!/usr/bin/env python3, then the Emacs mode line the Harmony tools carry, which sets two-space indentation and no tabs. The rest of the skeleton follows: +

+ + + """ + tool_name + + What the tool does, in a paragraph. + + Usage: tool_name <command> [file ...] + tool_name preview <command> [file ...] + + help | -h | --help print this message, also printed given no arguments + project walk the tree rooted at $REPO_HOME + tree walk the tree rooted at the current directory + local only the current directory + file <name> ... only the named files + preview <command> print the log of <command>, change nothing + """ + + import sys ,os + from pathlib import Path + + def work(dirpa_root ,recursive ,preview): + ... + + def print_help(): + print(__doc__.strip()) + return 0 + + def CLI(argv=None): + if argv is None: argv = sys.argv[1:] + if not argv or argv[0] in ("help" ,"-h" ,"--help"): return print_help() + + preview = argv[0] == "preview" + if preview: argv = argv[1:] + if not argv: + print("Error: 'preview' needs a command after it." ,file=sys.stderr) + return 1 + + command ,rest = argv[0] ,argv[1:] + ... + + if __name__ == "__main__": + sys.exit(CLI()) + + +

Exercises

+ +

Exercise 1: The bare tool

+

+ A tool named tidy deletes editor backup files ending in ~. When run with no arguments it deletes them from the current directory. List what is wrong with that behavior under these conventions, and give the command line it should have instead. +

+ +

Exercise 2: A log line

+

+ A tool prints Processing /home/Thomas/subu/developer/project/Two/document/a.html ... done for every file it looks at, whether or not it changed it. Rewrite the log for a run in which one of three files was changed, one was already correct, and one could not be read. +

+ +

Exercise 3: A safe delete

+

+ A tool replaces each include.css link in an HTML document with an inline style block, then deletes include.css from the directory. Describe the run in which this breaks a document, and the check that prevents it. +

+ + + + diff --git a/release/python3/Man_In_Grey.py b/release/python3/Man_In_Grey.py deleted file mode 100755 index 22853b6..0000000 --- a/release/python3/Man_In_Grey.py +++ /dev/null @@ -1,346 +0,0 @@ -#!/usr/bin/env -S python3 -B -""" -Man_In_Grey.py — orchestration entrypoint - -Phase 0 (bootstrap): - - Ensure acceptance filter exists (create default in CWD if --input_acceptance omitted) - - Validate --stage - - If --phase-0-then-stop: exit here (no scan ,no execution) - -Phase 1 (outer): - - Discover every file under --stage; acceptance filter decides which to include - - Execute each config’s configure(prov ,planner ,WriteFileMeta) into ONE Planner - - Optionally print the planner; optionally stop - -Phase 2 (apply): - - Encode plan to CBOR - - Prefer piping CBOR to privileged gasket at release//man_in_grey_apply - - Else fall back to release/python3/executor_inner.py --plan - -""" - -from __future__ import annotations - -# no bytecode anywhere -import sys ,os -sys.dont_write_bytecode = True -os.environ.setdefault("PYTHONDONTWRITEBYTECODE" ,"1") - -from pathlib import Path -import argparse -import getpass -import runpy -import subprocess -import datetime as _dt -import platform -import stat as _stat - -# Local model types (Planner.py beside this file during dev; in release it’s also shipped) -from Planner import ( - Planner - ,PlanProvenance - ,WriteFileMeta - ,Journal - ,Command -) - -# ---------- constants ---------- - -DEFAULT_FILTER_FILENAME = "Man_In_Grey_input_acceptance.py" - -DEFAULT_FILTER_SOURCE = """# Man_In_Grey acceptance filter (default template) -# Return True to include a config file ,False to skip it. -# You receive a PlanProvenance object named `prov`. -# -# Common fields: -# prov.stage_root_dpath : Path -# prov.config_abs_fpath : Path -# prov.config_rel_fpath : Path -# prov.read_dir_dpath : Path -# prov.read_fname : str -# -# 1) Accept everything (default): -# def accept(prov): -# return True -# -# 2) Only a namespace: -# def accept(prov): -# return prov.config_rel_fpath.as_posix().startswith("dns/") -# -# 3) Exclude editor junk: -# def accept(prov): -# r = prov.config_rel_fpath.as_posix() -# return not (r.endswith("~") or r.endswith(".swp")) -# -def accept(prov): - return True -""" - -# ---------- small utils ---------- - -def iso_utc_now_str()-> str: - return _dt.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") - -def _repo_root_from(start: Path)-> Path|None: - cur = start.resolve() - for p in (cur ,*cur.parents): - if (p/"release").is_dir(): - return p - return None - -def _norm_arch_name()-> str: - m = (platform.machine() or "").lower() - table = { - "amd64": "x86_64" - ,"x64": "x86_64" - ,"x86_64": "x86_64" - ,"i386": "i686" - ,"i486": "i686" - ,"i586": "i686" - ,"i686": "i686" - ,"arm64": "aarch64" - ,"aarch64": "aarch64" - ,"armv7l": "armv7l" - ,"armv6l": "armv6l" - ,"riscv64": "riscv64" - ,"ppc64le": "ppc64le" - ,"powerpc64le": "ppc64le" - ,"s390x": "s390x" - } - return table.get(m ,m or "unknown") - -def _ensure_filter_file(filter_arg: str|None)-> Path: - if filter_arg: - p = Path(filter_arg) - if not p.is_file(): - raise RuntimeError(f"--input_acceptance file not found: {p}") - return p - p = Path.cwd()/DEFAULT_FILTER_FILENAME - if not p.exists(): - try: - p.write_text(DEFAULT_FILTER_SOURCE ,encoding="utf-8") - print(f"(created default filter at {p})") - except Exception as e: - raise RuntimeError(f"failed to create default filter {p}: {e}") - return p - -def _load_accept_func(filter_path: Path): - env = runpy.run_path(str(filter_path)) - fn = env.get("accept") - if not callable(fn): - raise RuntimeError(f"{filter_path}: missing callable 'accept(prov)'") - return fn - -def _walk_all_files(stage_root: Path): - root = stage_root.resolve() - for dirpath ,dirnames ,filenames in os.walk(root ,followlinks=False): - # prune symlinked dirs - dirnames[:] = [d for d in dirnames if not os.path.islink(os.path.join(dirpath ,d))] - for fname in filenames: - p = Path(dirpath ,fname) - try: - st = p.lstat() - if _stat.S_ISREG(st.st_mode) or _stat.S_ISLNK(st.st_mode): - yield p.resolve() - except Exception: - continue - -def find_config_paths(stage_root: Path ,accept_func)-> list[Path]: - out: list[tuple[int ,str ,Path]] = [] - root = stage_root.resolve() - for p in _walk_all_files(stage_root): - prov = PlanProvenance(stage_root=stage_root ,config_path=p) - try: - if accept_func(prov): - rel = p.resolve().relative_to(root) - out.append((len(rel.parts) ,rel.as_posix() ,p.resolve())) - except Exception as e: - raise RuntimeError(f"accept() failed on {prov.config_rel_fpath.as_posix()}: {e}") - out.sort(key=lambda t: (t[0] ,t[1])) # breadth-first ,then lexicographic - return [t[2] for t in out] - -def _run_all_configs_into_single_planner(stage_root: Path ,cfgs: list[Path])-> Planner: - agg = PlanProvenance(stage_root=stage_root ,config_path=stage_root/"(aggregate).py") - planner = Planner(provenance=agg) - for cfg in cfgs: - prov = PlanProvenance(stage_root=stage_root ,config_path=cfg) - # requires Planner.set_provenance(prov) (already added previously) - planner.set_provenance(prov) - env = runpy.run_path(str(cfg)) - fn = env.get("configure") - if not callable(fn): - raise RuntimeError(f"{cfg}: missing callable configure(prov ,planner ,WriteFileMeta)") - fn(prov ,planner ,WriteFileMeta) - j = planner.journal() - j.set_meta( - generator_prog_str="Man_In_Grey.py" - ,generated_at_utc_str=iso_utc_now_str() - ,user_name_str=getpass.getuser() - ,host_name_str=os.uname().nodename if hasattr(os ,"uname") else "unknown" - ,stage_root_dpath_str=str(stage_root.resolve()) - ,configs_list=[str(p.resolve().relative_to(stage_root.resolve())) for p in cfgs] - ) - return planner - -def _plan_to_cbor_bytes(planner: Planner)-> bytes: - try: - import cbor2 - except Exception as e: - raise RuntimeError(f"cbor2 is required: {e}") - return cbor2.dumps(planner.journal().as_dictionary() ,canonical=True) - -# ---------- apply paths ---------- - -def _find_apply_cmd(repo_root: Path)-> Path|None: - arch = _norm_arch_name() - cand = repo_root/"release"/arch/"man_in_grey_apply" - if cand.exists() and os.access(cand ,os.X_OK): - return cand - return None - -def _find_inner_py(repo_root: Path)-> Path|None: - cand = repo_root/"release"/"python3"/"executor_inner.py" - return cand if cand.is_file() else None - -def _apply_via_gasket(cbor_bytes: bytes, apply_cmd: Path, args) -> int: - cmd = [str(apply_cmd), "--plan", "-"] # <— tell gasket to read from stdin - if args.phase_2_print: cmd.append("--phase-2-print") - if args.phase_2_then_stop: cmd.append("--phase-2-then-stop") - if args.phase_2_wellformed_then_stop: cmd.append("--phase-2-wellformed-then-stop") - if args.phase_2_sanity1_then_stop: cmd.append("--phase-2-sanity1-then-stop") - if args.phase_2_validity_then_stop: cmd.append("--phase-2-validity-then-stop") - if args.phase_2_sanity2_then_stop: cmd.append("--phase-2-sanity2-then-stop") - proc = subprocess.run(cmd, input=cbor_bytes) - return proc.returncode - -def _apply_via_inner_py(cbor_bytes: bytes ,inner_py: Path ,args)-> int: - cmd = [ - sys.executable - ,str(inner_py) - ,"--plan" ,"-" - ] - if args.phase_2_print: cmd.append("--phase-2-print") - if args.phase_2_then_stop: cmd.append("--phase-2-then-stop") - if args.phase_2_wellformed_then_stop: cmd.append("--phase-2-wellformed-then-stop") - if args.phase_2_sanity1_then_stop: cmd.append("--phase-2-sanity1-then-stop") - if args.phase_2_validity_then_stop: cmd.append("--phase-2-validity-then-stop") - if args.phase_2_sanity2_then_stop: cmd.append("--phase-2-sanity2-then-stop") - proc = subprocess.run(cmd ,input=cbor_bytes) - return proc.returncode - -# ---------- CLI / orchestration ---------- - -def main(argv: list[str]|None=None)-> int: - ap = argparse.ArgumentParser( - prog="Man_In_Grey.py" - ,description="Man_In_Grey orchestrator (discover → plan → CBOR → apply)" - ) - ap.add_argument("--stage" ,default="stage" - ,help="stage root directory (default: ./stage)") - ap.add_argument("--input_acceptance" ,default="" - ,help=f"path to acceptance filter exporting accept(prov) " - f"(default: ./{DEFAULT_FILTER_FILENAME}; created if missing)") - ap.add_argument("--phase-0-then-stop" ,action="store_true" - ,help="stop after arg checks & filter bootstrap (no stage scan)") - # Phase-1 controls - ap.add_argument("--phase-1-print" ,action="store_true" - ,help="print master planner (phase 1)") - ap.add_argument("--phase-1-then-stop" ,action="store_true" - ,help="stop after phase 1") - # Phase-2 controls (forwarded to gasket/inner) - ap.add_argument("--phase-2-print" ,action="store_true" - ,help="print decoded journal (phase 2)") - ap.add_argument("--phase-2-then-stop" ,action="store_true" - ,help="stop after phase 2 decode") - ap.add_argument("--phase-2-wellformed-then-stop" ,action="store_true") - ap.add_argument("--phase-2-sanity1-then-stop" ,action="store_true") - ap.add_argument("--phase-2-validity-then-stop" ,action="store_true") - ap.add_argument("--phase-2-sanity2-then-stop" ,action="store_true") - # Optional explicit paths - ap.add_argument("--apply-cmd" ,default="" - ,help="override path to privileged gasket (pipes CBOR to stdin)") - ap.add_argument("--inner-py" ,default="" - ,help="override path to executor_inner.py (fallback path)") - - args = ap.parse_args(argv) - - # Repo root - repo_root = _repo_root_from(Path.cwd()) or _repo_root_from(Path(__file__).resolve()) or Path.cwd() - - # Phase 0 bootstrap - stage_root = Path(args.stage) - try: - filter_path = _ensure_filter_file(args.input_acceptance or None) - except Exception as e: - print(f"error: {e}" ,file=sys.stderr) - return 2 - - if not stage_root.exists(): - print(f"error: --stage not found: {stage_root}" ,file=sys.stderr) - return 2 - if not stage_root.is_dir(): - print(f"error: --stage is not a directory: {stage_root}" ,file=sys.stderr) - return 2 - - if args.phase_0_then_stop: - print(f"phase-0 OK: stage at {stage_root.resolve()} ,filter at {filter_path}") - return 0 - - # Acceptance - try: - accept_func = _load_accept_func(filter_path) - except Exception as e: - print(f"error: {e}" ,file=sys.stderr) - return 2 - - # Phase 1: discover + plan - cfgs = find_config_paths(stage_root ,accept_func) - if not cfgs: - print("No configuration files found.") - return 0 - - try: - master = _run_all_configs_into_single_planner(stage_root ,cfgs) - except SystemExit: - raise - except Exception as e: - print(f"error: executing configs: {e}" ,file=sys.stderr) - return 2 - - if args.phase_1_print: - master.print() - if args.phase_1_then_stop: - return 0 - - # Phase 2: encode + apply - try: - cbor_bytes = _plan_to_cbor_bytes(master) - except Exception as e: - print(f"error: CBOR encode failed: {e}" ,file=sys.stderr) - return 2 - - # Always use the gasket under release//man_in_grey_apply (or explicit --apply-cmd) - apply_cmd = Path(args.apply_cmd).resolve() if args.apply_cmd else (_find_apply_cmd(repo_root) or None) - if not apply_cmd: - print("error: gasket not found; build/release first (release//man_in_grey_apply)", file=sys.stderr) - return 2 - - try: - return _apply_via_gasket(cbor_bytes ,apply_cmd ,args) - except Exception as e: - print(f"error: apply-cmd failed: {e}" ,file=sys.stderr) - return 2 - - inner_py = Path(args.inner_py).resolve() if args.inner_py else (_find_inner_py(repo_root) or None) - if inner_py: - try: - return _apply_via_inner_py(cbor_bytes ,inner_py ,args) - except Exception as e: - print(f"error: inner executor failed: {e}" ,file=sys.stderr) - return 2 - - print("error: no apply path found (neither gasket nor inner Python)", file=sys.stderr) - return 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/release/python3/Planner.py b/release/python3/Planner.py deleted file mode 100644 index 94d8226..0000000 --- a/release/python3/Planner.py +++ /dev/null @@ -1,515 +0,0 @@ -#!/usr/bin/env -S python3 -B -""" -Planner.py — plan builder for staged configuration (UNPRIVILEGED). - -Given: runner-side provenance (PlanProvenance) and optional defaults (WriteFileMeta). -Does: expose Planner whose command methods (copy/displace/delete) build Command entries, - resolving arguments with precedence: kwarg > per-call WriteFileMeta > planner default - (and for filename, fallback to provenance-derived basename). On any argument error, - the Command is returned with errors and NOT appended to the Journal. -Returns: Journal (model only; dict in/out) via planner.journal(). -""" - -from __future__ import annotations - -# no bytecode anywhere (works under sudo/root shells too) -import sys ,os -sys.dont_write_bytecode = True -os.environ.setdefault("PYTHONDONTWRITEBYTECODE" ,"1") - -from pathlib import Path -import getpass - -# ===== Utilities ===== - -def norm_perm(value: int|str)-> tuple[int ,str]|None: - "Given int or 3/4-char octal string (optionally 0o-prefixed). Does validate/normalize. Returns (int ,'%04o') or None." - if isinstance(value ,int): - if 0 <= value <= 0o7777: - return value ,f"{value:04o}" - return None - if isinstance(value ,str): - s = value.strip().lower() - if s.startswith("0o"): - try: - v = int(s ,8) - return v ,f"{v:04o}" - except Exception: - return None - if len(s) in (3 ,4) and all(ch in "01234567" for ch in s): - try: - v = int(s ,8) - return v ,f"{v:04o}" - except Exception: - return None - return None - -def is_abs_dpath(dpath_str: str|None)-> bool: - "Given path string. Does quick abs dir check. Returns bool." - return isinstance(dpath_str ,str) and dpath_str.startswith("/") and "\x00" not in dpath_str - -def norm_abs_dpath_str(value: str|Path|None)-> str|None: - "Given str/Path/None. Does normalize absolute dir path string. Returns str or None." - if value is None: return None - s = value.as_posix() if isinstance(value ,Path) else str(value) - return s if is_abs_dpath(s) else None - -def norm_dpath_str(value: str|Path|None)-> str|None: - "Given str/Path/None. Does minimal sanitize; allows relative. Returns str or None." - if value is None: return None - s = value.as_posix() if isinstance(value ,Path) else str(value) - if not s or "\x00" in s: return None - return s - -def norm_fname_or_none(value: str|None)-> str|None: - "Given candidate filename or None. Does validate bare filename. Returns str or None." - if value is None: return None - s = str(value) - if not s: return None - if "/" in s or s in ("." ,"..") or "\x00" in s: return None - return s - -def norm_nonempty_owner(value: str|None)-> str|None: - "Given owner string or None. Does minimally validate (non-empty). Returns str or None." - if value is None: return None - s = str(value).strip() - return s if s else None - -def parse_mode(value: int|str|None)-> tuple[int|None ,str|None]: - "Given int/str/None. Does normalize via norm_perm. Returns (int ,'%04o') or (None ,None)." - if value is None: return None ,None - r = norm_perm(value) - return r if r is not None else (None ,None) - -def norm_content_bytes(value: bytes|str|None)-> bytes|None: - "Given bytes/str/None. Does normalize to UTF-8 bytes or None. Returns bytes|None." - if value is None: return None - if isinstance(value ,bytes): return value - return value.encode("utf-8") - -# ===== Wire-ready model types (no CBOR here) ===== - -class Command: - """ - Command — a single planned operation. - - Given name_str ('copy'|'displace'|'delete'), optional arg_dict, optional errors_list. - Does hold op name, own a fresh arg_dict, collect per-entry errors. - Returns dictionary via as_dictionary(). - """ - __slots__ = ( - "name_str" - ,"arg_dict" - ,"errors_list" - ) - - def __init__(self ,name_str: str ,arg_dict: dict|None=None ,errors_list: list[str]|None=None)-> None: - self.name_str = name_str - self.arg_dict = dict(arg_dict) if arg_dict is not None else {} - self.errors_list = list(errors_list) if errors_list is not None else [] - - def add_error(self ,msg_str: str)-> None: - self.errors_list.append(msg_str) - - def as_dictionary(self)-> dict: - return { - "op": self.name_str - ,"arg_dict": dict(self.arg_dict) - ,"errors_list": list(self.errors_list) - } - - def print(self ,* ,index: int|None=None ,file=None)-> None: - """ - Given: optional index for numbering and optional file-like (defaults to stdout). - Does: print a compact, human-readable one-line summary of this command; prints any errors indented below. - Returns: None. - """ - if file is None: - import sys as _sys - file = _sys.stdout - - op = self.name_str - ad = self.arg_dict or {} - - # Compose destination path for display (normalize to collapse '..') - d = ad.get("write_file_dpath_str") or "" - f = ad.get("write_file_fname") or "" - try: - from pathlib import Path as _Path - if d and f and "/" not in f: - dst = (_Path(d)/f).resolve().as_posix() - else: - dst = "?" - except Exception: - dst = "?" - - prefix = f"{index:02d}. " if index is not None else "" - - if op == "copy": - mode = ad.get("mode_int") - owner = ad.get("owner_name") - size = len(ad.get("content_bytes") or b"") - line = f"{prefix}copy -> {dst} mode {mode:04o} owner {owner} bytes {size}" - elif op == "displace": - line = f"{prefix}displace -> {dst}" - elif op == "delete": - line = f"{prefix}delete -> {dst}" - else: - line = f"{prefix}?op? -> {dst}" - - print(line ,file=file) - - for err in self.errors_list: - print(f" ! {err}" ,file=file) - -class Journal: - """ - Journal — ordered list of Command plus provenance metadata (model only; no CBOR). - - Given optional plan_dict in wire shape (for reconstruction). - Does manage meta, append commands, expose entries, and pack to dict. - Returns dict via as_dictionary(). - """ - __slots__ = ( - "meta_dict" - ,"command_list" - ) - - def __init__(self ,plan_dict: dict|None=None)-> None: - self.meta_dict = {} - self.command_list = [] - if plan_dict is not None: - self._init_from_dict(plan_dict) - - def _init_from_dict(self ,plan_dict: dict)-> None: - if not isinstance(plan_dict ,dict): - raise ValueError("plan_dict must be a dict") - meta = dict(plan_dict.get("meta_dict") or {}) - entries = plan_dict.get("entries_list") or [] - self.meta_dict.update(meta) - for e in entries: - if not isinstance(e ,dict): - continue - op = e.get("op") or "?" - args = e.get("arg_dict") or {} - errs = e.get("errors_list") or [] - self.command_list.append(Command(name_str=op ,arg_dict=dict(args) ,errors_list=list(errs))) - - def set_meta(self ,**kv)-> None: - self.meta_dict.update(kv) - - def append(self ,cmd: Command)-> None: - self.command_list.append(cmd) - - def entries_list(self)-> list[dict]: - return [c.as_dictionary() for c in self.command_list] - - def as_dictionary(self)-> dict: - return { - "version_int": 1 - ,"meta_dict": dict(self.meta_dict) - ,"entries_list": self.entries_list() - } - - def print(self ,* ,index_start: int=1 ,file=None)-> None: - """ - Given: optional starting index and optional file-like (defaults to stdout). - Does: print each Command on a single line via Command.print(), numbered. - Returns: None. - """ - if file is None: - import sys as _sys - file = _sys.stdout - - if not self.command_list: - print("(plan is empty)" ,file=file) - return - - for i ,cmd in enumerate(self.command_list ,start=index_start): - cmd.print(index=i ,file=file) - -# ===== Runner-provided provenance ===== - -class PlanProvenance: - """ - Runner-provided, read-only provenance for a single config script. - """ - __slots__ = ( - "stage_root_dpath" - ,"config_abs_fpath" - ,"config_rel_fpath" - ,"read_dir_dpath" - ,"read_fname" - ,"process_user" - ,"cwd_dpath" - ) - - def __init__(self ,* ,stage_root: Path ,config_path: Path): - self.stage_root_dpath = stage_root.resolve() - self.config_abs_fpath = config_path.resolve() - try: - self.config_rel_fpath = self.config_abs_fpath.relative_to(self.stage_root_dpath) - except Exception: - self.config_rel_fpath = Path(self.config_abs_fpath.name) - - self.read_dir_dpath = self.config_abs_fpath.parent - - name = self.config_abs_fpath.name - if name.endswith(".stage.py"): - self.read_fname = name[:-len(".stage.py")] - elif name.endswith(".py"): - self.read_fname = name[:-3] - else: - self.read_fname = name - - self.process_user = getpass.getuser() - self.cwd_dpath = Path.cwd().resolve() - - def print(self ,* ,file=None)-> None: - if file is None: - import sys as _sys - file = _sys.stdout - print(f"Stage root: {self.stage_root_dpath}" ,file=file) - print(f"Config (rel): {self.config_rel_fpath.as_posix()}" ,file=file) - print(f"Config (abs): {self.config_abs_fpath}" ,file=file) - print(f"Read dir: {self.read_dir_dpath}" ,file=file) - print(f"Read fname: {self.read_fname}" ,file=file) - print(f"Process user: {self.process_user}" ,file=file) - -# ===== Admin-facing defaults carrier ===== - -class WriteFileMeta: - """ - WriteFileMeta — per-call or planner-default write-file attributes. - - Given dpath (str/Path, may be relative) ,fname (bare name or None) ,owner (str) - ,mode (int|'0644') ,content (bytes|str|None). - Does normalize into fields (may remain None if absent/invalid). - Returns object suitable for providing defaults to Planner methods. - """ - __slots__ = ( - "dpath_str" - ,"fname" - ,"owner_name_str" - ,"mode_int" - ,"mode_octal_str" - ,"content_bytes" - ) - - def __init__(self - ,* - ,dpath="/" - ,fname=None - ,owner="root" - ,mode=0o444 - ,content=None - ): - self.dpath_str = norm_dpath_str(dpath) - self.fname = norm_fname_or_none(fname) - self.owner_name_str = norm_nonempty_owner(owner) - self.mode_int ,self.mode_octal_str = parse_mode(mode) - self.content_bytes = norm_content_bytes(content) - - def print(self ,* ,label: str|None=None ,file=None)-> None: - if file is None: - import sys as _sys - file = _sys.stdout - dpath = self.dpath_str or "?" - fname = self.fname or "?" - owner = self.owner_name_str or "?" - mode_str = f"{self.mode_int:04o}" if isinstance(self.mode_int ,int) else (self.mode_octal_str or "?") - size = len(self.content_bytes) if isinstance(self.content_bytes ,(bytes ,bytearray)) else 0 - prefix = (label + ": ") if label else "" - print(f"{prefix}dpath={dpath} fname={fname} owner={owner} mode={mode_str} bytes={size}" ,file=file) - -# ===== Planner ===== - -class Planner: - """ - Planner — constructs a Journal of Commands from config scripts. - - Given provenance (PlanProvenance) and optional default WriteFileMeta. - Does resolve command parameters by precedence: kwarg > per-call WriteFileMeta > planner default, - with a final filename fallback to provenance basename if still missing. - On any argument error, returns the Command with errors and DOES NOT append it to Journal. - Returns live Journal via journal(). - """ - __slots__ = ( - "_prov" - ,"_defaults" - ,"_journal" - ) - - def __init__(self ,provenance: PlanProvenance ,defaults: WriteFileMeta|None=None)-> None: - self._prov = provenance - self._defaults = defaults if defaults is not None else WriteFileMeta( - dpath="/" - ,fname=provenance.read_fname - ,owner="root" - ,mode=0o444 - ,content=None - ) - self._journal = Journal() - self._journal.set_meta( - stage_root_dpath_str=str(self._prov.stage_root_dpath) - ,config_rel_fpath_str=self._prov.config_rel_fpath.as_posix() - ) - - # --- provenance/defaults/journal access --- - - def set_provenance(self ,prov: PlanProvenance)-> None: - self._prov = prov - - def set_defaults(self ,defaults: WriteFileMeta)-> None: - self._defaults = defaults - - def defaults(self)-> WriteFileMeta: - return self._defaults - - def journal(self)-> Journal: - return self._journal - - # --- resolution helpers --- - - def _pick(self ,kw ,meta_attr ,default_attr): - "Pick first non-None among kw ,meta_attr ,default_attr." - return kw if kw is not None else (meta_attr if meta_attr is not None else default_attr) - - def _resolve_write_file(self ,wfm ,dpath ,fname)-> tuple[str|None ,str|None]: - # normalize explicit kwargs - dpath_str = norm_dpath_str(dpath) if dpath is not None else None - fname_str = norm_fname_or_none(fname) if fname is not None else None - - # precedence: kwarg > per-call meta > planner default - dpath_val = self._pick(dpath_str ,(wfm.dpath_str if wfm else None) ,self._defaults.dpath_str) - fname_val = self._pick(fname_str ,(wfm.fname if wfm else None) ,self._defaults.fname) - - # final fallback for filename: derive from config name - if fname_val is None: - fname_val = self._prov.read_fname - - # anchor/normalize dpath - if dpath_val is not None: - p = Path(dpath_val) - if not p.is_absolute(): - p = (self._prov.cwd_dpath/p) - dpath_val = p.resolve().as_posix() - - return dpath_val ,fname_val - - def _resolve_owner_mode_content(self - ,wfm: WriteFileMeta|None - ,owner: str|None - ,mode: int|str|None - ,content: bytes|str|None - )-> tuple[str|None ,tuple[int|None ,str|None] ,bytes|None]: - owner_norm = norm_nonempty_owner(owner) if owner is not None else None - mode_norm = parse_mode(mode) if mode is not None else (None ,None) - content_b = norm_content_bytes(content) if content is not None else None - - owner_v = self._pick(owner_norm ,(wfm.owner_name_str if wfm else None) ,self._defaults.owner_name_str) - mode_v = (mode_norm if mode_norm != (None ,None) else - ((wfm.mode_int ,wfm.mode_octal_str) if wfm else (self._defaults.mode_int ,self._defaults.mode_octal_str))) - content_v = self._pick(content_b ,(wfm.content_bytes if wfm else None) ,self._defaults.content_bytes) - return owner_v ,mode_v ,content_v - - # --- printing --- - - def print(self ,* ,show_journal: bool=True ,file=None)-> None: - if file is None: - import sys as _sys - file = _sys.stdout - - print("== Provenance ==" ,file=file) - self._prov.print(file=file) - - print("\n== Defaults ==" ,file=file) - self._defaults.print(label="defaults" ,file=file) - - if show_journal: - entries = getattr(self._journal ,"command_list" ,[]) - n_total = len(entries) - n_copy = sum(1 for c in entries if getattr(c ,"name_str" ,None) == "copy") - n_disp = sum(1 for c in entries if getattr(c ,"name_str" ,None) == "displace") - n_del = sum(1 for c in entries if getattr(c ,"name_str" ,None) == "delete") - - print("\n== Journal ==" ,file=file) - print(f"entries: {n_total} copy:{n_copy} displace:{n_disp} delete:{n_del}" ,file=file) - if n_total: - self._journal.print(index_start=1 ,file=file) - else: - print("(plan is empty)" ,file=file) - - # --- Command builders (first arg may be WriteFileMeta) --- - - def copy(self - ,wfm: WriteFileMeta|None=None - ,* - ,write_file_dpath: str|Path|None=None - ,write_file_fname: str|None=None - ,owner: str|None=None - ,mode: int|str|None=None - ,content: bytes|str|None=None - )-> Command: - cmd = Command("copy") - dpath ,fname = self._resolve_write_file(wfm ,write_file_dpath ,write_file_fname) - owner_v ,(mode_int ,mode_oct) ,content_b = self._resolve_owner_mode_content(wfm ,owner ,mode ,content) - - # well-formed checks - if not is_abs_dpath(dpath): cmd.add_error("write_file_dpath must be absolute") - if norm_fname_or_none(fname) is None: cmd.add_error("write_file_fname must be a bare filename") - if not owner_v: cmd.add_error("owner must be non-empty") - if (mode_int ,mode_oct) == (None ,None): - cmd.add_error("mode must be int <= 0o7777 or 3/4-digit octal string") - if content_b is None: - cmd.add_error("content is required for copy() (bytes or str)") - - cmd.arg_dict.update({ - "write_file_dpath_str": dpath - ,"write_file_fname": fname - ,"owner_name": owner_v - ,"mode_int": mode_int - ,"mode_octal_str": mode_oct - ,"content_bytes": content_b - ,"provenance_config_rel_fpath_str": self._prov.config_rel_fpath.as_posix() - }) - - if not cmd.errors_list: - self._journal.append(cmd) - return cmd - - def displace(self - ,wfm: WriteFileMeta|None=None - ,* - ,write_file_dpath: str|Path|None=None - ,write_file_fname: str|None=None - )-> Command: - cmd = Command("displace") - dpath ,fname = self._resolve_write_file(wfm ,write_file_dpath ,write_file_fname) - if not is_abs_dpath(dpath): cmd.add_error("write_file_dpath must be absolute") - if norm_fname_or_none(fname) is None: cmd.add_error("write_file_fname must be a bare filename") - cmd.arg_dict.update({ - "write_file_dpath_str": dpath - ,"write_file_fname": fname - }) - if not cmd.errors_list: - self._journal.append(cmd) - return cmd - - def delete(self - ,wfm: WriteFileMeta|None=None - ,* - ,write_file_dpath: str|Path|None=None - ,write_file_fname: str|None=None - )-> Command: - cmd = Command("delete") - dpath ,fname = self._resolve_write_file(wfm ,write_file_dpath ,write_file_fname) - if not is_abs_dpath(dpath): cmd.add_error("write_file_dpath must be absolute") - if norm_fname_or_none(fname) is None: cmd.add_error("write_file_fname must be a bare filename") - cmd.arg_dict.update({ - "write_file_dpath_str": dpath - ,"write_file_fname": fname - }) - if not cmd.errors_list: - self._journal.append(cmd) - return cmd diff --git a/release/python3/executor_inner.py b/release/python3/executor_inner.py deleted file mode 100755 index eb6bf5c..0000000 --- a/release/python3/executor_inner.py +++ /dev/null @@ -1,473 +0,0 @@ -#!/usr/bin/env -S python3 -B -""" -executor_inner.py — Man_In_Gray phase-2 inner executor - -- Reads a CBOR plan file (--plan) -- Decodes to Journal (via Planner.py model) -- Optional checkpoints: - wellformed → sanity-1 → validity → sanity-2 → execute -- Default behavior (no stop flags): apply the journal -""" - -from __future__ import annotations - -# no bytecode anywhere -import sys ,os -sys.dont_write_bytecode = True -os.environ.setdefault("PYTHONDONTWRITEBYTECODE" ,"1") - -from pathlib import Path -import argparse -import pwd -import stat as _stat - -# Journal model comes from the same directory's Planner.py -from Planner import ( - Journal, -) - -# -- helpers -- - -def _realpath(p: str|Path)-> Path: - "Resolve as much as possible without requiring target leaf to exist." - return Path(os.path.realpath(str(p))) - -def _is_under(child: Path ,root: Path)-> bool: - "True if child is the same as or within root (after realpath)." - try: - child_r = _realpath(child) - root_r = _realpath(root) - # Python <3.9 compat for is_relative_to: - child_parts = child_r.as_posix().rstrip("/") + "/" - root_parts = root_r.as_posix().rstrip("/") + "/" - return child_parts.startswith(root_parts) - except Exception: - return False - -# --- CBOR load --- - -def _journal_from_cbor_bytes(data: bytes)-> Journal: - try: - import cbor2 - except Exception as e: - raise RuntimeError(f"cbor2 is required: {e}") - obj = cbor2.loads(data) - if not isinstance(obj ,dict): - raise ValueError("CBOR root must be a dict") - return Journal(plan_dict=obj) - -# --- pretty helpers --- - -def _dst_from(ad: dict)-> str: - d = ad.get("write_file_dpath_str") or "?" - f = ad.get("write_file_fname") or "?" - try: - from pathlib import Path as _P - if isinstance(d ,str) and isinstance(f ,str) and "/" not in f: - return (_P(d)/f).as_posix() - except Exception: - pass - return f"{d}/{f}" - -def _mode_from_entry(ad: dict)-> int: - m = ad.get("mode_int") - if isinstance(m ,int): return m - s = ad.get("mode_octal_str") - if isinstance(s ,str): - try: - return int(s ,8) - except Exception: - pass - raise ValueError("invalid mode") - -# --- Phase: wellformed (schema/shape) --- - -def check_wellformed(journal: Journal)-> list[str]: - errs: list[str] = [] - for i ,cmd in enumerate(journal.command_list ,start=1): - op = getattr(cmd ,"name_str" ,None) - ad = getattr(cmd ,"arg_dict" ,None) - if op not in {"copy" ,"displace" ,"delete"}: - errs.append(f"[{i}] unknown op: {op!r}") - continue - if not isinstance(ad ,dict): - errs.append(f"[{i}] arg_dict missing") - continue - d = ad.get("write_file_dpath_str") - f = ad.get("write_file_fname") - if not (isinstance(d ,str) and d.startswith("/")): - errs.append(f"[{i}] write_file_dpath_str must be absolute: {d!r}") - if not (isinstance(f ,str) and "/" not in f and f not in {"." ,""}): - errs.append(f"[{i}] write_file_fname must be a bare filename: {f!r}") - if op == "copy": - if "owner_name" not in ad: - errs.append(f"[{i}] copy: owner_name missing") - if "content_bytes" not in ad: - errs.append(f"[{i}] copy: content_bytes missing") - if "mode_int" not in ad and "mode_octal_str" not in ad: - errs.append(f"[{i}] copy: mode missing") - return errs - -# --- Phase: sanity-1 (cheap static sanity) --- - -def check_sanity_1(journal: Journal ,allowed_roots: list[Path])-> list[str]: - """ - Scope fence: every destination directory must be under at least one allowed root. - Default allowed roots = [/etc, cwd_of_inner]. - """ - errs: list[str] = [] - allowed_str = ", ".join(r.as_posix() for r in allowed_roots) - for i ,cmd in enumerate(journal.command_list ,start=1): - ad = cmd.arg_dict - d = ad.get("write_file_dpath_str") - if not isinstance(d ,str): - # wellformed will report it; skip here - continue - d_real = _realpath(d) - if not any(_is_under(d_real ,root) for root in allowed_roots): - errs.append(f"[{i}] dst dir outside allowed roots: {d_real.as_posix()} (allowed: {allowed_str})") - return errs - -# --- Phase: validity (system lookups) --- - -def check_validity(journal: Journal)-> list[str]: - errs: list[str] = [] - for i ,cmd in enumerate(journal.command_list ,start=1): - ad = cmd.arg_dict - if cmd.name_str == "copy": - owner = ad.get("owner_name") - try: - pwd.getpwnam(owner) - except Exception: - errs.append(f"[{i}] unknown owner_name: {owner!r} (dst={_dst_from(ad)})") - try: - _ = _mode_from_entry(ad) - except Exception as e: - errs.append(f"[{i}] bad mode: {e} (dst={_dst_from(ad)})") - cb = ad.get("content_bytes") - if not isinstance(cb ,(bytes ,bytearray)): - errs.append(f"[{i}] content_bytes not bytes-like (dst={_dst_from(ad)})") - return errs - -# --- Phase: sanity-2 (filesystem checks, no mutation) --- - -def _safe_open_dir(dpath: str)-> int: - fd = os.open(dpath ,os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) - st = os.fstat(fd) - if not _stat.S_ISDIR(st.st_mode): - os.close(fd) ; raise OSError("not a directory") - return fd - - - - -def check_sanity_2(journal: Journal)-> list[str]: - errs: list[str] = [] - opened: dict[str ,int] = {} - try: - # ensure destination directories are openable (and not symlinked dirs) - for i ,cmd in enumerate(journal.command_list ,start=1): - d = cmd.arg_dict.get("write_file_dpath_str") - if not isinstance(d ,str): # already flagged in wellformed - continue - if d in opened: - continue - try: - opened[d] = _safe_open_dir(d) - except Exception as e: - errs.append(f"[{i}] cannot open destination dir: {d} ({e})") - - # detect multiple writes to same target without a reset (displace/delete) - last_action: dict[tuple[str, str], str] = {} - for i, cmd in enumerate(journal.command_list, start=1): - ad = cmd.arg_dict - key = (ad.get("write_file_dpath_str"), ad.get("write_file_fname")) - op = cmd.name_str - if op == "copy": - if last_action.get(key) == "copy": - errs.append(f"[{i}] multiple writes to same target without prior displace/delete: {_dst_from(ad)}") - last_action[key] = "copy" - elif op in {"displace", "delete"}: - last_action[key] = op - - finally: - for fd in opened.values(): - try: - os.close(fd) - except Exception: - pass - return errs - -# --- Execute (mutation) --- - -def _fsync_dirfd(dirfd: int)-> None: - try: - os.fsync(dirfd) - except Exception: - pass - -def _exists_regular_nosymlink_at(dirfd: int ,fname: str)-> bool: - try: - st = os.lstat(fname ,dir_fd=dirfd) - except FileNotFoundError: - return False - if _stat.S_ISLNK(st.st_mode): raise OSError("target is a symlink") - if not _stat.S_ISREG(st.st_mode): raise OSError("target not a regular file") - return True - -def _apply_displace(d: str ,f: str)-> None: - dirfd = _safe_open_dir(d) - try: - if not _exists_regular_nosymlink_at(dirfd ,f): - return - import time as _time - ts = _time.strftime("%Y%m%dT%H%M%SZ" ,_time.gmtime()) - bak = f"{f}.{ts}" - os.rename(f ,bak ,src_dir_fd=dirfd ,dst_dir_fd=dirfd) - _fsync_dirfd(dirfd) - finally: - os.close(dirfd) - -def _apply_copy(d: str ,f: str ,owner: str ,mode_int: int ,content: bytes)-> None: - pw = pwd.getpwnam(owner) - uid ,gid = pw.pw_uid ,pw.pw_gid - dirfd = _safe_open_dir(d) - try: - tmp = f".{f}.mig.tmp.{os.getpid()}" - tfd = os.open(tmp ,os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW ,0o600 ,dir_fd=dirfd) - try: - mv = memoryview(content) - off = 0 - while off < len(mv): - n = os.write(tfd ,mv[off:]) - if n <= 0: raise OSError("short write") - off += n - os.fsync(tfd) - os.fchown(tfd ,uid ,gid) - os.fchmod(tfd ,mode_int) - os.fsync(tfd) - finally: - os.close(tfd) - os.rename(tmp ,f ,src_dir_fd=dirfd ,dst_dir_fd=dirfd) - _fsync_dirfd(dirfd) - finally: - os.close(dirfd) - -def _apply_delete(d: str ,f: str)-> None: - dirfd = _safe_open_dir(d) - try: - if not _exists_regular_nosymlink_at(dirfd ,f): - return - os.unlink(f ,dir_fd=dirfd) - _fsync_dirfd(dirfd) - finally: - os.close(dirfd) - -def apply_journal(journal: Journal)-> int: - errs = 0 - for idx ,entry in enumerate(journal.command_list ,start=1): - op = getattr(entry ,"name_str" ,"?") - ad = getattr(entry ,"arg_dict" ,{}) or {} - try: - d = ad["write_file_dpath_str"] - f = ad["write_file_fname"] - if not (isinstance(d ,str) and d.startswith("/") and isinstance(f ,str) and "/" not in f): - raise ValueError("bad path or filename") - if op == "displace": - _apply_displace(d ,f) - elif op == "copy": - owner = ad["owner_name"] - mode = _mode_from_entry(ad) - content = ad["content_bytes"] - if not isinstance(content ,(bytes ,bytearray)): raise ValueError("content_bytes missing") - _apply_copy(d ,f ,owner ,mode ,bytes(content)) - elif op == "delete": - _apply_delete(d ,f) - else: - raise ValueError(f"unknown op: {op}") - except Exception as e: - errs += 1 - print(f"apply error [{idx} {op}] {_dst_from(ad)}: {e}" ,file=sys.stderr) - return 0 if errs == 0 else 1 - -# --- Orchestration --- - -def _phase_gate(name: str ,errors: list[str] ,then_stop: bool)-> bool: - if errors: - print(f"{name}: {len(errors)} issue(s)") - for e in errors: - print(f" ! {e}") - return True - if then_stop: - print(f"{name}: OK") - return True - return False - -def executor_inner( - journal: Journal - ,* - ,phase_2_print: bool=False - ,phase_2_then_stop: bool=False - ,phase_2_wellformed_then_stop: bool=False - ,phase_2_sanity1_then_stop: bool=False - ,phase_2_validity_then_stop: bool=False - ,phase_2_sanity2_then_stop: bool=False - ,allowed_roots: list[Path]|None=None -)-> int: - """ - Core pipeline for the inner executor. Returns a process-style exit code. - """ - if phase_2_print: - journal.print() - if phase_2_then_stop: - return 0 - - roots = allowed_roots or [Path("/etc").resolve() ,Path.cwd().resolve()] - - wf = check_wellformed(journal) - if _phase_gate("wellformed" ,wf ,phase_2_wellformed_then_stop): - return 1 if wf else 0 if phase_2_wellformed_then_stop else 0 - - s1 = check_sanity_1(journal ,roots) - if _phase_gate("sanity-1" ,s1 ,phase_2_sanity1_then_stop): - return 1 if s1 else 0 if phase_2_sanity1_then_stop else 0 - - v = check_validity(journal) - if _phase_gate("validity" ,v ,phase_2_validity_then_stop): - return 1 if v else 0 if phase_2_validity_then_stop else 0 - - s2 = check_sanity_2(journal) - if _phase_gate("sanity-2" ,s2 ,phase_2_sanity2_then_stop): - return 1 if s2 else 0 if phase_2_sanity2_then_stop else 0 - - return apply_journal(journal) - -# --- CLI wrapper --- - -# --- new worker -------------------------------------------------------------- - -def run_executor_inner( - * - ,plan_bytes: bytes - ,phase2_print: bool - ,phase2_then_stop: bool - ,phase2_wellformed_then_stop: bool - ,phase2_sanity1_then_stop: bool - ,phase2_validity_then_stop: bool - ,phase2_sanity2_then_stop: bool -)-> int: - try: - journal = _journal_from_cbor_bytes(plan_bytes) - except Exception as e: - print(f"error: failed to decode CBOR: {e}" ,file=sys.stderr) - return 2 - - if phase2_print: - journal.print() - if phase2_then_stop: - return 0 - - allowed_roots = [Path("/etc").resolve() ,Path.cwd().resolve()] - - wf = check_wellformed(journal) - if _phase_gate("wellformed" ,wf ,phase2_wellformed_then_stop): return 1 if wf else 0 if phase2_wellformed_then_stop else 0 - - s1 = check_sanity_1(journal ,allowed_roots) - if _phase_gate("sanity-1" ,s1 ,phase2_sanity1_then_stop): return 1 if s1 else 0 if phase2_sanity1_then_stop else 0 - - v = check_validity(journal) - if _phase_gate("validity" ,v ,phase2_validity_then_stop): return 1 if v else 0 if phase2_validity_then_stop else 0 - - s2 = check_sanity_2(journal) - if _phase_gate("sanity-2" ,s2 ,phase2_sanity2_then_stop): return 1 if s2 else 0 if phase2_sanity2_then_stop else 0 - - return apply_journal(journal) - -# --- main stays a thin arg wrapper ------------------------------------------ - -# --- plan input helpers ------------------------------------------------------- - -def _read_fd_all(fd: int) -> bytes: - "Read all bytes from an already-open file descriptor without closing it." - chunks: list[bytes] = [] - while True: - try: - b = os.read(fd, 65536) - except InterruptedError: - continue - if not b: - break - chunks.append(b) - return b"".join(chunks) - -def _read_plan_bytes_from_args(args) -> bytes: - """ - Input priority: - 1) --plan-fd (gasket path; do not close fd) - 2) --plan - (stdin) - 3) --plan (read from filesystem) - """ - if getattr(args, "plan_fd", -1) is not None and args.plan_fd >= 0: - return _read_fd_all(args.plan_fd) - if args.plan in ("", "-"): - return sys.stdin.buffer.read() - return Path(args.plan).read_bytes() - -def main(argv: list[str] | None = None) -> int: - ap = argparse.ArgumentParser( - prog="executor_inner.py", - description="Man_In_Grey inner executor (decode → validate → apply)" - ) - - # Single --plan plus a hidden --plan-fd used by the gasket - ap.add_argument( - "--plan", - default="-", - help="path to CBOR plan file or '-' for stdin" - ) - ap.add_argument( - "--plan-fd", - type=int, - default=-1, - help=argparse.SUPPRESS - ) - - # phase-2 gates (same semantics as before) - ap.add_argument("--phase-2-print", action="store_true", help="print decoded journal") - ap.add_argument("--phase-2-then-stop", action="store_true", help="stop after print (no apply)") - ap.add_argument("--phase-2-wellformed-then-stop", action="store_true", help="stop after wellformed checks") - ap.add_argument("--phase-2-sanity1-then-stop", action="store_true", help="stop after sanity-1 checks") - ap.add_argument("--phase-2-validity-then-stop", action="store_true", help="stop after validity checks") - ap.add_argument("--phase-2-sanity2-then-stop", action="store_true", help="stop after sanity-2 checks") - - args = ap.parse_args(argv) - - # Read plan bytes from fd/stdin/file - try: - data = _read_plan_bytes_from_args(args) - except Exception as e: - print(f"error: failed to read plan: {e}", file=sys.stderr) - return 2 - - # Decode CBOR → Journal - try: - journal = _journal_from_cbor_bytes(data) - except Exception as e: - print(f"error: failed to decode CBOR: {e}", file=sys.stderr) - return 2 - - # Run the pipeline - return executor_inner( - journal, - phase_2_print=args.phase_2_print, - phase_2_then_stop=args.phase_2_then_stop, - phase_2_wellformed_then_stop=args.phase_2_wellformed_then_stop, - phase_2_sanity1_then_stop=args.phase_2_sanity1_then_stop, - phase_2_validity_then_stop=args.phase_2_validity_then_stop, - phase_2_sanity2_then_stop=args.phase_2_sanity2_then_stop, - ) - -if __name__ == "__main__": - sys.exit(main()) - - diff --git a/release/python3/executor_outer.py b/release/python3/executor_outer.py deleted file mode 100755 index 0b5e87b..0000000 --- a/release/python3/executor_outer.py +++ /dev/null @@ -1,487 +0,0 @@ -#!/usr/bin/env -S python3 -B -""" -executor.py — StageHand outer/inner executor (MVP; UNPRIVILEGED for now) - -Phase 0 (bootstrap): - - Ensure filter program exists (create default in CWD if --filter omitted) - - Validate --stage exists - - If --phase-0-then-stop: exit here (no scan, no execution) - -Phase 1 (outer): - - Discover every file under --stage; acceptance filter decides which to include - - Execute each config’s configure(prov ,planner ,WriteFileMeta) into ONE Planner - - Optionally print the planner; optionally stop - -Phase 2 (inner shim in same program for now; no privilege yet): - - Encode plan to CBOR and hand to inner path - - Inner decodes to a Journal and can print it -""" - -from __future__ import annotations - -# no bytecode anywhere -import sys ,os -sys.dont_write_bytecode = True -os.environ.setdefault("PYTHONDONTWRITEBYTECODE" ,"1") - -from pathlib import Path -import argparse -import getpass -import tempfile -import runpy -import subprocess -import datetime as _dt -import stat - -# Local module: Planner.py (same directory) -from Planner import ( - Planner - ,PlanProvenance - ,WriteFileMeta - ,Journal - ,Command -) - -# -------- default filter template (written to CWD when --input_acceptance not provided) -------- - -DEFAULT_FILTER_FILENAME = "Man_In_Gray_input_acceptance.py" - -DEFAULT_FILTER_SOURCE = """# Man_In_Gray_input_acceptance (default template) -# Return True to include a config file, False to skip it. -# You receive a PlanProvenance object named `prov`. -# -# prov fields commonly used here: -# prov.stage_root_dpath : Path → absolute path to the stage root -# prov.config_abs_fpath : Path → absolute path to the candidate file -# prov.config_rel_fpath : Path → path relative to the stage root -# prov.read_dir_dpath : Path → directory of the candidate file -# prov.read_fname : str → filename with trailing '.py' stripped (if present) -# -# Examples: -# -# 1) Accept everything (default behavior): -# def accept(prov): -# return True -# -# 2) Only accept configs in a 'dns/' namespace under the stage: -# def accept(prov): -# return prov.config_rel_fpath.as_posix().startswith("dns/") -# -# 3) Exclude editor backup files: -# def accept(prov): -# rel = prov.config_rel_fpath.as_posix() -# return not (rel.endswith("~") or rel.endswith(".swp")) -# -# 4) Only accept Python files + a few non-Python names: -# def accept(prov): -# name = prov.config_abs_fpath.name -# return name.endswith(".py") or name in {"hosts" ,"resolv.conf"} -# -# Choose ONE 'accept' definition. Below is the default: - -def accept(prov): - return True -""" - -# -------- utilities -------- - -def iso_utc_now_str()-> str: - return _dt.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") - -def _ensure_filter_file(filter_arg: str|None)-> Path: - """ - If --input_acceptance is provided, return that path (must exist). - Otherwise, create ./stagehand_filter.py in the CWD if missing (writing a helpful template), - and return its path. - """ - if filter_arg: - p = Path(filter_arg) - if not p.is_file(): - raise RuntimeError(f"--input_acceptance file not found: {p}") - return p - - p = Path.cwd()/DEFAULT_FILTER_FILENAME - if not p.exists(): - try: - p.write_text(DEFAULT_FILTER_SOURCE ,encoding="utf-8") - print(f"(created default filter at {p})") - except Exception as e: - raise RuntimeError(f"failed to create default filter {p}: {e}") - return p - -def _load_accept_func(filter_path: Path): - env = runpy.run_path(str(filter_path)) - fn = env.get("accept") - if not callable(fn): - raise RuntimeError(f"{filter_path}: missing callable 'accept(prov)'") - return fn - -def _walk_all_files(stage_root: Path): - """ - Yield every file (regular or symlink) under stage_root recursively. - We do not follow symlinked directories to avoid cycles. - """ - root = stage_root.resolve() - for dirpath ,dirnames ,filenames in os.walk(root ,followlinks=False): - # prune symlinked dirs (files can still be symlinks) - dirnames[:] = [d for d in dirnames if not os.path.islink(os.path.join(dirpath ,d))] - for fname in filenames: - p = Path(dirpath ,fname) - try: - st = p.lstat() - if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode): - yield p.resolve() - except Exception: - # unreadable/broken entries skipped - continue - -def find_config_paths(stage_root: Path ,accept_func)-> list[Path]: - """ - Return files accepted by the Python acceptance function: accept(prov) → True/False. - Ordered breadth-first by depth, then lexicographically by relative path. - """ - out: list[tuple[int ,str ,Path]] = [] - root = stage_root.resolve() - for p in _walk_all_files(stage_root): - prov = PlanProvenance(stage_root=stage_root ,config_path=p) - try: - if accept_func(prov): - rel = p.resolve().relative_to(root) - out.append((len(rel.parts) ,rel.as_posix() ,p.resolve())) - except Exception as e: - raise RuntimeError(f"accept() failed on {prov.config_rel_fpath.as_posix()}: {e}") - out.sort(key=lambda t: (t[0] ,t[1])) # (depth ,name) - return [t[2] for t in out] - -# --- run all configs into ONE planner --- - -def _run_all_configs_into_single_planner(stage_root: Path ,cfgs: list[Path])-> Planner: - """ - Create a single Planner and execute each config's configure(prov, planner, WriteFileMeta) - against it. Returns that single Planner containing the entire plan. - """ - # seed with synthetic provenance; we overwrite per config before execution - aggregate_prov = PlanProvenance(stage_root=stage_root ,config_path=stage_root/"(aggregate).py") - planner = Planner(provenance=aggregate_prov) - - for cfg in cfgs: - prov = PlanProvenance(stage_root=stage_root ,config_path=cfg) - planner.set_provenance(prov) - - env = runpy.run_path(str(cfg)) - fn = env.get("configure") - if not callable(fn): - raise RuntimeError(f"{cfg}: missing callable configure(prov ,planner ,WriteFileMeta)") - - fn(prov ,planner ,WriteFileMeta) - - # annotate meta once, on the single planner's journal - j = planner.journal() - j.set_meta( - generator_prog_str="executor.py" - ,generated_at_utc_str=iso_utc_now_str() - ,user_name_str=getpass.getuser() - ,host_name_str=os.uname().nodename if hasattr(os ,"uname") else "unknown" - ,stage_root_dpath_str=str(stage_root.resolve()) - ,configs_list=[str(p.resolve().relative_to(stage_root.resolve())) for p in cfgs] - ) - return planner - -# ----- CBOR “matchbox” (simple wrapper kept local to executor) ----- - -def _plan_to_cbor_bytes(planner: Planner)-> bytes: - "Serialize a Planner's Journal to CBOR bytes." - try: - import cbor2 - except Exception as e: - raise RuntimeError(f"cbor2 is required: {e}") - plan_dict = planner.journal().as_dictionary() - return cbor2.dumps(plan_dict ,canonical=True) - -def _journal_from_cbor_bytes(data: bytes)-> Journal: - "Rebuild a Journal from CBOR bytes." - try: - import cbor2 - except Exception as e: - raise RuntimeError(f"cbor2 is required: {e}") - obj = cbor2.loads(data) - if not isinstance(obj ,dict): - raise ValueError("CBOR root must be a dict") - return Journal(plan_dict=obj) - -# -------- inner executor (phase 2) -------- - -def _inner_main(plan_path: Path ,phase2_print: bool ,phase2_then_stop: bool)-> int: - "Inner executor path: decode CBOR → Journal; optionally print; (apply TBD)." - try: - data = Path(plan_path).read_bytes() - except Exception as e: - print(f"error: failed to read plan file: {e}" ,file=sys.stderr) - return 2 - - try: - journal = _journal_from_cbor_bytes(data) - except Exception as e: - print(f"error: failed to decode CBOR: {e}" ,file=sys.stderr) - return 2 - - if phase2_print: - journal.print() - - if phase2_then_stop: - return 0 - - # (Stage 3 apply would go here; omitted in MVP) - return 0 - -# -------- outer executor (phase 1 & handoff) -------- - -def _outer_main(stage_root: Path ,accept_func ,args)-> int: - if not stage_root.is_dir(): - print(f"error: --stage not a directory: {stage_root}" ,file=sys.stderr) - return 2 - - cfgs = find_config_paths(stage_root ,accept_func) - if not cfgs: - print("No configuration files found.") - return 0 - - try: - master = _run_all_configs_into_single_planner(stage_root ,cfgs) - except SystemExit: - raise - except Exception as e: - print(f"error: executing configs: {e}" ,file=sys.stderr) - return 2 - - if args.phase_1_print: - master.print() - - if args.phase_1_then_stop: - return 0 - - # Phase 2: encode CBOR and invoke inner path (same script, --inner) - try: - cbor_bytes = _plan_to_cbor_bytes(master) - except Exception as e: - print(f"error: CBOR encode failed: {e}" ,file=sys.stderr) - return 2 - - with tempfile.NamedTemporaryFile(prefix="stagehand_plan_" ,suffix=".cbor" ,delete=False) as tf: - tf.write(cbor_bytes) - plan_path = tf.name - - try: - cmd = [ - sys.executable - ,str(Path(__file__).resolve()) - ,"--inner" - ,"--plan" ,plan_path - ] - if args.phase_2_print: - cmd.append("--phase-2-print") - if args.phase_2_then_stop: - cmd.append("--phase-2-then-stop") - - proc = subprocess.run(cmd) - return proc.returncode - finally: - try: - os.unlink(plan_path) - except Exception: - pass - -# -------- CLI -------- - -def main(argv: list[str]|None=None)-> int: - ap = argparse.ArgumentParser( - prog="executor.py" - ,description="StageHand outer/inner executor (plan → CBOR → decode)." - ) - ap.add_argument("--stage" ,default="stage" - ,help="stage root directory (default: ./stage)") - ap.add_argument( - "--input_acceptance" - ,default="" - ,help=f"path to acceptance filter program exporting accept(prov) " - f"(default: ./{DEFAULT_FILTER_FILENAME}; created if missing)" - ) - ap.add_argument( - "--phase-0-then-stop" - ,action="store_true" - ,help="stop after arg checks & filter bootstrap (no stage scan)" - ) - - # Phase-1 (outer) controls - ap.add_argument("--phase-1-print" ,action="store_true" ,help="print master planner (phase 1)") - ap.add_argument("--phase-1-then-stop" ,action="store_true" ,help="stop after phase 1") - - # Phase-2 (inner) controls (outer forwards these to inner) - ap.add_argument("--phase-2-print" ,action="store_true" ,help="print decoded journal (phase 2)") - ap.add_argument("--phase-2-then-stop" ,action="store_true" ,help="stop after phase 2 decode") - - # Inner-only flags (not for users) - ap.add_argument("--inner" ,action="store_true" ,help=argparse.SUPPRESS) - ap.add_argument("--plan" ,default=None ,help=argparse.SUPPRESS) - - args = ap.parse_args(argv) - - # Inner path - if args.inner: - if not args.plan: - print("error: --inner requires --plan " ,file=sys.stderr) - return 2 - return _inner_main(Path(args.plan) - ,phase2_print=args.phase_2_print - ,phase2_then_stop=args.phase_2_then_stop) - - # Phase 0: bootstrap & stop (no scan) - stage_root = Path(args.stage) - try: - filter_path = _ensure_filter_file(args.input_acceptance or None) - except Exception as e: - print(f"error: {e}" ,file=sys.stderr) - return 2 - - if not stage_root.exists(): - print(f"error: --stage not found: {stage_root}" ,file=sys.stderr) - return 2 - if not stage_root.is_dir(): - print(f"error: --stage is not a directory: {stage_root}" ,file=sys.stderr) - return 2 - - if args.phase_0_then_stop: - print(f"phase-0 OK: stage at {stage_root.resolve()} and filter at {filter_path}") - return 0 - - # Load acceptance function and proceed with outer - try: - accept_func = _load_accept_func(filter_path) - except Exception as e: - print(f"error: {e}" ,file=sys.stderr) - return 2 - - return _outer_main(stage_root ,accept_func ,args) - -# inner executor -# --- secure apply helpers (inner path) --- - -import pwd ,errno ,stat as _stat - -def _safe_open_dir(dpath: str)-> int: - "Open directory without following symlinks; return dirfd." - fd = os.open(dpath ,os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) - st = os.fstat(fd) - if not _stat.S_ISDIR(st.st_mode): - os.close(fd) ; raise OSError("not a directory") - return fd - -def _exists_regular_nosymlink_at(dirfd: int ,fname: str)-> bool: - "True if a regular ,non-symlink file exists at dirfd/fname." - try: - st = os.lstat(fname ,dir_fd=dirfd) - except FileNotFoundError: - return False - if _stat.S_ISLNK(st.st_mode): raise OSError("target is a symlink") - if not _stat.S_ISREG(st.st_mode): raise OSError("target not a regular file") - return True - -def _fsync_dirfd(dirfd: int)-> None: - try: - os.fsync(dirfd) - except Exception: - pass # some FS may not support; best effort - -def _apply_displace(d: str ,f: str)-> None: - dirfd = _safe_open_dir(d) - try: - if not _exists_regular_nosymlink_at(dirfd ,f): - return - import time as _time - ts = _time.strftime("%Y%m%dT%H%M%SZ" ,_time.gmtime()) - bak = f"{f}.{ts}" - os.rename(f ,bak ,src_dir_fd=dirfd ,dst_dir_fd=dirfd) - _fsync_dirfd(dirfd) - finally: - os.close(dirfd) - -def _apply_copy(d: str ,f: str ,owner: str ,mode_int: int ,content: bytes)-> None: - pw = pwd.getpwnam(owner) - uid ,gid = pw.pw_uid ,pw.pw_gid - dirfd = _safe_open_dir(d) - try: - tmp = f".{f}.mig.tmp.{os.getpid()}" - tfd = os.open(tmp ,os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW ,0o600 ,dir_fd=dirfd) - try: - # write all bytes - mv = memoryview(content) - off = 0 - while off < len(mv): - n = os.write(tfd ,mv[off:]) - if n <= 0: raise OSError("short write") - off += n - os.fsync(tfd) - os.fchown(tfd ,uid ,gid) - os.fchmod(tfd ,mode_int) - os.fsync(tfd) - finally: - os.close(tfd) - os.rename(tmp ,f ,src_dir_fd=dirfd ,dst_dir_fd=dirfd) - _fsync_dirfd(dirfd) - finally: - os.close(dirfd) - -def _apply_delete(d: str ,f: str)-> None: - dirfd = _safe_open_dir(d) - try: - if not _exists_regular_nosymlink_at(dirfd ,f): - return - os.unlink(f ,dir_fd=dirfd) - _fsync_dirfd(dirfd) - finally: - os.close(dirfd) - -def _mode_from_entry(ad: dict)-> int: - m = ad.get("mode_int") - if isinstance(m ,int): return m - s = ad.get("mode_octal_str") - if isinstance(s ,str): - try: - return int(s ,8) - except Exception: - pass - raise ValueError("invalid mode") - -def apply_journal(journal: Journal)-> int: - """ - Apply the decoded journal. Returns 0 on success ,1 if any hard errors occurred. - """ - errs = 0 - for idx ,entry in enumerate(journal.command_list ,start=1): - op = getattr(entry ,"name_str" ,"?") - ad = getattr(entry ,"arg_dict" ,{}) or {} - try: - d = ad["write_file_dpath_str"] - f = ad["write_file_fname"] - if not (isinstance(d ,str) and d.startswith("/") and isinstance(f ,str) and "/" not in f): - raise ValueError("bad path or filename") - if op == "displace": - _apply_displace(d ,f) - elif op == "copy": - owner = ad["owner_name"] - mode = _mode_from_entry(ad) - content = ad["content_bytes"] - if not isinstance(content ,(bytes ,bytearray)): raise ValueError("content_bytes missing") - _apply_copy(d ,f ,owner ,mode ,bytes(content)) - elif op == "delete": - _apply_delete(d ,f) - else: - raise ValueError(f"unknown op: {op}") - except Exception as e: - errs += 1 - print(f"apply error [{idx} {op}]: {e}" ,file=sys.stderr) - return 0 if errs == 0 else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/release/shell/Man_In_Grey b/release/shell/Man_In_Grey deleted file mode 100755 index 2ad1606..0000000 --- a/release/shell/Man_In_Grey +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -# man_in_grey — canonical entrypoint for Man_In_Grey -# - Resolves repo → release dirs -# - Requires the privileged gasket (single code path) -# - Always invokes the Python orchestrator with --apply-cmd -set -euo pipefail - -# --- resolve this script’s absolute path (portable) --- -_this="${BASH_SOURCE[0]:-$0}" - -if command -v realpath >/dev/null 2>&1; then - _this_abs="$(realpath -- "$_this")" -else - if _tmp="$(readlink -f -- "$_this" 2>/dev/null)"; then - _this_abs="${_tmp}" - else - _dir="$(cd "$(dirname -- "$_this")" && pwd -P)" - _base="$(basename -- "$_this")" - _this_abs="${_dir}/${_base}" - fi -fi - -_shell_dir="$(cd "$(dirname -- "$_this_abs")" && pwd -P)" # .../release/shell -_release_dir="$(cd "${_shell_dir}/.." && pwd -P)" # .../release -_repo_root="$(cd "${_release_dir}/.." && pwd -P)" # repo root - -# --- orchestrator path (prefer released copy) --- -_py_release="${_release_dir}/python3/Man_In_Grey.py" -_py_dev="${_repo_root}/developer/source/Man_In_Grey.py" -if [[ -f "$_py_release" ]]; then - _py_entry="$_py_release" -elif [[ -f "$_py_dev" ]]; then - _py_entry="$_py_dev" -else - echo "error: Man_In_Grey.py not found in release/python3/ or developer/source/" >&2 - exit 2 -fi - -# --- arch normalize (uname -m → release/) --- -_arch_raw="$(uname -m | tr '[:upper:]' '[:lower:]')" -case "$_arch_raw" in - amd64|x64) _arch="x86_64" ;; - x86_64) _arch="x86_64" ;; - i386|i486|i586|i686) _arch="i686" ;; - arm64|aarch64) _arch="aarch64";; - armv7l) _arch="armv7l" ;; - armv6l) _arch="armv6l" ;; - riscv64) _arch="riscv64";; - ppc64le|powerpc64le) _arch="ppc64le";; - s390x) _arch="s390x" ;; - *) _arch="$_arch_raw" ;; -esac - -_gasket="${_release_dir}/${_arch}/man_in_grey_apply" -if [[ ! -x "$_gasket" ]]; then - echo "error: gasket missing: ${_gasket}" >&2 - echo "hint: from \$REPO_HOME/developer: run compile then release" >&2 - exit 2 -fi - -# --- run orchestrator (single code path via gasket) --- -exec "${PYTHON:-python3}" "$_py_entry" --apply-cmd "$_gasket" "$@" diff --git a/release/shell/diag_perm b/release/shell/diag_perm deleted file mode 100755 index ba10d90..0000000 --- a/release/shell/diag_perm +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# --- tiny helpers ------------------------------------------------------------- -say(){ printf "%s\n" "$*"; } -have(){ command -v "$1" >/dev/null 2>&1; } - -longest_mount_covering() { - # Arg: absolute path - # Output: "MOUNTPOINT FSTYPE OPTIONS" (best effort) - local target="$1" - # Parse /proc/self/mountinfo to find the longest mountpoint prefix - # Fields: ... 5:mountpoint 6:opts ... " - " 9:fstype ... - awk -v P="$target" ' - BEGIN{best=""; bestlen=-1} - { - # split into pre/post " - " - line=$0 - split(line,parts," - ") - pre=parts[1]; post=parts[2]; - n=split(pre,a," ") - mp=a[5]; opts=a[6]; - nf=split(post,b," ") - fstype=b[1]; - - # mountpoints in mountinfo use escape sequences for spaces; safe for our use - if (index(P, mp)==1) { - l=length(mp) - if (l>bestlen) { best=mp; bestlen=l; BO=opts; FT=fstype; } - } - } - END{ - if (bestlen>=0) { print best " " FT " " BO; } - }' /proc/self/mountinfo -} - -# --- locate repo bits --------------------------------------------------------- -script_afp="$(realpath "${BASH_SOURCE[0]}")" -RELEASE_DIR="$(cd "$(dirname "$script_afp")/.." && pwd -P)" -WRAP="$RELEASE_DIR/shell/Man_In_Grey" - -# Compute arch dir like the rest of the tools -arch_raw="$(uname -m | tr '[:upper:]' '[:lower:]')" -case "$arch_raw" in - amd64|x64) arch="x86_64" ;; - x86_64) arch="x86_64" ;; - i386|i486|i586|i686) arch="i686" ;; - arm64|aarch64) arch="aarch64" ;; - armv7l) arch="armv7l" ;; - armv6l) arch="armv6l" ;; - riscv64) arch="riscv64" ;; - ppc64le|powerpc64le) arch="ppc64le" ;; - s390x) arch="s390x" ;; - *) arch="$arch_raw" ;; -esac -GASKET="$RELEASE_DIR/$arch/man_in_grey_apply" - -# --- who/where ---------------------------------------------------------------- -u="$(id -un)"; uid="$(id -u)" -groups="$(id -nG 2>/dev/null || true)" -pwd="$(pwd -P)" -say "== who/where ==" -say "user: $u (uid=$uid) groups: ${groups:-unknown}" -say "pwd : $pwd" -say -say "== paths ==" -say "wrapper: $WRAP" -say "gasket : $GASKET" -say - -# --- wrapper sanity ------------------------------------------------------------ -say "== wrapper sanity ==" -if [[ -f "$WRAP" && -x "$WRAP" ]]; then - read -r firstline <"$WRAP" || true - say "shebang: ${firstline:-}" - say "exec path: $WRAP" -else - say "!! wrapper missing/non-exec" -fi -say - -# --- gasket file sanity & mount flags ----------------------------------------- -say "== gasket file sanity ==" -if [[ -e "$GASKET" ]]; then - ls -l "$GASKET" - if [[ -u "$GASKET" ]]; then - say "setuid: YES" - else - say "setuid: NO" - fi - - # try to find findmnt robustly - FINDMNT_BIN="" - for c in "${FINDMNT_BIN:-}" "$(command -v findmnt 2>/dev/null || true)" /usr/bin/findmnt /bin/findmnt /usr/sbin/findmnt; do - [[ -n "$c" && -x "$c" ]] && { FINDMNT_BIN="$c"; break; } - done - - gasket_abs="$(realpath "$GASKET")" - if [[ -n "$FINDMNT_BIN" ]]; then - # prefer findmnt -T if supported - if "$FINDMNT_BIN" -T "$gasket_abs" >/dev/null 2>&1; then - out="$("$FINDMNT_BIN" -no TARGET,FSTYPE,OPTIONS -T "$gasket_abs" || true)" - else - # fallback: search by path in the tree - out="$("$FINDMNT_BIN" -no TARGET,FSTYPE,OPTIONS "$gasket_abs" 2>/dev/null || true)" - fi - if [[ -n "${out:-}" ]]; then - say "mount: $out" - mp="$(awk '{print $1}' <<<"$out")" - fstype="$(awk '{print $2}' <<<"$out")" - opts="$(awk '{$1="";$2=""; sub(/^ */,""); print}' <<<"$out")" - else - say "mount: findmnt returned no match, falling back to /proc/self/mountinfo" - fi - else - say "mount: findmnt not found, falling back to /proc/self/mountinfo" - fi - - if [[ -z "${opts:-}" ]]; then - parsed="$(longest_mount_covering "$gasket_abs" || true)" - if [[ -n "$parsed" ]]; then - mp="$(awk '{print $1}' <<<"$parsed")" - fstype="$(awk '{print $2}' <<<"$parsed")" - opts="$(awk '{$1="";$2=""; sub(/^ */,""); print}' <<<"$parsed")" - say "mount: $mp $fstype $opts (from /proc/self/mountinfo)" - else - say "mount: could not determine mount options (even via mountinfo)" - fi - fi - - if [[ -n "${opts:-}" ]]; then - if grep -qw nosuid <<<"$opts"; then - say "‼ detected nosuid on mount → setuid will NOT take effect here" - else - say "✓ nosuid NOT present on mount" - fi - if ! grep -qw exec <<<"$opts"; then - say "‼ mount appears noexec (or exec not set) → binary execution may fail" - fi - fi -else - say "!! gasket not found" -fi -say - -# --- gasket self-report -------------------------------------------------------- -say "== gasket self-report (--print-flags) ==" -if [[ -x "$GASKET" ]]; then - "$GASKET" --print-flags || true -else - say "skip (no gasket)" -fi -say - -# --- python inner EUID test ---------------------------------------------------- -say "== python inner EUID test ==" -if [[ -x "$GASKET" ]]; then - tmp_py="$(mktemp /tmp/mig_ids_XXXXXX.py)" - cat >"$tmp_py" <<'PY' -import os, pwd -uid = os.getuid() -euid = os.geteuid() -name = pwd.getpwuid(uid).pw_name -print(f"py.real_uid={uid} ({name})") -print(f"py.effective_uid={euid}") -print("py.groups=", sorted(os.getgroups())) -PY - say "running: $GASKET --inner $tmp_py" - "$GASKET" --inner "$tmp_py" || true - rm -f "$tmp_py" -else - say "skip (no gasket)" -fi -say - -# --- wrapper execution trace (dry) -------------------------------------------- -say "== wrapper execution trace (dry) ==" -if [[ -x "$WRAP" ]]; then - set +e - bash -x "$WRAP" --phase-2-sanity2-then-stop --stage dummy >/dev/null 2>&1 \ - && say "+ bash -x \"$WRAP\" --phase-2-sanity2-then-stop --stage dummy (trace only)" \ - || say "+ bash -x \"$WRAP\" --phase-2-sanity2-then-stop --stage dummy (trace only)" - set -e -else - say "skip (no wrapper)" -fi -say - -# --- verdict ------------------------------------------------------------------- -say "== verdict (rules of thumb) ==" -say "- If --print-flags shows flag.this_process_privileged=0: setuid isn’t taking effect (check nosuid mount, bit not set, or wrong binary)." -say "- If that flag is 1 and the Python test shows euid=0: privileges flow is OK; if you still get EPERM to /etc, the wrapper may bypass the gasket sometimes." -say "- If the wrapper trace never execs the gasket: replace the wrapper so it always runs the gasket." -say "- If root is refused for “not in sudo”, update gasket policy to allow UID 0 (you have that patch)." diff --git a/release/x86_64/man_in_grey_apply b/release/x86_64/man_in_grey_apply deleted file mode 100755 index 48dfade7b32168aaf29ea41e2fc4c59b54986c31..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17232 zcmeHO3vg6bnm&1;Q86756rc29M}kf|X&?wlGz|&4jSWNyK6cbgryofVo$gFOcsR@q zP6%ahrjc=dOgZB&YdP%BtX4TQF6AhTK_IBLPDR|Ar4(J(8lCP@9=eE#O7Hidd;aEj zSMKcAKB~61a#HF4{NMlo=ltit=bm%!z2|(d&AYg~tW0pK5H}0rwqI@_ElMzQlgfa! zh`Hio{9Yrj7UzMll$arJF$knaI#gavs}x=VN_JByGYY-RfCW?bkSN(%rK*b!2~$yH z@MJfRvWhRIcU@%YF=ct7KD!|oB{ZV^XKJ>i*o=JLup3!WOk0#)-z5glb`&nMJEH84 zC_APn$RQ-A98Zb~{m)h9mQp)plrw_)@epL zrF6Cd3#MG(ZLp)b{AUwi<`t?wtLxCH+RapjS2RSzU2|qPL<04Za5S~Pe*N4z^>b!B z6EWvZ=>qvhbx@ybU$#n=ZxTaR(VYZf@&2BZ0Zodr(UWHkNXTi{(5_&XLj!Z2$2w^`s7Fc{6B z-&*MJvA{pHzzQe4_>aiUt0*1s<_zm)k=B zc?WhT$Gs;CZs20K)oH7cL)x}bMk#Hg@LVc;EU#tzOmO)0I!NlL&hw@+;jp?Y`mkh_E zvv83<(>ZHaAv?Q}n{_Tb)7d0+Pv=q{jt09?b1>eybWtQ04R-pv zB0*Hu-5ZOlmg$D-sERaZR^Y!3|L6WHP5E;u>5MCqn61h%i(Z62#`DLoe)uSc-Em^F z!X|~oSYcEOyW*dd@(OXi;!{_IFCcH4z*FLRpZm+ED-8S)-Ec^ZwrZBOn9|PUo+u06W(FMxsQ?fP7^-Cq`%sPo5$xK6V6LWGV3$pwEp0<-h@wP z5Ok9Xzs!U`V8W-E@NFhsJt9cyk4(7g$`aph!ml)y^S37aDii*!3BTHeKX1aXG2ufd z+-|~OG2ve`;Ugw|v~k{~I+h)a!2kOQX#F2mX@le5X%Ipi7)qAsN3{NZReKBV&Nuzi zDMY^ZIsDqDwjf7*56N>!@<_EmC!VH&+=%3#B%Y?I+>qoSC!VH)+_RE@hb#6%VR}fD_bnaQnPb8j(=G=D4Ur0O+$+>NkFDIU+?A#{FpQ#5=LvyZA@}Ce- zLvn7l_Ww^3JWCIALmOeoE=2vmGvk6r0&Fpg2V6@&%%P%c6`481( zut!Vp^``SE=@#gqfb2Sy=<$EGn|guPf1oTIg20pBCrkS_N^|#rRc6~-g`8{1o8IWK zyLVF$ZFA+l>40O3JH2nX)!`KQX%HU(5XMo%Gn23q9pGig7o3N6c_k#k-53!P0byc09^_sH8w{~Xqz_JN=_givgEChvoI8C%Ein z0OEY~q#QR$v3sC6RRw!$k z1p~6e=Vj?e-@5^V>|^Ilz3DTs-2nh*4~!!-KT02X)wcCx3}WuV+iSe()1J)u2S1!3 zsm*)Ew)ss+=hdW+{{wY2bl2>`Gx_|cjp${zfje8W%E(n z=26Bu2V0;ak}@Op@3zYho$0kjuWK#GnA0shaYeHkGlbox9L4lK0<;)Sez4lCItsb z@DP4*32p_EetFYpQF0t5d(!)}4#3{@DdlnVH_6o+U})777@U_598Few22+9+?vx%i zr6J2^KhNhkzs=TYR|+V~-h?=KY^|SW4?qDCV-3O}u0;`ArhXn(^;2k2*Rvzy4Y&oL zc8s85Gzc96wGCX02zv%A9WTOC5C` z)G|p&i`Be;5N4mU&JE;&^r<+|DjNozD->&7MLrnk=`S zLY;mYRec|!apiZT4I;Y}YSjOmsnpE4_fff)x%dl^rGpC`A_>-8JitP zV6=1~Z`*tlnKswa7peQTxlVm(_ojCl(L;RQ{rk(7`afMhxZtEz{xOZOnWm>OEzk|~ z3t4IE^;-uwo~+Ua@=3Vg^h4B{eaw*jiVFDf*XV@9pONL{MKqyMT(v=rdCeWu$<3zE zjZnp5^gP{Ixz|yTYsizXbY!mvh-fZ`A=>xw-t0yI#z>RA{@k1(=ZxGI+zPIvxdsdd zitV{I6aJLPa@$< zhOoyFzEKc<2CuUdQC(=8>!>IF#=TnlJ+1%uCptRi4UQ^;Fv(B8>^dP%hMcykn_*zA z&mfxz6c=qKk$I<+ld-%zRf{|VbwU1!>hF8G-)EXHkoryKL}r}T(kHY%$8OQ~oUYKy zhP5}&CMThQ{i=Yf{E?7tYO4@z|8KTy&TMLa2V#-BVU^b3{3KaFj%Ov^M}R!3zcC<$t^PvDAagH0begm?*eD6a3P7>;RB$(ADrHCj*i zn^(|~jis>&j74B90{;giK<~a&(KXT7x~Oo8M$yn0kNF!C!APh9UlBws)+_4k!_jCk z4%!#-MTyV{kNQx=*PRfFR3Ik5Aq3-2d<1B&jRY=ap1xx=*Cs9$q)Jjf9M!wy_{yM@ zS#vE^8&ZjQL!v7joiB4TW3b4I4tJ*}7CBdSF0P*|yzXV~qCubrvDDM9FLHamZ7cLu zo>nv4Ud%3YFKv_J#a?$i>FSB+ERxvC*kOO zTOOJi5-7but@8eR+Ex`PNh0;y6w7)m9= z(Qf-WvEA3#7uhf$KKLT`R3aF+Pp?f(w}%tb|9A}U`Jw^IgrjzOeeBce0!>G$6loc| zPR+AY?^v;XQQJzQp4&a%w)Qrx1TdptS#c9hk>l`x_-H=A5%eFA=krey5Bg7_cViLY z1@&U#vI~^oO_$xbLX@qqDZ6s~`BmG>&aWYyUQSd&e;O#K(e;gQ5~P18>XmyUs%sWk zPg!ESXkFDNamy7qT|dhqiy?b*)((GYTIWRg-S~e1_3b9Gz4{0(8p+or@zDP+7G`2$ zb!7o+dJ1e7 zR#&;{<s1n^Ii?&m6;e(`)TOa6QvQA;dQdAPVkprCKxqes_1o!&QWx+qPHs=Qgp4NTNHg%(H)BJ zRCHL;-zoZmqNfx+&%8;fGzfo!uWB)b?TZ%8x7W1>yTZPx-8IuW)7e<>ntekc*yWm; za?MbXbwSjPE5*ln6Iz_8d(hsxjq1YNm6$$^kETBzc)75NZR&=T&t|}Gz<(L`VYaj& zLrKNmh3shMEC5dBjPQ+3Hb`{)5H9$JmZ%W7DV)a-1&eNEsNT9315vlDNJxCL;PJqa z3h_8HR6dVKjPH>8lM4F+8MpF(ztq1(^jyoD=jdqdeN*aRDoXEz!_sV`;QjVo%5kZG zSz$b3d>k4$T35Q%0-t4pcL29@p(gqe#w$CGGA$+$S=!$_TD{-3!2iMm-vPV^<=2#y z|BQwH%NF=Q6JA~;cz$4S-?7j?Zh>Ej_Kw!Bt1a+v<31Tp-)W)0$O6B^0$*c+f5!sf z2Atw4O5*v1h5mgO{yz(SI|tE3=^M&u?R~=n|5xA>%BKh(x45Wd7Wx(NX9E7X@=9ee z(ZWxM1%9Iiz65v;jY}Qui9sKg{@4oVsirCMa;0C|KffW%w+XunhJ5z~o7EQeZm__& z0;l+tmj5_#yYj(Ip0g0E@upT zzb%oH|y~`K6z54RS zi&wUF>YeU|-ZovH9cbar^m4O&=~n-O6uat6H|7^)|KT=!I!#cd&BqOjK;zUx5u^tL zzND{kc%ewfryYtw8BXIkL@{1+Y@#5XO@-srK>?t~(2 zoXg;DOb0ebJ%wQ$WhpY`V&y|eLLB?!f=X|E?RePw76lb>Q=j|{sQek_3Z@`q_6F@~KD#M(< z1%{r%*`A-fnJ!Vv>_6)<{VwG6OwT+&*E8kiO{xFvLk1a&DckdV2BwFUBG+Hq{>PO4 zLZ!#=Etv9q4zj0r8>RMt4UA&KGl5!P_w;V*)ipF*UG%Kc=0zS z_BCo9$CUkLd*+!wTVl`aK&DRho`vm|d(4WRCHDN>!nDL(UaNvJGP{&LUq4}2QbpYb zfq}E%N=xiRhT_L+{JhC@HY+Mx8vlbO_Uoq^icC+I*q7E%&$-w&F7F{y<@b1e{<~D4 zd8S#|;H^Nxo}cRveVs}>*WV289MAUTAN4(Tn-`wL`8+@OAG&`iu^nH>zXL|`V|#wj zH-f$Plp0BBN^H-RwuYPSMT=5yG$_}j&_eyp(|Khoi|e 0: + print(f"setup: already inside a role shell ({env.get('PROJECT', '?')} {env.get('ROLE', '?')}).") + print("setup: type 'exit' to leave it, then run setup again.") + sys.exit(1) + env["RT_SETUP_DEPTH"] = str(depth + 1) + + # Python-domain logic: Variables and Paths + # The project is the directory holding this file, whatever REPO_HOME the + # calling shell carries; an inherited REPO_HOME may name another project. + repo_home = os.path.dirname(os.path.realpath(__file__)) + role = args.role + role_home = os.path.join(repo_home, role) + tool_dir = os.path.join(role_home, "tool") + setup_file = os.path.join(role, "tool", "setup") + + env["ROLE"] = role + env["ROLE_HOME"] = role_home + env["SETUP"] = setup_file + + print(f"ROLE_HOME {role_home}") + + # Python-domain logic: Directory validation + if not os.path.isdir(role_home): + print(f"setup: failed to locate directory for role '{role}'") + sys.exit(1) + + # Bash-domain logic: The rcfile only handles operations Python cannot inherit + # (sourcing bash functions) and performs the directory change internally to + # guarantee the shared scripts are sourced from the project root. + bash_logic = [ + 'if [[ -f ~/.bashrc ]]; then source ~/.bashrc; fi', + 'if [[ -f "shared/tool/setup" ]]; then source "shared/tool/setup"; fi', + 'if [[ -f "shared/authored/setup" ]]; then source "shared/authored/setup"; fi', + '# the role tool directory is searched before everything the shared setup added', + f'PATH="{tool_dir}:$PATH"; export PATH', + f'cd "{role}"', + 'if [[ -f "tool/setup" ]]; then', + ' source "tool/setup"', + f' echo "in environment: {setup_file}"', + 'else', + f' echo "not found: {setup_file}"', + 'fi', + 'if [[ -n "${INSIDE_EMACS}" ]] && command -v env_to_emacs >/dev/null 2>&1; then', + ' env_to_emacs', + 'fi', + 'rm -f "${BASH_SOURCE[0]}"' + ] + + fd, rcfile_path = tempfile.mkstemp(suffix="-rt-setup.sh") + with os.fdopen(fd, "w") as f: + f.write("\n".join(bash_logic) + "\n") + + os.chdir(repo_home) + os.execvpe("bash", ["bash", "--rcfile", rcfile_path], env) + +if __name__ == "__main__": + main() diff --git a/release/python3/.githolder b/shared/authored/.gitkeep similarity index 100% rename from release/python3/.githolder rename to shared/authored/.gitkeep diff --git a/shared/project-dependency/.gitignore b/shared/project-dependency/.gitignore new file mode 100644 index 0000000..a033c31 --- /dev/null +++ b/shared/project-dependency/.gitignore @@ -0,0 +1,11 @@ +# shared/project-dependency/.gitignore +# Tracks the skeleton's parts. Add a ! line for anything else to track, +# such as a dependency's link. +* +!.gitignore +!project +!*.init +!*.install +!*.org +!*.html +!Python-3.12.3 diff --git a/shared/project-dependency/Python.init b/shared/project-dependency/Python.init new file mode 100644 index 0000000..f0b108d --- /dev/null +++ b/shared/project-dependency/Python.init @@ -0,0 +1,25 @@ +# -*- mode: sh; sh-shell: bash; -*- +# init files are bash script + +# Python.init +# -------------------------------------------------------------------------------- +# Put the project's Python build at the front of PATH, but only when it runs. +# +# Every '#!/usr/bin/env python3' script the person runs, their own tools +# included, takes the first python3 on PATH. A build that cannot start, for +# example one that lost its libpython after its directory moved, must not be +# put there, or all of those scripts fail with it. The test runs the +# interpreter once with an empty program. + +python_dp="$REPO_HOME/shared/project-dependency/Python-3.12.3" +if [[ ! -e "$python_dp" ]]; then + if [[ "$ROLE" == "administrator" ]]; then + echo "Python.init: no $python_dp; using the system python3" + fi +elif "$python_dp/bin/python3" -c '' 2>/dev/null; then + PATH="$python_dp/bin:$PATH" +else + echo "Python.init: $python_dp/bin/python3 does not run; using the system python3" >&2 + echo "Python.init: run it directly to see why, and see Python.org beside this file" >&2 +fi +unset python_dp diff --git a/shared/project-dependency/Python.org b/shared/project-dependency/Python.org new file mode 100644 index 0000000..8e67c79 --- /dev/null +++ b/shared/project-dependency/Python.org @@ -0,0 +1,71 @@ +#+TITLE: Python Installation Guide +#+AUTHOR: Project Administrator + +This project requires a dedicated Python environment. The configuration plugin (`shared/project-dependency/Python.init`) was written targeting Python 3.12.3, though the architecture does not rely on any known version-dependent features. + +The build below can be moved. Nothing in it records the absolute directory it was built in where the interpreter depends on it, so the directory can move with a home directory, or to another machine, and keep working. + +* 1. Install the development libraries + +Python's configure step looks for the development files of each optional library, and silently leaves out any module whose library it does not find. A build without sqlite3-devel, for example, has no sqlite3 module, and the gap shows only when a program imports it. Install these first. The names are for openSUSE; zypper search finds the name of one that differs. + +#+begin_src bash +sudo zypper install gcc make sqlite3-devel libopenssl-devel libffi-devel zlib-devel \ + readline-devel ncurses-devel xz-devel libbz2-devel gdbm-devel tk-devel libuuid-devel +#+end_src + +zlib-ng-compat-devel serves in place of zlib-devel, and the two conflict; if zypper reports that conflict, keep the one already installed. + +tk-devel gives the tkinter module, which the window programs use. + +* 2. Download and Extract +Download the Python source code and extract it as a sibling to the main project directory. Assuming you are at the top level of your project workspace: + +#+begin_src bash +cd ../ +wget https://www.python.org/ftp/python/3.12.3/Python-3.12.3.tgz +tar -xzf Python-3.12.3.tgz +cd Python-3.12.3 +#+end_src + +* 3. Build the Environment +Compile the Python binaries in place. Setting the installation prefix to the current directory isolates the build. + +#+begin_src bash +CONFIG_SITE= ./configure --prefix="$(pwd)" --libdir="$(pwd)/lib" --enable-optimizations +make -j8 2>&1 | tee build.log +make install +#+end_src + +Everything installs inside this directory; nothing is written to the system. + +CONFIG_SITE= and --libdir keep the machine's own defaults out of the build, so the recipe gives the same layout on every Linux. Some distributions, openSUSE and Fedora among them, name a site file in CONFIG_SITE that sets the library directory to lib64 for every configure script. Python then installs its compiled modules under lib64 while the interpreter looks for them under lib, and every import of a compiled module, such as ssl, fails. + +The build is static: the interpreter is one program, with no separate libpython3.12.so to find when it starts. A build with --enable-shared records the absolute path of its library directory in the program, and stops working when the directory moves. C extension modules, such as those pip installs, work with the static build. + +Near the end of build.log, make lists any optional modules it could not build, under "The necessary bits to build these optional modules were not found". nis, an obsolete network directory service, and _curses_panel may appear there; the project uses neither. For any other module listed, install the missing development library and build again. + +* 4. Check the build + +#+begin_src bash +bin/python3 -c 'import sys ,ssl ,sqlite3 ,ctypes ,lzma ,bz2 ,readline ,tkinter; print(sys.prefix)' +ldd bin/python3 | grep libpython # prints nothing: no shared libpython +#+end_src + +* 5. Use pip through the interpreter + +The scripts make install writes into bin/, such as pip3, start with a #! line naming the absolute path of the interpreter, so they break when the directory moves. Run pip as a module instead, which uses whichever interpreter runs it: + +#+begin_src bash +python3 -m pip install +#+end_src + +* 6. Link the Project + +Python-3.12.3 is linked to a Harmony derived project by default. + +See administrator/document/how-to_add-project-dependency.html for details about linking a project, if you want to check the setup. + +If you are installing a different version, then at least the Python-3.12.3 version project linking is there as a good example. + +Project linking is not difficult, make a sym link to `project/`, and then add an init file, as described in administrator/document/how-to_add-project-dependency.html. The 'project' in the link is literal. diff --git a/shared/project-dependency/project b/shared/project-dependency/project new file mode 120000 index 0000000..1b20c9f --- /dev/null +++ b/shared/project-dependency/project @@ -0,0 +1 @@ +../../../ \ No newline at end of file diff --git a/release/shell/.githolder b/shared/tool/PATH.sh similarity index 100% rename from release/shell/.githolder rename to shared/tool/PATH.sh diff --git a/shared/tool/env_to_emacs b/shared/tool/env_to_emacs new file mode 100755 index 0000000..8a65dba --- /dev/null +++ b/shared/tool/env_to_emacs @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# env_to_emacs - give the Emacs shell buffer this setup runs in the role's environment +# +# Emacs starts programs (compile, grep, and the like) with its own copy of the +# environment, which a shell running inside it cannot change. This tool sets +# REPO_HOME, PROJECT, ROLE and PATH in Emacs, and exec-path to match, for the +# one shell buffer whose shell this setup runs under, and for no other. +# +# The values are buffer-local. Killing the buffer discards them, and other +# buffers, including shells started later, keep Emacs's own environment. (An +# earlier version set them globally, so every shell started afterward began +# in the last role set up, even after that role's buffer was gone.) +# +# The buffer is found by process: Emacs knows the process id of the shell in +# each shell buffer, and that shell is an ancestor of this script, perhaps +# several generations up, since setup starts a new bash under the buffer's own. + +script_afp=$(realpath "${BASH_SOURCE[0]}") + +set -euo pipefail + +if [ -z "${REPO_HOME:-}" ]; then + echo "env_to_emacs: error: REPO_HOME is not set. Source a setup environment first." + exit 1 +fi + +if ! command -v emacsclient >/dev/null 2>&1; then + echo "env_to_emacs: error: emacsclient is not installed or not in PATH." + exit 1 +fi + +# quote a value as an Emacs Lisp string +elisp_string(){ + local s="$1" + s=${s//\\/\\\\} + s=${s//\"/\\\"} + printf '"%s"' "$s" +} + +form=$(cat < pid 1) (not buffer)) + (setq buffer + (seq-find (lambda (b) + (let ((p (get-buffer-process b))) + (and p (eql (process-id p) pid)))) + (buffer-list))) + (unless buffer + (setq pid (alist-get 'ppid (process-attributes pid))))) + (if (not buffer) + "no shell buffer found" + (with-current-buffer buffer + (setq-local process-environment (copy-sequence process-environment)) + (setenv "REPO_HOME" $(elisp_string "$REPO_HOME")) + (setenv "PROJECT" $(elisp_string "${PROJECT:-}")) + (setenv "ROLE" $(elisp_string "${ROLE:-}")) + (setenv "PATH" $(elisp_string "$PATH")) + (setq-local exec-path (append (parse-colon-path $(elisp_string "$PATH")) (list exec-directory))) + (buffer-name)))) +EOF +) + +if ! result=$(emacsclient --eval "$form" 2>&1); then + echo "env_to_emacs: cannot reach the Emacs server; Emacs environment left unchanged" + echo " $result" + exit 0 +fi +result=${result#\"} +result=${result%\"} + +if [[ "$result" == "no shell buffer found" ]]; then + echo "env_to_emacs: no Emacs shell buffer runs this shell; Emacs environment left unchanged" + exit 0 +fi + +echo "Emacs buffer $result has the environment of $PROJECT ${ROLE:-}" diff --git a/shared/tool/makefile/Harmony.mk b/shared/tool/makefile/Harmony.mk new file mode 100644 index 0000000..b168ac5 --- /dev/null +++ b/shared/tool/makefile/Harmony.mk @@ -0,0 +1,137 @@ +# make/Harmony.mk - build *.lib.c and *.CLI.c +# written for the Harmony skeleton, always invoked from cwd $REPO_HOME/ +# files have two suffixes by convention, e.g.: X.lib.c or Y.CLI.c + +.SUFFIXES: +.DELETE_ON_ERROR: + +#-------------------------------------------------------------------------------- +# Harmony structure + +SHELL=/bin/bash + +ECHO := printf "%b\n" + +C_SOURCE_DIR ?= authored +C ?= gcc +CFLAGS ?= -std=gnu11 -Wall -Wextra -Wpedantic -finput-charset=UTF-8 +CFLAGS += -MMD -MP +CFLAGS += -include "$(REPO_HOME)/shared/tool/makefile/RT_global.h" +CFLAGS += -I $(C_SOURCE_DIR) + +# Project administrators can override this in their local makefile +LIBRARY_NAME ?= $(PROJECT) +LIBRARY_NAME := $(subst -,_,$(LIBRARY_NAME)) + +BUILD_DIR ?= scratchpad/build +OBJECT_DIR ?= $(BUILD_DIR)/object +LIBRARY_DIR ?= scratchpad/made +MACHINE_DIR ?= scratchpad/made + +LIBRARY_FILE ?= $(LIBRARY_DIR)/lib$(LIBRARY_NAME).a + +# the project library is linked only when the namespace makes one +LN_FLAGS ?= -L$(LIBRARY_DIR) -L/lib64 -L/lib $(if $(object_lib),-l$(LIBRARY_NAME)) +# LN_FLAGS ?= -L$(LIBRARY_DIR) -L/lib64 -L/lib -l$(LIBRARY_NAME) + +KMOD_SOURCE_DIR ?= authored +KMOD_CCFLAGS ?= -I $(KMOD_SOURCE_DIR) +# Pass the global header to Kbuild exactly as done for user-space +KMOD_CCFLAGS += -include $(REPO_HOME)/shared/tool/makefile/RT_global.h +KMOD_OUTPUT_DIR ?= scratchpad/kmod + +#-------------------------------------------------------------------------------- +# derived variables + +# source discovery (single dir) +c_source_lib := $(wildcard $(C_SOURCE_DIR)/*.lib.c) +c_source_exec := $(wildcard $(C_SOURCE_DIR)/*.CLI.c) + +# remove suffix to get base name +c_base_lib := $(sort $(patsubst %.lib.c,%, $(notdir $(c_source_lib)))) +c_base_exec := $(sort $(patsubst %.CLI.c,%, $(notdir $(c_source_exec)))) + +# two sets of object files, one for the lib, and one for the CLI programs +object_lib := $(patsubst %, $(OBJECT_DIR)/%.lib.o, $(c_base_lib)) +object_exec := $(patsubst %, $(OBJECT_DIR)/%.CLI.o, $(c_base_exec)) + +# executables are made from exec_ sources +exec_ := $(patsubst %, $(MACHINE_DIR)/%, $(c_base_exec)) + +#-------------------------------------------------------------------------------- +# pull in dependencies + +-include $(object_lib:.o=.d) $(object_exec:.o=.d) + + +#-------------------------------------------------------------------------------- +# targets + +# when no target is given make uses the first target, this one +.PHONY: usage +usage: + @echo example usage: make clean + @echo example usage: make library + @echo example usage: make CLI + @echo example usage: make library CLI + +.PHONY: version +version: + @echo makefile version 8.0 + if [ ! -z "$(C)" ]; then $(C) -v; fi + /bin/make -v + +.PHONY: information +information: + @printf "· → Unicode middle dot - visible: [%b]\n" "·" + @echo "C_SOURCE_DIR: " $(C_SOURCE_DIR) + @echo "BUILD_DIR: " $(BUILD_DIR) + @echo "c_source_lib: " $(c_source_lib) + @echo "c_source_exec: " $(c_source_exec) + @echo "c_base_lib: " $(c_base_lib) + @echo "c_base_exec: " $(c_base_exec) + @echo "object_lib: " $(object_lib) + @echo "object_exec: " $(object_exec) + @echo "exec_: " $(exec_) + +.PHONY: library +library: $(LIBRARY_FILE) + +$(LIBRARY_FILE): $(object_lib) + @mkdir -p $(MACHINE_DIR) + @if [ -s "$@" ] || [ -n "$(object_lib)" ]; then \ + echo "ar rcs $@ $^"; \ + ar rcs $@ $^; \ + else \ + rm -f "$@"; \ + fi + +#.PHONY: CLI +#CLI: $(LIBRARY_FILE) $(exec_) + +.PHONY: CLI +CLI: library $(exec_) + + +# generally better to use the project local clean scripts, but this will make it so that the make targets can be run again + +.PHONY: clean +clean: + rm -f $(LIBRARY_FILE) + for obj in $(object_lib) $(object_exec); do rm -f $$obj $${obj%.o}.d || true; done + for i in $(exec_); do [ -e $$i ] && rm $$i || true; done + rm -rf $(BUILD_DIR) + + +# recipes +#$(OBJECT_DIR)/%.o: $(C_SOURCE_DIR)/%.c +# @mkdir -p $(OBJECT_DIR) +# $(C) $(CFLAGS) $(CPPFLAGS) -o $@ -c $< + +$(OBJECT_DIR)/%.o: $(C_SOURCE_DIR)/%.c + @mkdir -p $(OBJECT_DIR) + $(C) $(CFLAGS) $(if $(findstring .lib,$@),-D$(if $(NAMESPACE),$(NAMESPACE)·,)$(basename $*)) -o $@ -c $< + +$(MACHINE_DIR)/%: $(OBJECT_DIR)/%.CLI.o + @mkdir -p $(MACHINE_DIR) + $(C) -o $@ $< $(LN_FLAGS) diff --git a/shared/tool/makefile/RT_global.h b/shared/tool/makefile/RT_global.h new file mode 100644 index 0000000..29f2e63 --- /dev/null +++ b/shared/tool/makefile/RT_global.h @@ -0,0 +1,11 @@ +#ifndef RT_global_H +#define RT_global_H + #include + #include + + typedef unsigned int uint; + + #define Local static + #define Free(pt) free(pt); (pt) = NULL; + +#endif diff --git a/shared/tool/makefile/target_kernel-module.mk b/shared/tool/makefile/target_kernel-module.mk new file mode 100644 index 0000000..b3684c4 --- /dev/null +++ b/shared/tool/makefile/target_kernel-module.mk @@ -0,0 +1,130 @@ +# make/target_kernel-module.mk — build *.kmod.c as kernel modules (single-pass, kmod-only) +# invoked from $REPO_HOME/ +# version 1.4 + +.SUFFIXES: +.DELETE_ON_ERROR: + +#-------------------------------------------------------------------------------- +# defaults for environment variables (override from outer make/env as needed) + +# Kernel build tree, which is part of the Linux system, use running kernel if unset +KMOD_BUILD_DIR ?= /lib/modules/$(shell uname -r)/build + +# Authored source directory (single dir) +KMOD_SOURCE_DIR ?= cc + +# Extra compiler flags passed to Kbuild (e.g., -I $(KMOD_SOURCE_DIR)) +KMOD_CCFLAGS ?= + +# Include *.lib.c into modules (1=yes, 0=no) +KMOD_INCLUDE_LIB ?= 1 + +# KMOD_OUTPUT_DIR is constrained, relative path, on the scratchpad, and ends in kmod +# Require: non-empty, relative, no '..', ends with 'kmod' dir +define assert_kmod_output_dir_ok + $(if $(strip $(1)),,$(error KMOD_OUTPUT_DIR is empty)) + $(if $(filter /%,$(1)),$(error KMOD_OUTPUT_DIR must be relative: '$(1)'),) + $(if $(filter %/../% ../% %/.. ..,$(1)),$(error KMOD_OUTPUT_DIR must not contain '..': '$(1)'),) + $(if $(filter %/kmod %/kmod/ kmod,$(1)),,$(error KMOD_OUTPUT_DIR must end with 'kmod': '$(1)')) +endef +KMOD_OUTPUT_DIR ?= scratchpad/kmod +$(eval $(call assert_kmod_output_dir_ok,$(KMOD_OUTPUT_DIR))) + +# The kernel make needs and absolute path to find the output directory +ABS_KMOD_OUTPUT_DIR := $(CURDIR)/$(KMOD_OUTPUT_DIR) + +#-------------------------------------------------------------------------------- +# derived variables (computed from the above) + +# Authored basenames (without suffix) +base_list := $(patsubst %.kmod.c,%,$(notdir $(wildcard $(KMOD_SOURCE_DIR)/*.kmod.c))) + +# Optional library sources (without suffix) to include inside modules +ifeq ($(KMOD_INCLUDE_LIB),1) +lib_base := $(patsubst %.lib.c,%,$(notdir $(wildcard $(KMOD_SOURCE_DIR)/*.lib.c))) +else +lib_base := +endif + +# Staged sources (kept namespaced to prevent .o collisions) +all_kmod_c := $(addsuffix .kmod.c,$(addprefix $(KMOD_OUTPUT_DIR)/,$(base_list))) +all_lib_c := $(addsuffix .lib.c,$(addprefix $(KMOD_OUTPUT_DIR)/,$(lib_base))) + + + +#-------------------------------------------------------------------------------- +# targets + +.PHONY: usage +usage: + @printf "Usage: make [kmod|clean|information|version]\n" + +.PHONY: version +version: + @echo target_kmod version 1.4 + +.PHONY: information +information: + @echo "KMOD_SOURCE_DIR: " $(KMOD_SOURCE_DIR) + @echo "KMOD_BUILD_DIR: " $(KMOD_BUILD_DIR) + @echo "KMOD_OUTPUT_DIR: " $(KMOD_OUTPUT_DIR) + @echo "base_list: " $(base_list) + @echo "lib_base: " $(lib_base) + @echo "all_kmod_c: " $(all_kmod_c) + @echo "all_lib_c: " $(all_lib_c) + @echo "KMOD_INCLUDE_LIB=" $(KMOD_INCLUDE_LIB) + + +ifeq ($(strip $(base_list)),) + $(warning No *.kmod.c found under $(KMOD_SOURCE_DIR); nothing to build) +endif + +# --- Parallel-safe preparation as real targets --- + +# ensure the staging dir exists (order-only prereq) +$(KMOD_OUTPUT_DIR): + @mkdir -p "$(KMOD_OUTPUT_DIR)" + +# generate the Kbuild control Makefile +$(KMOD_OUTPUT_DIR)/Makefile: | $(KMOD_OUTPUT_DIR) + @{ \ + printf "ccflags-y += %s\n" "$(KMOD_CCFLAGS)"; \ + printf "obj-m := %s\n" "$(foreach m,$(base_list),$(m).o)"; \ + for m in $(base_list); do \ + printf "%s-objs := %s.kmod.o" "$$m" "$$m"; \ + for lb in $(lib_base); do printf " %s.lib.o" "$$lb"; done; \ + printf "\n"; \ + done; \ + } > "$@" + +# stage kmod sources (one rule per file; parallelizable) +$(KMOD_OUTPUT_DIR)/%.kmod.c: $(KMOD_SOURCE_DIR)/%.kmod.c | $(KMOD_OUTPUT_DIR) + @echo "--- Stage: $@ ---" + @cp -f "$(abspath $<)" "$@" + +# stage library sources (optional; also parallelizable) +$(KMOD_OUTPUT_DIR)/%.lib.c: $(KMOD_SOURCE_DIR)/%.lib.c | $(KMOD_OUTPUT_DIR) + @echo "--- Stage: $@ ---" + @cp -f "$(abspath $<)" "$@" + + +.PHONY: kmod +kmod: $(KMOD_OUTPUT_DIR)/Makefile $(all_kmod_c) $(all_lib_c) +ifeq ($(strip $(base_list)),) + @echo "--- No kmod sources; nothing to do ---" +else + @echo "--- Invoking Kbuild for kmod: $(base_list) ---" + $(MAKE) -C "$(KMOD_BUILD_DIR)" M="$(ABS_KMOD_OUTPUT_DIR)" modules +endif + +# quality-of-life: allow 'make scratchpad/kmod/foo.ko' after batch build +$(KMOD_OUTPUT_DIR)/%.ko: kmod + @true + +.PHONY: clean +clean: + @echo "Cleaning: $(KMOD_BUILD_DIR)" + @$(MAKE) -C "$(KMOD_BUILD_DIR)" M="$(ABS_KMOD_OUTPUT_DIR)" clean >/dev/null 2>&1 || true + @echo "Cleaning: $(KMOD_OUTPUT_DIR)" + @rm -rf -- "$(KMOD_OUTPUT_DIR)" diff --git a/shared/tool/replace_from_tar b/shared/tool/replace_from_tar new file mode 100755 index 0000000..6da5c0f --- /dev/null +++ b/shared/tool/replace_from_tar @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- +""" +replace_from_tar + +Makes directories hold exactly what a tar file holds. For each directory at +the top of the tar file, such as SubU.copy/, the directory of the same name in +the extract dir is brought to match: files the tar file has are written when +they are new or differ, and files and directories it does not have are +deleted. The extract dir is the directory 'tar -x' would be run in: the one +that holds SubU.copy, not SubU.copy itself. It is the current directory unless +given. Extracting a tar file over a directory only adds and overwrites, so +a file that was removed from the tar file would otherwise stay behind. + +The tool deletes files. Checkpoint the project with git first, and run +preview to read what it would do. + +Usage: replace_from_tar [argument ...] + replace_from_tar preview [argument ...] + help | -h | --help print this message (also printed when given no arguments) + replace [] + bring each directory at the top of the tar file, + in the extract dir, to match the tar file + preview print the log of what would do, change nothing + +Example, run in developer/authored, which holds SubU.copy: + replace_from_tar preview replace SubU.copy.tar + replace_from_tar replace SubU.copy.tar +""" + +import sys ,os ,tarfile +from pathlib import Path, PurePosixPath + +TAG_WIDTH = 12 + + +class Log: + PREVIEW_TAG = {"made": "would make" ,"replaced": "would replace" ,"deleted": "would delete"} + + def __init__(self ,preview): + self.preview = preview + self.count = {} + self.error_count = 0 + + def line(self ,tag ,subject ,reason=None): + self.count[tag] = self.count.get(tag ,0) + 1 + if tag == "unchanged": return + shown = self.PREVIEW_TAG.get(tag ,tag) if self.preview else tag + print(f" {shown:<{TAG_WIDTH + 2}}{subject}" + (f" ({reason})" if reason else "")) + + def error(self ,subject ,reason): + self.error_count += 1 + print(f" {'error':<{TAG_WIDTH + 2}}{subject} ({reason})") + + def summary(self): + print("summary:") + for tag ,what in (("made" ,"files") ,("replaced" ,"files") ,("deleted" ,"files and directories") ,("unchanged" ,"files")): + n = self.count.get(tag ,0) + verb = f"that would be {tag}" if self.preview and tag != "unchanged" else tag + print(f" {n:>4} {what} {verb}") + if self.error_count: + print(f" {self.error_count:>4} errors") + if self.preview: + print("preview only: nothing was changed.") + return 1 if self.error_count else 0 + + +def member_problem(member): + path = PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts: + return "an absolute path or '..'" + if len(path.parts) < 1: + return "an empty name" + if not (member.isfile() or member.isdir() or member.issym()): + return "neither a file, a directory nor a symbolic link" + if member.issym() and (PurePosixPath(member.linkname).is_absolute() or ".." in PurePosixPath(member.linkname).parts): + return "a symbolic link leaving its directory" + return None + +def replace(tar_path ,extract_dir ,log ,preview): + extract_dir = Path(extract_dir) + if not extract_dir.is_dir(): + log.error(str(extract_dir) ,"not a directory"); return + with tarfile.open(tar_path) as tar: + member_list = tar.getmembers() + for m in member_list: + problem = member_problem(m) + if problem: + log.error(m.name ,f"refused: {problem}; nothing was changed"); return + top_set = sorted({PurePosixPath(m.name).parts[0] for m in member_list}) + for top in top_set: + if any(PurePosixPath(m.name) == PurePosixPath(top) and not m.isdir() for m in member_list): + log.error(top ,"refused: the top of the tar file must hold directories; nothing was changed"); return + target = extract_dir / top + if target.is_symlink(): + log.error(str(target) ,"refused: it is a symbolic link; nothing was changed"); return + + wanted = {str(PurePosixPath(m.name)): m for m in member_list} + + # ---- delete what the tar file does not have, deepest first + for top in top_set: + target = extract_dir / top + if not target.exists(): continue + for dirpath ,dirname_list ,filename_list in os.walk(target ,topdown=False ,followlinks=False): + for name in sorted(filename_list) + sorted(d for d in dirname_list if (Path(dirpath) / d).is_symlink()): + path = Path(dirpath) / name + key = str(PurePosixPath(path.relative_to(extract_dir))) + if key not in wanted or wanted[key].isdir(): + if not preview: path.unlink() + log.line("deleted" ,key) + for name in sorted(d for d in dirname_list if not (Path(dirpath) / d).is_symlink()): + path = Path(dirpath) / name + key = str(PurePosixPath(path.relative_to(extract_dir))) + if key not in wanted: + if not preview: + if any(path.iterdir()): + log.error(key ,"not empty after its files were deleted"); continue + path.rmdir() + log.line("deleted" ,key + "/") + + # ---- write what is new or differs + for m in sorted(member_list ,key=lambda m: m.name): + key = str(PurePosixPath(m.name)) + path = extract_dir / key + if m.isdir(): + if not path.is_dir(): + if not preview: path.mkdir(parents=True ,exist_ok=True) + log.line("made" ,key + "/") + continue + if m.issym(): + if path.is_symlink() and os.readlink(path) == m.linkname: + log.line("unchanged" ,key); continue + if not preview: + if path.is_symlink() or path.exists(): path.unlink() + path.parent.mkdir(parents=True ,exist_ok=True) + os.symlink(m.linkname ,path) + log.line("replaced" if path.is_symlink() else "made" ,key); continue + data = tar.extractfile(m).read() + mode = m.mode & 0o777 + if path.is_file() and not path.is_symlink(): + if path.read_bytes() == data and (path.stat().st_mode & 0o777) == mode: + log.line("unchanged" ,key); continue + tag = "replaced" + else: + tag = "made" + if not preview: + path.parent.mkdir(parents=True ,exist_ok=True) + if path.is_symlink(): path.unlink() + path.write_bytes(data) + os.chmod(path ,mode) + log.line(tag ,key) + + +def print_help(): + print(__doc__.strip()) + return 0 + +def CLI(argv=None): + if argv is None: argv = sys.argv[1:] + if not argv or argv[0] in ("help" ,"-h" ,"--help"): return print_help() + preview = argv[0] == "preview" + if preview: argv = argv[1:] + if not argv: + print("Error: 'preview' needs a command after it." ,file=sys.stderr) + return 1 + command ,rest = argv[0] ,argv[1:] + if command == "replace" and len(rest) in (1 ,2): + if len(rest) == 1: rest = [rest[0] ,"."] + if not Path(rest[0]).is_file(): + print(f"Error: no tar file at {rest[0]}." ,file=sys.stderr) + return 1 + log = Log(preview) + print(f"replace_from_tar {'preview ' if preview else ''}replace {rest[0]} in {os.path.abspath(rest[1])}") + replace(rest[0] ,rest[1] ,log ,preview) + return log.summary() + print(f"Error: unknown command or wrong arguments: '{command}'." ,file=sys.stderr) + print_help() + return 1 + + +if __name__ == "__main__": + sys.exit(CLI()) diff --git a/shared/tool/scratchpad b/shared/tool/scratchpad new file mode 100755 index 0000000..f14f140 --- /dev/null +++ b/shared/tool/scratchpad @@ -0,0 +1,225 @@ +#!/usr/bin/env -S python3 -B +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- + +import os, sys, shutil, stat, pwd, grp, subprocess + +HELP = """usage: scratchpad {ls|clear|help|make|write|size|find|lock|unlock} [ARGS] + ls List scratchpad in an indented tree with perms and owner (quiet if missing). + clear Remove all contents of scratchpad/ except top-level .gitignore. + clear NAME Remove scratchpad/NAME only. + make [NAME] Ensure scratchpad/ exists with .gitignore; with NAME, mkdir scratchpad/NAME. + write SRC [DST] Copy file/dir SRC into scratchpad (to DST if given; parents created). + size Print 'empty' if only .gitignore; else total bytes and item count. + find [OPTS...] Run system 'find' rooted at scratchpad/ with OPTS (omit literal 'scratchpad'). + lock PATH... Attempt 'chattr +i' on given paths under scratchpad/ (no state kept). + unlock PATH... Attempt 'chattr -i' on given paths under scratchpad/. + +Examples: + scratchpad make + scratchpad write ~/Downloads/test.tar.gz + scratchpad find -type f -mtime +30 -print # files older than 30 days + scratchpad lock some/dir important.txt + scratchpad unlock some/dir important.txt +""" + +CWD = os.getcwd() +SP = os.path.join(CWD, "scratchpad") +GITIGNORE = os.path.join(SP, ".gitignore") + +def have_sp() -> bool: + return os.path.isdir(SP) + +def ensure_sp(): + os.makedirs(SP, exist_ok=True) + ensure_gitignore() + +def ensure_gitignore(): + os.makedirs(SP, exist_ok=True) + if not os.path.isfile(GITIGNORE): + with open(GITIGNORE, "w", encoding="utf-8") as f: + f.write("*\n!.gitignore\n") + +def filemode(mode: int) -> str: + try: + return stat.filemode(mode) + except Exception: + return oct(mode & 0o777) + +def owner_group(st) -> str: + try: + return f"{pwd.getpwuid(st.st_uid).pw_name}:{grp.getgrgid(st.st_gid).gr_name}" + except Exception: + return f"{st.st_uid}:{st.st_gid}" + +def rel_depth(base: str, root: str) -> int: + rel = os.path.relpath(base, root) + return 0 if rel == "." else rel.count(os.sep) + 1 + +def ls_tree(root: str) -> None: + if not have_sp(): + return + print("scratchpad/") + + def walk(path: str, indent: str, is_root: bool) -> None: + try: + it = list(os.scandir(path)) + except FileNotFoundError: + return + + dirs = [e for e in it if e.is_dir(follow_symlinks=False)] + files = [e for e in it if not e.is_dir(follow_symlinks=False)] + dirs.sort(key=lambda e: e.name) + files.sort(key=lambda e: e.name) + + if is_root: + # 1) root-level hidden files first + for f in (e for e in files if e.name.startswith(".")): + st = os.lstat(f.path) + print(f"{filemode(st.st_mode)} {owner_group(st)} {indent}{f.name}") + # 2) then directories (and recurse so children sit under the parent) + for d in dirs: + st = os.lstat(d.path) + print(f"{filemode(st.st_mode)} {owner_group(st)} {indent}{d.name}/") + walk(d.path, indent + ' ', False) + # 3) then non-hidden files + for f in (e for e in files if not e.name.startswith(".")): + st = os.lstat(f.path) + print(f"{filemode(st.st_mode)} {owner_group(st)} {indent}{f.name}") + else: + # subdirs: keep previous order (dirs first, then files; dotfiles naturally sort first) + for d in dirs: + st = os.lstat(d.path) + print(f"{filemode(st.st_mode)} {owner_group(st)} {indent}{d.name}/") + walk(d.path, indent + ' ', False) + for f in files: + st = os.lstat(f.path) + print(f"{filemode(st.st_mode)} {owner_group(st)} {indent}{f.name}") + + walk(root, " ", True) + + +def clear_all() -> None: + if not have_sp(): + return + for name in os.listdir(SP): + p = os.path.join(SP, name) + if name == ".gitignore" and os.path.isfile(p): + continue # preserve only top-level .gitignore + if os.path.isdir(p) and not os.path.islink(p): + shutil.rmtree(p, ignore_errors=True) + else: + try: os.unlink(p) + except FileNotFoundError: pass + +def clear_subdir(sub: str) -> None: + if not have_sp(): + return + target = os.path.normpath(os.path.join(SP, sub)) + try: + if os.path.commonpath([SP]) != os.path.commonpath([SP, target]): + return + except Exception: + return + if os.path.isdir(target) and not os.path.islink(target): + shutil.rmtree(target, ignore_errors=True) + +def cmd_make(args): + ensure_sp() + if args: + os.makedirs(os.path.join(SP, args[0]), exist_ok=True) + +def cmd_write(args): + if len(args) < 1: + print(HELP); return + if not have_sp(): + ensure_sp() + src = args[0] + dst = args[1] if len(args) >= 2 else (os.path.basename(src.rstrip(os.sep)) or "untitled") + dst_path = os.path.normpath(os.path.join(SP, dst)) + try: + if os.path.commonpath([SP]) != os.path.commonpath([SP, dst_path]): + print("refusing to write outside scratchpad", file=sys.stderr); return + except Exception: + print("invalid destination", file=sys.stderr); return + os.makedirs(os.path.dirname(dst_path), exist_ok=True) + if os.path.isdir(src): + if os.path.exists(dst_path): + shutil.rmtree(dst_path, ignore_errors=True) + shutil.copytree(src, dst_path, dirs_exist_ok=False) + else: + shutil.copy2(src, dst_path) + +def cmd_size(): + if not have_sp(): + return + names = os.listdir(SP) + if [n for n in names if n != ".gitignore"] == []: + print("empty"); return + total = 0; count = 0 + for base, dirs, files in os.walk(SP): + for fn in files: + if fn == ".gitignore": + continue + p = os.path.join(base, fn) + try: + total += os.path.getsize(p); count += 1 + except OSError: + pass + print(f"bytes={total} items={count}") + +def cmd_find(args): + if not have_sp(): + return + try: + subprocess.run(["find", SP] + args, check=False) + except FileNotFoundError: + print("find not available", file=sys.stderr) + +def cmd_chattr(flag: str, paths): + if not have_sp() or not paths: + return + try: + subprocess.run(["chattr", "-V"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) + except FileNotFoundError: + print("chattr not available; lock/unlock skipped", file=sys.stderr); return + for rel in paths: + target = os.path.normpath(os.path.join(SP, rel)) + try: + if os.path.commonpath([SP]) != os.path.commonpath([SP, target]): + continue + except Exception: + continue + try: + subprocess.run(["chattr", flag, target], check=False) + except Exception: + pass + +def CLI(): + if len(sys.argv) < 2: + print(HELP); return + cmd, *args = sys.argv[1:] + if cmd == "ls": + if have_sp(): ls_tree(SP) + else: return + elif cmd == "clear": + if len(args) >= 1: clear_subdir(args[0]) + else: clear_all() + elif cmd == "help": + print(HELP) + elif cmd == "make": + cmd_make(args) + elif cmd == "write": + cmd_write(args) + elif cmd == "size": + cmd_size() + elif cmd == "find": + cmd_find(args) + elif cmd == "lock": + cmd_chattr("+i", args) + elif cmd == "unlock": + cmd_chattr("-i", args) + else: + print(HELP) + +if __name__ == "__main__": + CLI() diff --git a/shared/tool/setup b/shared/tool/setup new file mode 100644 index 0000000..7dc24d3 --- /dev/null +++ b/shared/tool/setup @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +script_afp=$(realpath "${BASH_SOURCE[0]}") +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + echo "$script_afp:: This script must be sourced, not executed." + exit 1 +fi + +# without this bash takes non-matching globs literally +shopt -s nullglob + +# does not presume sharing or world permissions +umask 0077 + +# -------------------------------------------------------------------------------- +# project definition + +# actual absolute directory path for this script file + script_adp(){ + dirname "$script_afp" + } + +# This script is located at $REPO_HOME/shared/tool/setup. +# Ascend two directories to discover the absolute $REPO_HOME path. + REPO_HOME=$(dirname "$(dirname "$(script_adp)")") + echo REPO_HOME "$REPO_HOME" + + PROJECT=$(basename "$REPO_HOME") + echo PROJECT "$PROJECT" + + # set the prompt decoration to the name of the project + PROMPT_DECOR=$PROJECT + + export REPO_HOME PROJECT PROMPT_DECOR + +# -------------------------------------------------------------------------------- +# the following functions are provided for other scripts to use. +# at the top of files that make use of these functions put the following line: +# script_afp=$(realpath "${BASH_SOURCE[0]}") +# + + ## script's filename + script_fn(){ + basename "$script_afp" + } + + ## script's file path relative to $REPO_HOME + script_fp(){ + realpath --relative-to="${REPO_HOME}" "$script_afp" + } + + ## script's dirpath relative to $REPO_HOME + script_dp(){ + dirname "$(script_fp)" + } + + export -f script_adp script_fn script_dp script_fp + +# -------------------------------------------------------------------------------- +# PATH & Plugin Sourcing +# Each addition goes to the front, so the last added is searched first. +# Resulting search order, most specific first: +# shared/authored, shared/made, shared/tool, project dependencies, the rest. +# The top-level setup then puts the role's tool directory in front of these. + + # Project dependencies: source each .init file. Each prepends its own entries. + for init_file in "$REPO_HOME/shared/project-dependency/"*.init; do + if [[ -f "$init_file" ]]; then + source "$init_file" + fi + done + + PATH="$REPO_HOME/shared/tool:$PATH" + PATH="$REPO_HOME/shared/made:$PATH" + PATH="$REPO_HOME/shared/authored:$PATH" + + # Remove duplicates, keeping the first occurrence + # Done with bash built-ins only. A pipeline through awk and sed left PATH + # empty whenever either program failed to run. + clean_path(){ + local entry result="" + local -A seen=() + local -a part + IFS=: read -r -a part <<< "$PATH" + for entry in "${part[@]}"; do + [[ -z "$entry" || -n "${seen[$entry]:-}" ]] && continue + seen[$entry]=1 + result="${result:+$result:}$entry" + done + PATH="$result" + } + clean_path + + export PATH + +# -------------------------------------------------------------------------------- +# closing +# + if [[ -z "$ENV" ]]; then + export ENV=$(script_fp) + fi diff --git a/shared/tool/version b/shared/tool/version new file mode 100755 index 0000000..68977e4 --- /dev/null +++ b/shared/tool/version @@ -0,0 +1,3 @@ +echo "Harmony v3.5 2026-10-14 04:51:11 Z" + + diff --git a/release/x86_64/.githolder b/tester/.gitkeep similarity index 100% rename from release/x86_64/.githolder rename to tester/.gitkeep diff --git a/tester/authored/.gitkeep b/tester/authored/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tester/test_0/Man_In_Grey_input_acceptance.py b/tester/authored/test_0/Man_In_Grey_input_acceptance.py similarity index 100% rename from tester/test_0/Man_In_Grey_input_acceptance.py rename to tester/authored/test_0/Man_In_Grey_input_acceptance.py diff --git a/tester/test_0/cat_stage b/tester/authored/test_0/cat_stage similarity index 100% rename from tester/test_0/cat_stage rename to tester/authored/test_0/cat_stage diff --git a/tester/test_0/clean_out b/tester/authored/test_0/clean_out similarity index 100% rename from tester/test_0/clean_out rename to tester/authored/test_0/clean_out diff --git a/tester/test_0/ls_stage b/tester/authored/test_0/ls_stage similarity index 100% rename from tester/test_0/ls_stage rename to tester/authored/test_0/ls_stage diff --git a/tester/test_0/run b/tester/authored/test_0/run similarity index 100% rename from tester/test_0/run rename to tester/authored/test_0/run diff --git a/tester/test_0/run_check b/tester/authored/test_0/run_check similarity index 100% rename from tester/test_0/run_check rename to tester/authored/test_0/run_check diff --git a/tester/test_0/run_check.transcript b/tester/authored/test_0/run_check.transcript similarity index 100% rename from tester/test_0/run_check.transcript rename to tester/authored/test_0/run_check.transcript diff --git a/tester/test_0/stage_test_0/DNS/unbound.conf.py b/tester/authored/test_0/stage_test_0/DNS/unbound.conf.py similarity index 100% rename from tester/test_0/stage_test_0/DNS/unbound.conf.py rename to tester/authored/test_0/stage_test_0/DNS/unbound.conf.py diff --git a/tester/test_0/stage_test_0/unbound_conf.py b/tester/authored/test_0/stage_test_0/unbound_conf.py similarity index 100% rename from tester/test_0/stage_test_0/unbound_conf.py rename to tester/authored/test_0/stage_test_0/unbound_conf.py diff --git a/tester/test_0/stage_test_0/web/site_conf.py b/tester/authored/test_0/stage_test_0/web/site_conf.py similarity index 100% rename from tester/test_0/stage_test_0/web/site_conf.py rename to tester/authored/test_0/stage_test_0/web/site_conf.py diff --git a/tester/test_0/stage_test_0_out/net/unbound.conf b/tester/authored/test_0/stage_test_0_out/net/unbound.conf similarity index 100% rename from tester/test_0/stage_test_0_out/net/unbound.conf rename to tester/authored/test_0/stage_test_0_out/net/unbound.conf diff --git a/tester/test_0/stage_test_0_out/unbound_conf b/tester/authored/test_0/stage_test_0_out/unbound_conf similarity index 100% rename from tester/test_0/stage_test_0_out/unbound_conf rename to tester/authored/test_0/stage_test_0_out/unbound_conf diff --git a/tester/test_0/stage_test_0_out/unbound_conf.20250920T161232Z b/tester/authored/test_0/stage_test_0_out/unbound_conf.20250920T161232Z similarity index 100% rename from tester/test_0/stage_test_0_out/unbound_conf.20250920T161232Z rename to tester/authored/test_0/stage_test_0_out/unbound_conf.20250920T161232Z diff --git a/tester/test_0/stage_test_0_out/web/nginx.conf b/tester/authored/test_0/stage_test_0_out/web/nginx.conf similarity index 100% rename from tester/test_0/stage_test_0_out/web/nginx.conf rename to tester/authored/test_0/stage_test_0_out/web/nginx.conf diff --git a/tester/test_0/stage_test_0_out/web/nginx.conf.20250920T161233Z b/tester/authored/test_0/stage_test_0_out/web/nginx.conf.20250920T161233Z similarity index 100% rename from tester/test_0/stage_test_0_out/web/nginx.conf.20250920T161233Z rename to tester/authored/test_0/stage_test_0_out/web/nginx.conf.20250920T161233Z diff --git a/tester/test_0/stage_test_0_out_expected_0/net/unbound.conf b/tester/authored/test_0/stage_test_0_out_expected_0/net/unbound.conf similarity index 100% rename from tester/test_0/stage_test_0_out_expected_0/net/unbound.conf rename to tester/authored/test_0/stage_test_0_out_expected_0/net/unbound.conf diff --git a/tester/test_0/stage_test_0_out_expected_0/unbound_conf b/tester/authored/test_0/stage_test_0_out_expected_0/unbound_conf similarity index 100% rename from tester/test_0/stage_test_0_out_expected_0/unbound_conf rename to tester/authored/test_0/stage_test_0_out_expected_0/unbound_conf diff --git a/tester/test_0/stage_test_0_out_expected_0/web/nginx.conf b/tester/authored/test_0/stage_test_0_out_expected_0/web/nginx.conf similarity index 100% rename from tester/test_0/stage_test_0_out_expected_0/web/nginx.conf rename to tester/authored/test_0/stage_test_0_out_expected_0/web/nginx.conf diff --git a/tester/test_0/stage_test_0_out_expected_1/net/unbound.conf b/tester/authored/test_0/stage_test_0_out_expected_1/net/unbound.conf similarity index 100% rename from tester/test_0/stage_test_0_out_expected_1/net/unbound.conf rename to tester/authored/test_0/stage_test_0_out_expected_1/net/unbound.conf diff --git a/tester/test_0/stage_test_0_out_expected_1/unbound_conf b/tester/authored/test_0/stage_test_0_out_expected_1/unbound_conf similarity index 100% rename from tester/test_0/stage_test_0_out_expected_1/unbound_conf rename to tester/authored/test_0/stage_test_0_out_expected_1/unbound_conf diff --git a/tester/test_0/stage_test_0_out_expected_1/unbound_conf.20250920T160648Z b/tester/authored/test_0/stage_test_0_out_expected_1/unbound_conf.20250920T160648Z similarity index 100% rename from tester/test_0/stage_test_0_out_expected_1/unbound_conf.20250920T160648Z rename to tester/authored/test_0/stage_test_0_out_expected_1/unbound_conf.20250920T160648Z diff --git a/tester/test_0/stage_test_0_out_expected_1/web/nginx.conf b/tester/authored/test_0/stage_test_0_out_expected_1/web/nginx.conf similarity index 100% rename from tester/test_0/stage_test_0_out_expected_1/web/nginx.conf rename to tester/authored/test_0/stage_test_0_out_expected_1/web/nginx.conf diff --git a/tester/test_0/stage_test_0_out_expected_1/web/nginx.conf.20250920T160648Z b/tester/authored/test_0/stage_test_0_out_expected_1/web/nginx.conf.20250920T160648Z similarity index 100% rename from tester/test_0/stage_test_0_out_expected_1/web/nginx.conf.20250920T160648Z rename to tester/authored/test_0/stage_test_0_out_expected_1/web/nginx.conf.20250920T160648Z diff --git a/tester/tool/setup b/tester/tool/setup new file mode 100644 index 0000000..0b993ad --- /dev/null +++ b/tester/tool/setup @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +script_afp=$(realpath "${BASH_SOURCE[0]}") + -- 2.20.1