From: Thomas Walker Lynch Date: Wed, 29 Oct 2025 16:12:03 +0000 (+0000) Subject: updating scripts from Rabbit experience X-Git-Url: https://git.reasoningtechnology.com/style/static/gitweb.js?a=commitdiff_plain;h=9ed807dff25cf7f74ccf7ea8209e0ef94118fca8;p=Harmony.git updating scripts from Rabbit experience --- diff --git a/developer/document/tests.sh b/developer/document/tests.sh deleted file mode 100644 index 6892145..0000000 --- a/developer/document/tests.sh +++ /dev/null @@ -1,10 +0,0 @@ - - -2025-05-19T09:56:17Z[Harmony] -Thomas-developer@StanleyPark§/home/Thomas/subu_data/developer/Harmony/developer/python§ -> python test_isqrt.py 500000 20000 -.................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................... -✅ Passed: 500000 -❌ Failed: 0 - - diff --git a/developer/tool/makefile b/developer/tool/makefile index 9a068f3..cb41bd4 100644 --- a/developer/tool/makefile +++ b/developer/tool/makefile @@ -1,19 +1,54 @@ +# developer/tool/makefile — Orchestrator (Hybrid) +.SUFFIXES: +.EXPORT_ALL_VARIABLES: -RT_project_share:=$(REPO_HOME)/tool_shared/third_party/RT-project-share/release +RT_INCOMMON := $(REPO_HOME)/tool_shared/third_party/RT-project-share/release +include $(RT_INCOMMON)/make/environment_RT_1.mk -include $(RT_project_share)/make/environment_RT_0 +.PHONY: usage +usage: + @printf "Usage: make [usage|information|all|lib|CLI|kmod|clean]\n" -# To compile the example directory uncomment the following assignments: -SRCDIR_List=cc -CFLAGS+=-Icc +.PHONY: version +version: + @printf "local ----------------------------------------\n" + @echo tool/makefile version 2.0 + @printf "target_library_CLI.mk ----------------------------------------\n" + @$(MAKE) -f $(RT_INCOMMON)/make/target_kmod.mk version + @printf "target_kmod.mk ----------------------------------------\n" + @$(MAKE) -f $(RT_INCOMMON)/make/target_library_CLI.mk version -CFLAGS+= -include "$(RT_project_share)/make/RT_0.h" -LINKFLAGS+= -l$(PROJECT) -LIBFILE=$(LIBDIR)/lib$(PROJECT).a +.PHONY: information +information: + @printf "local ----------------------------------------\n" + -@echo CURDIR='$(CURDIR)' + @echo REPO_HOME="$(REPO_HOME)" + @echo KMOD_BUILD_DIR="/lib/modules/$(shell uname -r)/build" + @echo CURDIR="$(CURDIR)" + @printf "target_library_CLI.mk ----------------------------------------\n" + @$(MAKE) -f $(RT_INCOMMON)/make/target_library_CLI.mk information + @printf "target_kmod.mk ----------------------------------------\n" + @$(MAKE) -f $(RT_INCOMMON)/make/target_kmod.mk information -include $(RT_project_share)/make/targets_developer --include $(DEPFILE) +.PHONY: all +all: library CLI kmod + +.PHONY: library lib +library lib: + @$(MAKE) -f $(RT_INCOMMON)/make/target_library_CLI.mk library + +.PHONY: CLI +CLI: + @$(MAKE) -f $(RT_INCOMMON)/make/target_library_CLI.mk CLI + +.PHONY: kmod +kmod: + @$(MAKE) -f $(RT_INCOMMON)/make/target_kmod.mk kmod -# shared targets .PHONY: clean -clean: _clean_developer +clean: + @printf "local ----------------------------------------\n" + @printf "target_library_CLI.mk ----------------------------------------\n" + @$(MAKE) -f $(RT_INCOMMON)/make/target_library_CLI.mk clean + @printf "target_kmod.mk ----------------------------------------\n" + @$(MAKE) -f $(RT_INCOMMON)/make/target_kmod.mk clean diff --git a/developer/tool/release b/developer/tool/release index afffc94..e29cb43 100755 --- a/developer/tool/release +++ b/developer/tool/release @@ -1,31 +1,287 @@ -#!/usr/bin/env bash -script_afp=$(realpath "${BASH_SOURCE[0]}") +#!/usr/bin/env -S python3 -B +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- -# This is a bespoke script for copying files into ../release. -# Files in ../release can be used by the tester role. -# On a release branch, files in the ../release constitute the public release. +import os, sys, shutil, stat, pwd, grp, glob, tempfile -# conventional preliminaries -# - env_must_be="developer/tool/env" - if [ "$ENV" != "$env_must_be" ]; then - echo "$(script_fp):: error: must be run in the $env_must_be environment" - exit 1 - fi +HELP = """usage: release {write|clean|ls|help|dry write} [DIR] + write [DIR] Writes released files into $REPO_HOME/release. If [DIR] is specified, only writes files found in scratchpad/DIR. + clean [DIR] Remove the contents of the release directories. If [DIR] is specified, clean only the contents of that release directory. + ls List release/ as an indented tree: PERMS OWNER NAME (root-level dotfiles printed first). + help Show this message. + dry write [DIR] + Preview what write would do without modifying the filesystem. +""" - cd "$REPO_HOME"/developer || exit 1 +ENV_MUST_BE = "developer/tool/env" +DEFAULT_DIR_MODE = 0o750 +PERM_BY_DIR = { + "kmod": 0o440, + "machine": 0o550, + "python3": 0o550, + "shell": 0o550, +} -#set -e -#set -x +def exit_with_status(msg, code=1): + print(f"release: {msg}", file=sys.stderr) + sys.exit(code) -release_dir="$REPO_HOME/release" +def assert_env(): + env = os.environ.get("ENV", "") + if env != ENV_MUST_BE: + hint = ( + "ENV is not 'developer/tool/env'.\n" + "Enter the project with: source ./env_developer\n" + "That script exports: ROLE=developer; ENV=$ROLE/tool/env" + ) + exit_with_status(f"bad environment: ENV='{env}'. {hint}") -# examples +def repo_home(): + rh = os.environ.get("REPO_HOME") + if not rh: + exit_with_status("REPO_HOME not set (did you 'source ./env_developer'?)") + return rh - # yes machine directory, but no release/kmod directory, on the skeleton - mkdir -p ${release_dir}/kmod && install -m 0600 scratchpad/*.ko "${release_dir}/kmod/" - install -m 0700 scratchpad/hello "${release_dir}/machine/" +def dpath(*parts): + return os.path.join(repo_home(), "developer", *parts) +def rpath(*parts): + return os.path.join(repo_home(), "release", *parts) -echo "$(script_fn) done." +def dev_root(): + return dpath() +def rel_root(): + return rpath() + +def _display_src(p_abs: str) -> str: + # Developer paths shown relative to $REPO_HOME/developer + try: + if os.path.commonpath([dev_root()]) == os.path.commonpath([dev_root(), p_abs]): + return os.path.relpath(p_abs, dev_root()) + except Exception: + pass + return p_abs + +def _display_dst(p_abs: str) -> str: + # Release paths shown as literal '$REPO_HOME/release/' + try: + rel = os.path.relpath(p_abs, rel_root()) + rel = "" if rel == "." else rel + return "$REPO_HOME/release" + ("/" + rel if rel else "") + except Exception: + return p_abs + +def ensure_mode(path, mode): + try: os.chmod(path, mode) + except Exception: pass + +def ensure_dir(path, mode=DEFAULT_DIR_MODE, dry=False): + if dry: + if not os.path.isdir(path): + shown = _display_dst(path) if path.startswith(rel_root()) else ( + os.path.relpath(path, dev_root()) if path.startswith(dev_root()) else path + ) + print(f"(dry) mkdir -m {oct(mode)[2:]} '{shown}'") + return + os.makedirs(path, exist_ok=True) + ensure_mode(path, mode) + +def filemode(m): + try: return stat.filemode(m) + except Exception: return oct(m & 0o777) + +def owner_group(st): + try: return f"{pwd.getpwuid(st.st_uid).pw_name}:{grp.getgrgid(st.st_gid).gr_name}" + except Exception: return f"{st.st_uid}:{st.st_gid}" + +# ---------- LS with two-pass column width for owner:group ---------- +def list_tree(root): + if not os.path.isdir(root): + return + + # gather entries in display order, record owner:group widths + entries = [] # list of (is_dir, depth, perms, ownergrp, name) + def gather(path: str, depth: int, is_root: bool): + try: + it = list(os.scandir(path)) + except FileNotFoundError: + return + dirs = [e for e in it if e.is_dir(follow_symlinks=False)] + files = [e for e in it if not e.is_dir(follow_symlinks=False)] + dirs.sort(key=lambda e: e.name) + files.sort(key=lambda e: e.name) + + if is_root: + # root-level: dotfiles first + for f in (e for e in files if e.name.startswith(".")): + st = os.lstat(f.path) + entries.append((False, depth, filemode(st.st_mode), owner_group(st), f.name)) + for d in dirs: + st = os.lstat(d.path) + entries.append((True, depth, filemode(st.st_mode), owner_group(st), d.name + "/")) + gather(d.path, depth + 1, False) + for f in (e for e in files if not e.name.startswith(".")): + st = os.lstat(f.path) + entries.append((False, depth, filemode(st.st_mode), owner_group(st), f.name)) + else: + # subdirs: dirs then files (dotfiles naturally sort first) + for d in dirs: + st = os.lstat(d.path) + entries.append((True, depth, filemode(st.st_mode), owner_group(st), d.name + "/")) + gather(d.path, depth + 1, False) + for f in files: + st = os.lstat(f.path) + entries.append((False, depth, filemode(st.st_mode), owner_group(st), f.name)) + + gather(root, depth=1, is_root=True) + + # compute max width for owner:group column + ogw = 0 + for (_isdir, _depth, _perms, ownergrp, _name) in entries: + if len(ownergrp) > ogw: + ogw = len(ownergrp) + + # print + print("release/") + for (isdir, depth, perms, ownergrp, name) in entries: + indent = " " * depth + # perms first, owner:group padded next, then name with tree indent + print(f"{perms} {ownergrp:<{ogw}} {indent}{name}") + +# ---------- end LS ---------- + +def iter_src_files(topdir, src_root): + base = os.path.join(src_root, topdir) if topdir else src_root + if not os.path.isdir(base): + return + yield + if topdir == "kmod": + for p in sorted(glob.glob(os.path.join(base, "*.ko"))): + yield (p, os.path.basename(p)) + else: + for root, dirs, files in os.walk(base): + dirs.sort(); files.sort() + for fn in files: + src = os.path.join(root, fn) + rel = os.path.relpath(src, base) + yield (src, rel) + +def target_mode(topdir): + return PERM_BY_DIR.get(topdir, 0o440) + +def copy_one(src_abs, dst_abs, mode, dry=False): + src_show = _display_src(src_abs) + dst_show = _display_dst(dst_abs) + parent = os.path.dirname(dst_abs) + os.makedirs(parent, exist_ok=True) + + if dry: + if os.path.exists(dst_abs): + print(f"(dry) unlink '{dst_show}'") + print(f"(dry) install -m {oct(mode)[2:]} -D '{src_show}' '{dst_show}'") + return + + # Replace even if dst exists and is read-only: write temp then atomic replace. + fd, tmp_path = tempfile.mkstemp(prefix=".tmp.", dir=parent) + try: + with os.fdopen(fd, "wb") as tmpf, open(src_abs, "rb") as sf: + shutil.copyfileobj(sf, tmpf) + tmpf.flush() + os.chmod(tmp_path, mode) + os.replace(tmp_path, dst_abs) + finally: + try: + if os.path.exists(tmp_path): + os.unlink(tmp_path) + except Exception: + pass + + print(f"+ install -m {oct(mode)[2:]} '{src_show}' '{dst_show}'") + +def write_one_dir(topdir, dry): + rel_root_dir = rpath() + src_root = dpath("scratchpad") + src_dir = os.path.join(src_root, topdir) + dst_dir = os.path.join(rel_root_dir, topdir) + + if not os.path.isdir(src_dir): + exit_with_status( + f"cannot write: expected '{_display_src(src_dir)}' to exist. " + f"Create scratchpad/{topdir} (Makefiles may need to populate it)." + ) + + ensure_dir(dst_dir, DEFAULT_DIR_MODE, dry=dry) + + wrote = False + mode = target_mode(topdir) + for src_abs, rel in iter_src_files(topdir, src_root): + dst_abs = os.path.join(dst_dir, rel) + copy_one(src_abs, dst_abs, mode, dry=dry) + wrote = True + if not wrote: + msg = "no matching artifacts found" + if topdir == "kmod": msg += " (looking for *.ko)" + print(f"(info) {msg} in {_display_src(src_dir)}") + +def cmd_write(dir_arg, dry=False): + assert_env() + ensure_dir(rpath(), DEFAULT_DIR_MODE, dry=dry) + + src_root = dpath("scratchpad") + if not os.path.isdir(src_root): + exit_with_status(f"cannot find developer scratchpad at '{_display_src(src_root)}'") + + if dir_arg: + write_one_dir(dir_arg, dry=dry) + else: + subs = sorted([e.name for e in os.scandir(src_root) if e.is_dir(follow_symlinks=False)]) + if not subs: + print(f"(info) nothing to release; no subdirectories found under {_display_src(src_root)}") + return + for td in subs: + write_one_dir(td, dry=dry) + +def _clean_contents(dir_path): + if not os.path.isdir(dir_path): return + for name in os.listdir(dir_path): + p = os.path.join(dir_path, name) + if os.path.isdir(p) and not os.path.islink(p): + shutil.rmtree(p, ignore_errors=True) + else: + try: os.unlink(p) + except FileNotFoundError: pass + +def cmd_clean(dir_arg): + assert_env() + rel_root_dir = rpath() + if not os.path.isdir(rel_root_dir): + return + if dir_arg: + _clean_contents(os.path.join(rel_root_dir, dir_arg)) + else: + for e in os.scandir(rel_root_dir): + if e.is_dir(follow_symlinks=False): + _clean_contents(e.path) + +def CLI(): + if len(sys.argv) < 2: + print(HELP); return + cmd, *args = sys.argv[1:] + if cmd == "write": + cmd_write(args[0] if args else None, dry=False) + elif cmd == "clean": + cmd_clean(args[0] if args else None) + elif cmd == "ls": + list_tree(rpath()) + elif cmd == "help": + print(HELP) + elif cmd == "dry": + if args and args[0] == "write": + cmd_write(args[1] if len(args) >= 2 else None, dry=True) + else: + print(HELP) + else: + print(HELP) + +if __name__ == "__main__": + CLI() diff --git a/developer/tool/release_clean b/developer/tool/release_clean deleted file mode 100755 index fc09a13..0000000 --- a/developer/tool/release_clean +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -script_afp=$(realpath "${BASH_SOURCE[0]}") - -# before running this, make library is built and is in the scratchpad directory - -# input guards - - env_must_be="developer/tool/env" - if [ "$ENV" != "$env_must_be" ]; then - echo "$(script_fp):: error: must be run in the $env_must_be environment" - exit 1 - fi - -set -e -set -x - - cd "$REPO_HOME"/developer || exit 1 - - release_dir=$(release_dir) - - if [ ! -d ${release_dir} ]; then - echo "$(script_fp):: no release directory: " ${release_dir} - exit 1 - fi - - rm_na -rf ${release_dir}/* - -set +x -echo "$(script_fn) done." - diff --git a/tool_shared/bespoke/scratchpad b/tool_shared/bespoke/scratchpad new file mode 100755 index 0000000..f14f140 --- /dev/null +++ b/tool_shared/bespoke/scratchpad @@ -0,0 +1,225 @@ +#!/usr/bin/env -S python3 -B +# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- + +import os, sys, shutil, stat, pwd, grp, subprocess + +HELP = """usage: scratchpad {ls|clear|help|make|write|size|find|lock|unlock} [ARGS] + ls List scratchpad in an indented tree with perms and owner (quiet if missing). + clear Remove all contents of scratchpad/ except top-level .gitignore. + clear NAME Remove scratchpad/NAME only. + make [NAME] Ensure scratchpad/ exists with .gitignore; with NAME, mkdir scratchpad/NAME. + write SRC [DST] Copy file/dir SRC into scratchpad (to DST if given; parents created). + size Print 'empty' if only .gitignore; else total bytes and item count. + find [OPTS...] Run system 'find' rooted at scratchpad/ with OPTS (omit literal 'scratchpad'). + lock PATH... Attempt 'chattr +i' on given paths under scratchpad/ (no state kept). + unlock PATH... Attempt 'chattr -i' on given paths under scratchpad/. + +Examples: + scratchpad make + scratchpad write ~/Downloads/test.tar.gz + scratchpad find -type f -mtime +30 -print # files older than 30 days + scratchpad lock some/dir important.txt + scratchpad unlock some/dir important.txt +""" + +CWD = os.getcwd() +SP = os.path.join(CWD, "scratchpad") +GITIGNORE = os.path.join(SP, ".gitignore") + +def have_sp() -> bool: + return os.path.isdir(SP) + +def ensure_sp(): + os.makedirs(SP, exist_ok=True) + ensure_gitignore() + +def ensure_gitignore(): + os.makedirs(SP, exist_ok=True) + if not os.path.isfile(GITIGNORE): + with open(GITIGNORE, "w", encoding="utf-8") as f: + f.write("*\n!.gitignore\n") + +def filemode(mode: int) -> str: + try: + return stat.filemode(mode) + except Exception: + return oct(mode & 0o777) + +def owner_group(st) -> str: + try: + return f"{pwd.getpwuid(st.st_uid).pw_name}:{grp.getgrgid(st.st_gid).gr_name}" + except Exception: + return f"{st.st_uid}:{st.st_gid}" + +def rel_depth(base: str, root: str) -> int: + rel = os.path.relpath(base, root) + return 0 if rel == "." else rel.count(os.sep) + 1 + +def ls_tree(root: str) -> None: + if not have_sp(): + return + print("scratchpad/") + + def walk(path: str, indent: str, is_root: bool) -> None: + try: + it = list(os.scandir(path)) + except FileNotFoundError: + return + + dirs = [e for e in it if e.is_dir(follow_symlinks=False)] + files = [e for e in it if not e.is_dir(follow_symlinks=False)] + dirs.sort(key=lambda e: e.name) + files.sort(key=lambda e: e.name) + + if is_root: + # 1) root-level hidden files first + for f in (e for e in files if e.name.startswith(".")): + st = os.lstat(f.path) + print(f"{filemode(st.st_mode)} {owner_group(st)} {indent}{f.name}") + # 2) then directories (and recurse so children sit under the parent) + for d in dirs: + st = os.lstat(d.path) + print(f"{filemode(st.st_mode)} {owner_group(st)} {indent}{d.name}/") + walk(d.path, indent + ' ', False) + # 3) then non-hidden files + for f in (e for e in files if not e.name.startswith(".")): + st = os.lstat(f.path) + print(f"{filemode(st.st_mode)} {owner_group(st)} {indent}{f.name}") + else: + # subdirs: keep previous order (dirs first, then files; dotfiles naturally sort first) + for d in dirs: + st = os.lstat(d.path) + print(f"{filemode(st.st_mode)} {owner_group(st)} {indent}{d.name}/") + walk(d.path, indent + ' ', False) + for f in files: + st = os.lstat(f.path) + print(f"{filemode(st.st_mode)} {owner_group(st)} {indent}{f.name}") + + walk(root, " ", True) + + +def clear_all() -> None: + if not have_sp(): + return + for name in os.listdir(SP): + p = os.path.join(SP, name) + if name == ".gitignore" and os.path.isfile(p): + continue # preserve only top-level .gitignore + if os.path.isdir(p) and not os.path.islink(p): + shutil.rmtree(p, ignore_errors=True) + else: + try: os.unlink(p) + except FileNotFoundError: pass + +def clear_subdir(sub: str) -> None: + if not have_sp(): + return + target = os.path.normpath(os.path.join(SP, sub)) + try: + if os.path.commonpath([SP]) != os.path.commonpath([SP, target]): + return + except Exception: + return + if os.path.isdir(target) and not os.path.islink(target): + shutil.rmtree(target, ignore_errors=True) + +def cmd_make(args): + ensure_sp() + if args: + os.makedirs(os.path.join(SP, args[0]), exist_ok=True) + +def cmd_write(args): + if len(args) < 1: + print(HELP); return + if not have_sp(): + ensure_sp() + src = args[0] + dst = args[1] if len(args) >= 2 else (os.path.basename(src.rstrip(os.sep)) or "untitled") + dst_path = os.path.normpath(os.path.join(SP, dst)) + try: + if os.path.commonpath([SP]) != os.path.commonpath([SP, dst_path]): + print("refusing to write outside scratchpad", file=sys.stderr); return + except Exception: + print("invalid destination", file=sys.stderr); return + os.makedirs(os.path.dirname(dst_path), exist_ok=True) + if os.path.isdir(src): + if os.path.exists(dst_path): + shutil.rmtree(dst_path, ignore_errors=True) + shutil.copytree(src, dst_path, dirs_exist_ok=False) + else: + shutil.copy2(src, dst_path) + +def cmd_size(): + if not have_sp(): + return + names = os.listdir(SP) + if [n for n in names if n != ".gitignore"] == []: + print("empty"); return + total = 0; count = 0 + for base, dirs, files in os.walk(SP): + for fn in files: + if fn == ".gitignore": + continue + p = os.path.join(base, fn) + try: + total += os.path.getsize(p); count += 1 + except OSError: + pass + print(f"bytes={total} items={count}") + +def cmd_find(args): + if not have_sp(): + return + try: + subprocess.run(["find", SP] + args, check=False) + except FileNotFoundError: + print("find not available", file=sys.stderr) + +def cmd_chattr(flag: str, paths): + if not have_sp() or not paths: + return + try: + subprocess.run(["chattr", "-V"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) + except FileNotFoundError: + print("chattr not available; lock/unlock skipped", file=sys.stderr); return + for rel in paths: + target = os.path.normpath(os.path.join(SP, rel)) + try: + if os.path.commonpath([SP]) != os.path.commonpath([SP, target]): + continue + except Exception: + continue + try: + subprocess.run(["chattr", flag, target], check=False) + except Exception: + pass + +def CLI(): + if len(sys.argv) < 2: + print(HELP); return + cmd, *args = sys.argv[1:] + if cmd == "ls": + if have_sp(): ls_tree(SP) + else: return + elif cmd == "clear": + if len(args) >= 1: clear_subdir(args[0]) + else: clear_all() + elif cmd == "help": + print(HELP) + elif cmd == "make": + cmd_make(args) + elif cmd == "write": + cmd_write(args) + elif cmd == "size": + cmd_size() + elif cmd == "find": + cmd_find(args) + elif cmd == "lock": + cmd_chattr("+i", args) + elif cmd == "unlock": + cmd_chattr("-i", args) + else: + print(HELP) + +if __name__ == "__main__": + CLI()