From 5c3fff55dc6f8eed67e0b0ce7e7f1ba418d80ded Mon Sep 17 00:00:00 2001 From: Thomas Walker Lynch Date: Tue, 23 Jun 2026 14:50:15 +0000 Subject: [PATCH] improved skeleton checker --- administrator/tool/skeleton_diff | 137 +++++++- document/introduction_Harmony.html | 92 ++--- document/todo.txt | 4 - tester/RT-formatter/RT-formatter | 307 ---------------- tester/RT-formatter/RT-formatter.el | 5 - tester/RT-formatter/RT-formatter_alt.el | 30 -- tester/RT-formatter/RT-formatter_script.el | 22 -- tester/RT-formatter/RT-formatter_with-compare | 331 ------------------ .../RT-formatter/RT-formatter_with-compare.el | 23 -- tester/RT-formatter/data_test-0.c | 20 -- tester/RT-formatter/data_test-1.py | 16 - 11 files changed, 155 insertions(+), 832 deletions(-) delete mode 100644 document/todo.txt delete mode 100644 tester/RT-formatter/RT-formatter delete mode 100644 tester/RT-formatter/RT-formatter.el delete mode 100644 tester/RT-formatter/RT-formatter_alt.el delete mode 100644 tester/RT-formatter/RT-formatter_script.el delete mode 100644 tester/RT-formatter/RT-formatter_with-compare delete mode 100644 tester/RT-formatter/RT-formatter_with-compare.el delete mode 100644 tester/RT-formatter/data_test-0.c delete mode 100644 tester/RT-formatter/data_test-1.py diff --git a/administrator/tool/skeleton_diff b/administrator/tool/skeleton_diff index ad7d031..3bac840 100755 --- a/administrator/tool/skeleton_diff +++ b/administrator/tool/skeleton_diff @@ -1,12 +1,16 @@ #!/usr/bin/env python3 # -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- -import sys ,os ,filecmp +import sys ,os ,filecmp ,stat from pathlib import Path def get_project_name(repo_path: Path) -> str: return repo_path.resolve().name +def format_perms(mode: int) -> str: + # Extract the 9-character permission string (e.g., rwxr-xr-x), dropping the leading file type char + return stat.filemode(mode)[1:] + def CLI(argv=None) -> int: if argv is None: argv = sys.argv[1:] @@ -23,6 +27,33 @@ def CLI(argv=None) -> int: return 1 project_name = get_project_name(project_dir) + + # Data structures for the output sections + opus_warnings = [] + missing_files = [] + + # Time-based modification buckets + mod_harmony_newer = [] + mod_same_time = [] + mod_project_newer = [] + + perms_only_changes = [] + added_files = [] + + # --- Section 1: Opus File Structural Audit --- + project_opus = f"0pus_{project_name}" + harmony_opus = "0pus_Harmony" + found_opus = False + + for TM_item in os.listdir(project_dir): + if TM_item.startswith("0pus_") and os.path.isfile(project_dir / TM_item): + if TM_item == project_opus: + found_opus = True + else: + opus_warnings.append(f"Incorrectly named Opus file found: '{TM_item}'. Expected '{project_opus}'.") + + if not found_opus: + opus_warnings.append(f"Expected Opus file '{project_opus}' is MISSING from project root.") # Semantic boundaries: isolate the skeleton by ignoring user zones IGNORED_DIRS = {".git" ,"scratchpad" ,"consumer" ,"authored" ,"__pycache__"} @@ -52,24 +83,24 @@ def CLI(argv=None) -> int: rel_path = f_path.relative_to(project_dir) project_files.add(str(rel_path)) - # Map the dynamically generated Opus file - harmony_opus = "0pus_Harmony" - project_opus = f"0pus_{project_name}" - + # Align the Opus file names for comparison if project_opus in project_files: project_files.remove(project_opus) project_files.add(harmony_opus) + else: + for TM_item in list(project_files): + if TM_item.startswith("0pus_") and "/" not in TM_item: + project_files.remove(TM_item) all_files = sorted(list(harmony_files | project_files)) - drift_count = 0 for TM_rel in all_files: if TM_rel not in project_files: - print(f"Missing in project: {TM_rel}") - drift_count += 1 + if TM_rel == harmony_opus and not found_opus: + continue + missing_files.append(TM_rel) elif TM_rel not in harmony_files: - print(f"Orphaned in project: {TM_rel}") - drift_count += 1 + added_files.append(TM_rel) else: h_abs = harmony_dir / TM_rel p_abs = project_dir / TM_rel @@ -77,14 +108,90 @@ def CLI(argv=None) -> int: if TM_rel == harmony_opus: p_abs = project_dir / project_opus - if not filecmp.cmp(h_abs ,p_abs ,shallow=False): - print(f"Modified: {TM_rel}") - drift_count += 1 + if p_abs.exists(): + h_stat = h_abs.stat() + p_stat = p_abs.stat() + + h_perms = format_perms(h_stat.st_mode) + p_perms = format_perms(p_stat.st_mode) + perms_differ = (h_perms != p_perms) + + display_str = f"({h_perms}, {p_perms}) {TM_rel}" if perms_differ else TM_rel + content_differs = not filecmp.cmp(h_abs ,p_abs ,shallow=False) + + if content_differs: + h_time = h_stat.st_mtime + p_time = p_stat.st_mtime + + if h_time > p_time: + mod_harmony_newer.append(display_str) + elif h_time < p_time: + mod_project_newer.append(display_str) + else: + mod_same_time.append(display_str) + elif perms_differ: + perms_only_changes.append(display_str) + + # --- Output Generation --- + print(f"\nSkeleton Audit: Harmony vs {project_name}") + print("=" * 60) + + print("\n1. 0pus File Audit") + print("-" * 20) + if not opus_warnings: + print(" [OK] Opus file is correctly named and present.") + else: + for TM_w in opus_warnings: + print(f" [WARNING] {TM_w}") + print("\n2. Missing from Project (In Harmony, missing locally)") + print("-" * 20) + if not missing_files: + print(" [OK] No Harmony skeleton files are missing.") + else: + for TM_f in missing_files: + print(f" - {TM_f}") + + print("\n3. Modified Skeleton Files (Diverged from Harmony)") + print("-" * 20) + if not (mod_harmony_newer or mod_same_time or mod_project_newer): + print(" [OK] No shared skeleton files have been modified.") + else: + if mod_harmony_newer: + print(" [Harmony newer]") + for TM_f in mod_harmony_newer: + print(f" ~ {TM_f}") + if mod_same_time: + print(" [Same modification time]") + for TM_f in mod_same_time: + print(f" ~ {TM_f}") + if mod_project_newer: + print(" [Project newer]") + for TM_f in mod_project_newer: + print(f" ~ {TM_f}") + + print("\n4. Permissions Only Changes") + print("-" * 20) + if not perms_only_changes: + print(" [OK] No permission-only changes detected.") + else: + for TM_f in perms_only_changes: + print(f" * {TM_f}") + + print("\n5. Project-Specific Files (Not in Harmony)") + print("-" * 20) + if not added_files: + print(" [INFO] No additional files outside the standard skeleton.") + else: + for TM_f in added_files: + print(f" + {TM_f}") + + print("\n" + "=" * 60) + drift_count = len(opus_warnings) + len(missing_files) + len(mod_harmony_newer) + len(mod_same_time) + len(mod_project_newer) + len(perms_only_changes) if drift_count == 0: - print("Skeleton is perfectly synchronized with Harmony.") + print("Result: Skeleton is perfectly synchronized with Harmony.") else: - print(f"\nTotal drift: {drift_count} files.") + print(f"Total Structural Drift (Sections 1-4): {drift_count} issues.") return 0 diff --git a/document/introduction_Harmony.html b/document/introduction_Harmony.html index 1bf20f4..2399686 100644 --- a/document/introduction_Harmony.html +++ b/document/introduction_Harmony.html @@ -12,7 +12,7 @@ @@ -97,6 +97,7 @@ @@ -221,8 +215,8 @@
  • authored/ : Human-written source. Tracked by Git.
  • made/ : Tracked artifacts generated by tools (e.g., links to CLI entry points).
  • experiment/ : Try-it-here code. Short-lived spot testing.
    • -
  • scratchpad/ : Git-ignored directory. Holds all intermediate build outputs, including the made directory for promotions.
    • -
  • tool/ : Developer-specific tools (like the promote and make scripts).
    • +
  • scratchpad/ : Git-ignored directory. Holds all intermediate build outputs, including the staged namespace directories for promotions.
    • +
  • tool/ : Developer-specific tools (like the promote and build scripts).

    • The tester work area

    @@ -232,15 +226,12 @@

    The shared tree

    - This directory contains ecosystem tools and global environments available to all roles. This includes the shared tool directory, as well as third-party installations (like Python virtual environments or compilers) required by the project. To assist in project specific modifications to the Harmony skeleton, Harmony comes with an empty shared/authored directory that is listed earlier in the executable search path than shared/tool. + This directory contains ecosystem tools and global environments available to all roles. This includes the shared tool directory, as well as third-party symlinks and .init scripts required by the project. To assist in project specific modifications to the Harmony skeleton, Harmony comes with an empty shared/authored directory that is listed earlier in the executable search path than shared/tool.

    The consumer tree

    - The consumer/made/ tree is where developers put work product that is ready to be consumed. The entire consumer/made directory is git-ignored and treated as a transient deployment target. -

    -

    - Artifacts arrive in the consumer/made/ tree only when the promote script is invoked, which performs a flat-copy from the developer's scratchpad/made directory. + The consumer/ tree is where developers put work product that is ready to be consumed. The entire directory is git-ignored and treated as a transient deployment target. Artifacts arrive in the consumer/ tree only when the promote script is invoked, which performs a flat-copy from the developer's scratchpad/made directory.

    Document directories

    @@ -249,7 +240,7 @@

    - Note that the consumer/made directory is untracked by git and maintained by a tool. (Said tool is developer/tool/promote. It is owned and used by the developer as part of the build process.) Documents that are being promoted for eventual release, and appear in the made directory originate from somewhere from the developer/authored directory depending on how the developer has organized his workspace, but probably in developer/authored/document. Perhaps a future version of Harmony will have a tech-writer role, but that is not the case now. -

    - -

    - Currently, our developers write documents directly in HTML using the RT semantic tags. See the RT-style-JS_public project and the documentation there. A common approach is to copy another document and the setup.js file, then to type over the top of that other document. Only one setup.js file is used per directory. Be sure to edit the relative root path found at the top of setup.js. Plain text, emacs org, and mark down have all been used in prior versions of Harmony. + Currently, our developers write documents directly in HTML using the RT semantic tags. See the RT-Style project and the documentation there. A common approach is to copy another document and the setup.js file, then to type over the top of that other document.

    Untracked directories

      -
    1. consumer/made/
      2. -
    2. shared/linked-project/
      3. +
    3. consumer/ (Excluding the base .gitignore)
      4. +
    4. shared/linked-project/ (Excluding .init and .gitignore files)
    5. **/scratchpad/
    @@ -276,10 +263,10 @@

    Developer promotion and project releases

    - As a first step, a developer creates a promotion candidate inside of the consumer/made/ directory. This is typically done by running make to stage the artifacts into scratchpad/made, where they can be experimented on, followed by running promote write. The developer will often modify the versions of one or both of those tools that come with the Harmony skeleton. The promotion candidate remains stable until the next promotion. + As a first step, a developer creates a promotion candidate inside of the consumer/ directory. This is typically done by running build to stage the artifacts into scratchpad/made, followed by running promote write. The developer will often modify the versions of one or both of those tools that come with the Harmony skeleton. The promotion candidate remains stable until the next promotion.

    - Then the tester runs tests on the promotion candidate. Tests must only read from the consumer/made/ directory, though local copies can be made and edited as experiments. Currently bugs are filed using an external issues tool. + Then the tester runs tests on the promotion candidate. Tests must only read from the consumer/ directory, though local copies can be made and edited as experiments.

    It is common for a developer to open a second window on his desktop, and then enter the project as a tester in that second window. The developer can then make a promotion candidate, run the tests, edit source code, and perhaps tests, and then quickly spin through the test-debug-fix-promote cycle repeatedly. @@ -288,10 +275,10 @@ When the product manager determines the work product to be sufficiently reliable and feature rich, the administrator will make a project release. He will do this by creating a branch called release_v<major> and tagging it. The major release numbers go up incrementally.

    -

    The version 2.2 Harmony directory tree

    +

    The Harmony directory tree

    - 2026-03-09 01:42:16 Z [Harmony:administrator] Thomas_developer@StanleyPark + 2026-06-23 13:41:45 Z [Harmony:administrator] Thomas_developer@StanleyPark §/home/Thomas/subu_data/developer/project§ > tree Harmony Harmony @@ -304,14 +291,10 @@ │ ├── archive │ └── setup ├── consumer - │ ├── scratchpad - │ └── tool - │ └── env + │ └── .gitignore ├── developer │ ├── authored │ │ └── hello.CLI.c - - │ ├── document │ │ ├── 02_RT_Code_Format.html │ │ ├── 03_Naming_and_Directory_Conventions.html @@ -322,7 +305,7 @@ │ ├── scratchpad │ └── tool │ ├── do_all - │ ├── make + │ ├── build │ ├── makefile │ ├── promote │ └── setup @@ -331,29 +314,26 @@ │ ├── role-and-workflow_product-development.html │ └── setup.js ├── LICENSE - - ├── README.md ├── scratchpad - │ ├── Harmony__79f9d52__2026-03-07_085628Z.tar - │ └── Harmony__e665bb7__2026-03-09_013712Z.tar ├── setup ├── shared │ ├── authored │ ├── document - │ │ ├── installation_generic.org - │ │ ├── installation_Python.org + │ │ ├── install_generic.org + │ │ ├── install_Python.org │ │ └── setup.js │ ├── made │ ├── dictionary_style-directory.js - │ ├── third_party - │ │ ├── RT-style-JS_public -> ../../../RT-style-JS_public/ - │ │ └── upstream + │ ├── linked-project + │ │ ├── project -> ../../../ + │ │ ├── Python-3.12.3 -> project/Python-3.12.3 + │ │ ├── Python.init + │ │ ├── RT-Style -> project/RT-Style/consumer + │ │ └── RT-Style.init │ └── tool │ ├── scratchpad │ ├── setup - - │ ├── style │ └── version └── tester @@ -366,12 +346,6 @@ │ └── data_test-1.py └── tool └── setup - - 30 directories, 36 files - - 2026-03-09 01:42:19 Z [Harmony:administrator] Thomas_developer@StanleyPark - §/home/Thomas/subu_data/developer/project§ - >     diff --git a/document/todo.txt b/document/todo.txt deleted file mode 100644 index 5e09e0e..0000000 --- a/document/todo.txt +++ /dev/null @@ -1,4 +0,0 @@ -2026-03-26 03:41:44 - -when making a skeleton from Harmony, set skeleton docs and other files not to be edited to read only - diff --git a/tester/RT-formatter/RT-formatter b/tester/RT-formatter/RT-formatter deleted file mode 100644 index 0451fcb..0000000 --- a/tester/RT-formatter/RT-formatter +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/env -S python3 -B -# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- -""" -RT_Format — Reasoning Technology code formatter (Shallow Tokenizer) - -Commands: - RT_Format write [--lisp] Format files in place (rewrite originals) - RT_Format copy [--lisp] Save backups as ~ then format originals - RT_Format pipe [--lisp] Read from stdin, write to stdout - RT_Format self_test Run built-in tests - RT_Format version Show tool version - RT_Format help | --help Show usage -""" - -import sys ,re ,shutil ,os -from typing import List ,Tuple ,Optional ,TextIO - -RTF_VERSION = "0.4.0-tokenized" - -USAGE = """\ -Usage: - RT_Format write [--lisp] - RT_Format copy [--lisp] - RT_Format pipe [--lisp] - RT_Format self_test - RT_Format version - RT_Format help | --help -""" - -BR_OPEN = "([{<" -BR_CLOSE = ")]}>" -PAIR = dict( zip(BR_OPEN ,BR_CLOSE) ) -REV = dict( zip(BR_CLOSE ,BR_OPEN) ) - -# --------------- Lexer ---------------- - -class RT_Token: - def __init__(self ,kind: str ,text: str): - self.kind = kind - self.text = text - - def __repr__(self): - return f"<{self.kind}:{repr(self.text)}>" - -# The regex prioritizes exact matches. -# Comments include //, #, and /* ... */ blocks. -# Strings include Python '''/""" blocks, plus standard single/double quotes. -TOKEN_REGEX = re.compile( - r'(?P//[^\n]*|#[^\n]*|(?s:/\*.*?\*/))' - r'|(?P"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\')' - r'|(?P[ \t]+)' - r'|(?P\n)' - r'|(?P,)' - r'|(?P[\[\(\{<])' - r'|(?P[\]\)\}>])' - r'|(?P[^ \t\n,\[\(\{<\]\)\}>"\'#/]+|/)' -) - -def tokenize(text: str) -> List[RT_Token]: - tokens = [] - for TM_match in TOKEN_REGEX.finditer(text): - kind = TM_match.lastgroup - text_val = TM_match.group(kind) - tokens.append( RT_Token(kind ,text_val) ) - return tokens - -def group_lines( tokens: List[RT_Token] ) -> List[ List[RT_Token] ]: - lines = [] - current = [] - for TM_tok in tokens: - current.append(TM_tok) - if TM_tok.kind == "NEWLINE": - lines.append(current) - current = [] - if current: - lines.append(current) - return lines - -# --------------- Formatting Passes ---------------- - -def pass_vertical_commas( lines: List[List[RT_Token]] ) -> None: - for TM_idx in range( len(lines) - 1 ): - current_line = lines[TM_idx] - - # Find the last significant token - last_sig_idx = -1 - for TM_i in range( len(current_line) - 1 ,-1 ,-1 ): - if current_line[TM_i].kind not in ("SPACE" ,"NEWLINE" ,"COMMENT"): - last_sig_idx = TM_i - break - - if last_sig_idx>= 0 and current_line[last_sig_idx].kind == "COMMA": - # Remove the trailing comma - comma_tok = current_line.pop(last_sig_idx) - - # Migrate to the next line with code - for TM_j in range( TM_idx + 1 ,len(lines) ): - next_line = lines[TM_j] - first_sig_idx = -1 - for TM_k ,TM_tok in enumerate(next_line): - if TM_tok.kind not in ("SPACE" ,"NEWLINE" ,"COMMENT"): - first_sig_idx = TM_k - break - - if first_sig_idx>= 0: - next_line.insert(first_sig_idx ,comma_tok) - break - -def pass_horizontal_commas( line: List[RT_Token] ) -> None: - new_line = [] - for TM_tok in line: - if TM_tok.kind == "COMMA": - is_vertical = all(t.kind == "SPACE" for t in new_line) - if not is_vertical: - while new_line and new_line[-1].kind == "SPACE": - new_line.pop() - if new_line: - new_line.append( RT_Token("SPACE" ," ") ) - new_line.append(TM_tok) - elif TM_tok.kind == "SPACE": - if new_line and new_line[-1].kind == "COMMA": - continue # Drop space after comma - new_line.append(TM_tok) - else: - new_line.append(TM_tok) - line[:] = new_line - -def pass_tighten_brackets( line: List[RT_Token] ) -> None: - new_line = [] - for TM_tok in line: - if TM_tok.kind == "SPACE": - if new_line and new_line[-1].kind == "BR_OPEN": - continue - new_line.append(TM_tok) - elif TM_tok.kind == "BR_CLOSE": - while new_line and new_line[-1].kind == "SPACE": - new_line.pop() - new_line.append(TM_tok) - else: - new_line.append(TM_tok) - line[:] = new_line - -def get_bracket_spans( line: List[RT_Token] ) -> List[ Tuple[int ,int] ]: - stack = [] - spans = [] - for TM_i ,TM_tok in enumerate(line): - if TM_tok.kind == "BR_OPEN": - stack.append( (TM_tok.text ,TM_i) ) - elif TM_tok.kind == "BR_CLOSE": - if stack and REV[TM_tok.text] == stack[-1][0]: - _ ,pos = stack.pop() - if not stack: - spans.append( (pos ,TM_i) ) - return spans - -def contains_inner_brackets( line: List[RT_Token] ,start: int ,end: int ) -> bool: - for TM_i in range(start + 1 ,end): - if line[TM_i].kind in ("BR_OPEN" ,"BR_CLOSE"): - return True - return False - -def pass_pad_outermost( line: List[RT_Token] ,is_lisp: bool ) -> None: - if is_lisp: - return - - while True: - spans = get_bracket_spans(line) - changed = False - - # Process from right to left to avoid shifting indices - for TM_start ,TM_end in reversed(spans): - if contains_inner_brackets(line ,TM_start ,TM_end): - left_has = (TM_start + 1 = 0 ) and ( line[TM_end - 1].kind == "SPACE" ) - - if not left_has or not right_has: - if not right_has: - line.insert( TM_end ,RT_Token("SPACE" ," ") ) - if not left_has: - line.insert( TM_start + 1 ,RT_Token("SPACE" ," ") ) - changed = True - break # Re-evaluate spans after mutation - if not changed: - break - -# --------------- Public API ---------------- - -def format_tokens( tokens: List[RT_Token] ,is_lisp: bool ) -> str: - lines = group_lines(tokens) - pass_vertical_commas(lines) - - for TM_line in lines: - pass_horizontal_commas(TM_line) - pass_tighten_brackets(TM_line) - pass_pad_outermost(TM_line ,is_lisp) - - return "".join(t.text for TM_line in lines for t in TM_line) - -def rt_format_text(text: str ,is_lisp: bool) -> str: - tokens = tokenize(text) - return format_tokens(tokens ,is_lisp) - -def rt_format_stream(inp: TextIO ,out: TextIO ,is_lisp: bool) -> None: - text = inp.read() - out.write( rt_format_text(text ,is_lisp) ) - -# --------------- Self-test ---------------- - -def run_self_test() -> bool: - ok = True - def chk(src ,exp): - nonlocal ok - got = rt_format_text(src ,False) - if got != exp: - print("FAIL:\n" + src + "\n=>\n" + got + "\nexpected:\n" + exp) - ok = False - - chk("a,b,c" ,"a ,b ,c") - chk("a , b , c" ,"a ,b ,c") - chk(" ,vertical_arg" ," ,vertical_arg") - - chk("int a=0,\n b=1,\n c=2;" ,"int a=0\n ,b=1\n ,c=2;") - - chk("f ( x )" ,"f(x)") - chk("f(x) + g(y)" ,"f(x) + g(y)") - chk(" {" ," {") - - src = "int g(){int a=0,b=1,c=2; return h(a,b,c);}" - exp = "int g(){ int a=0 ,b=1 ,c=2; return h(a ,b ,c); }" - chk(src ,exp) - - chk("outer( inner(a,b) )" ,"outer( inner(a ,b) )") - chk("compute(x, f(y" ,"compute( x ,f(y") # Tolerant fragment fallback omitted for brevity, but structurally sound. - - print("SELFTEST OK" if ok else "SELFTEST FAILED") - return ok - -# --------------- CLI ---------------- - -def write_files( paths: List[str] ,is_lisp: bool ) -> int: - for TM_path in paths: - with open(TM_path ,"r" ,encoding="utf-8") as f: - data = f.read() - formatted = rt_format_text(data ,is_lisp) - with open(TM_path ,"w" ,encoding="utf-8") as f: - f.write(formatted) - return 0 - -def copy_files( paths: List[str] ,is_lisp: bool ) -> int: - for TM_path in paths: - shutil.copy2(TM_path ,TM_path + "~") - return write_files(paths ,is_lisp) - -def get_usage() -> str: - prog_name = os.path.basename( sys.argv[0] ) - return f"""\ -Usage: - {prog_name} write [--lisp] - {prog_name} copy [--lisp] - {prog_name} pipe [--lisp] - {prog_name} self_test - {prog_name} version - {prog_name} help | --help -""" - -def CLI(argv=None) -> int: - args = list( sys.argv[1:] if argv is None else argv ) - usage_text = get_usage() - - if not args or args[0] in {"help" ,"--help" ,"-h"}: - print(usage_text) - return 0 - - is_lisp = "--lisp" in args - args = [TM_a for TM_a in args if TM_a != "--lisp"] - - if not args: - return 0 - - cmd = args[0] - rest = args[1:] - - if cmd == "version": - print(RT_FORMAT_VERSION) - return 0 - if cmd == "self_test": - ok = run_self_test() - return 0 if ok else 1 - if cmd == "pipe": - rt_format_stream(sys.stdin ,sys.stdout ,is_lisp) - return 0 - if cmd == "write": - if not rest: - print("write: missing \n" + usage_text) - return 2 - return write_files(rest ,is_lisp) - if cmd == "copy": - if not rest: - print("copy: missing \n" + usage_text) - return 2 - return copy_files(rest ,is_lisp) - - print(f"Unknown command: {cmd}\n" + usage_text) - return 2 - -if __name__ == "__main__": - sys.exit( CLI() ) \ No newline at end of file diff --git a/tester/RT-formatter/RT-formatter.el b/tester/RT-formatter/RT-formatter.el deleted file mode 100644 index 94dfaaa..0000000 --- a/tester/RT-formatter/RT-formatter.el +++ /dev/null @@ -1,5 +0,0 @@ -( defun RT-format-buffer() - (interactive) - (save-excursion - ( shell-command-on-region(point-min)(point-max) - "RT-formatter pipe" t t)) ) diff --git a/tester/RT-formatter/RT-formatter_alt.el b/tester/RT-formatter/RT-formatter_alt.el deleted file mode 100644 index dd0669e..0000000 --- a/tester/RT-formatter/RT-formatter_alt.el +++ /dev/null @@ -1,30 +0,0 @@ - -(defun RT-formatter-buffer () - "Format the current buffer using RTfmt." - (interactive) - (if (not (executable-find "RT-formatter")) - (message "Error: RTfmt executable not found in PATH.") - (let ((temp-buffer (generate-new-buffer " *RTfmt*")) - (args (list "pipe"))) - (when (derived-mode-p 'emacs-lisp-mode 'lisp-mode) - (setq args (append args (list "--lisp")))) - (unwind-protect - (let ((exit-code (apply #'call-process-region - (point-min) (point-max) - "RT-formatter" - nil temp-buffer nil - args))) - (if (zerop exit-code) - (let ((formatted-text (with-current-buffer temp-buffer (buffer-string)))) - (save-excursion - (delete-region (point-min) (point-max)) - (insert formatted-text)) - (message "RT-formatter formatting successful.")) - (message "RT-formatter failed with exit code %s. Buffer unchanged." exit-code))) - (kill-buffer temp-buffer))))) - -;; ( defun RT-format-buffer() -;; (interactive) -;; (save-excursion -;; ( shell-command-on-region(point-min)(point-max) -;; "RT-formatter pipe" t t)) ) diff --git a/tester/RT-formatter/RT-formatter_script.el b/tester/RT-formatter/RT-formatter_script.el deleted file mode 100644 index 45ff29b..0000000 --- a/tester/RT-formatter/RT-formatter_script.el +++ /dev/null @@ -1,22 +0,0 @@ -(defun RT-formatter-buffer () - "Format the current buffer using RTfmt." - (interactive) - (if (not (executable-find "RT-formatter")) - (message "Error: RTfmt executable not found in PATH.") - (let ((temp-buffer (generate-new-buffer " *RTfmt*")) - (args (list "pipe"))) - (when (derived-mode-p 'emacs-lisp-mode 'lisp-mode) - (setq args (append args (list "--lisp")))) - (unwind-protect - (let ((exit-code (apply #'call-process-region - (point-min) (point-max) - "RT-formatter" - nil temp-buffer nil - args))) - (if (zerop exit-code) - (progn - ;; Applies a non-destructive diff, preserving point and markers natively - (replace-buffer-contents temp-buffer) - (message "RT-formatter formatting successful.")) - (message "RT-formatter failed with exit code %s. Buffer unchanged." exit-code))) - (kill-buffer temp-buffer))))) diff --git a/tester/RT-formatter/RT-formatter_with-compare b/tester/RT-formatter/RT-formatter_with-compare deleted file mode 100644 index 09ee7a5..0000000 --- a/tester/RT-formatter/RT-formatter_with-compare +++ /dev/null @@ -1,331 +0,0 @@ -#!/usr/bin/env -S python3 -B -# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*- -""" -RTfmt — Reasoning Technology code formatter (Predicate Tokenizer) - -Commands: - RTfmt write [--lisp] Format files in place (rewrite originals) - RTfmt copy [--lisp] Save backups as ~ then format originals - RT-formatter pipe [--lisp] Read from stdin, write to stdout - RTfmt self_test Run built-in tests - RTfmt version Show tool version - RTfmt help | --help Show usage -""" - -import sys ,re ,shutil ,os -from typing import List ,Tuple ,Optional ,TextIO - -RTF_VERSION = "0.5.0-predicate" - -def get_usage() -> str: - prog_name = os.path.basename(sys.argv[0]) - return f"""\ -Usage: - {prog_name} write [--lisp] - {prog_name} copy [--lisp] - {prog_name} pipe [--lisp] - {prog_name} self_test - {prog_name} version - {prog_name} help | --help -""" - -# Removed < and > so they are treated as standard CODE operators -BR_OPEN = "([{" -BR_CLOSE = ")]}" -PAIR = dict( zip(BR_OPEN ,BR_CLOSE) ) -REV = dict( zip(BR_CLOSE ,BR_OPEN) ) - -# --------------- Lexer ---------------- - -class RT_Token: - def __init__(self ,kind: str ,text: str): - self.kind = kind - self.text = text - - def __repr__(self): - return f"<{self.kind}:{repr(self.text)}>" - -TOKEN_REGEX = re.compile( - r'(?P//[^\n]*|#[^\n]*|(?s:/\*.*?\*/))' - r'|(?P"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\')' - r'|(?P[ \t]+)' - r'|(?P\n)' - r'|(?P,)' - r'|(?P[\[\(\{])' - r'|(?P[\]\)\}])' - r'|(?P[^ \t\n,\[\(\{\]\)\}"\'#/]+|/)' -) - -def tokenize(text: str) -> List[RT_Token]: - tokens = [] - for TM_match in TOKEN_REGEX.finditer(text): - kind = TM_match.lastgroup - text_val = TM_match.group(kind) - tokens.append( RT_Token(kind ,text_val) ) - return tokens - -# --------------- Intelligence API ---------------- - -class TokenStream: - def __init__(self ,tokens: List[RT_Token]): - self.tokens = tokens - - def get_token(self ,index: int) -> Optional[RT_Token]: - if 0 <= index < len(self.tokens): - return self.tokens[index] - return None - - def next_sig_index(self ,index: int) -> Optional[int]: - for TM_i in range(index + 1 ,len(self.tokens)): - if self.tokens[TM_i].kind not in ("SPACE" ,"NEWLINE" ,"COMMENT"): - return TM_i - return None - - def is_first_on_line(self ,index: int) -> bool: - for TM_i in range(index - 1 ,-1 ,-1): - k = self.tokens[TM_i].kind - if k == "NEWLINE": - return True - if k != "SPACE": - return False - return True # Start of file - - def indent_of_line(self ,index: int) -> str: - for TM_i in range(index ,-1 ,-1): - if self.tokens[TM_i].kind == "NEWLINE": - if TM_i + 1 < len(self.tokens) and self.tokens[TM_i + 1].kind == "SPACE": - return self.tokens[TM_i + 1].text - return "" - if self.tokens and self.tokens[0].kind == "SPACE": - return self.tokens[0].text - return "" - - def indent_of_left_match(self ,index: int) -> Optional[str]: - tok = self.get_token(index) - if not tok or tok.kind != "BR_CLOSE": - return None - target_opener = REV[tok.text] - depth = 0 - for TM_i in range(index - 1 ,-1 ,-1): - t = self.tokens[TM_i] - if t.kind == "BR_CLOSE": - depth += 1 - elif t.kind == "BR_OPEN": - if depth > 0: - depth -= 1 - elif t.text == target_opener: - return self.indent_of_line(TM_i) - return None - -# --------------- Rule Engine ---------------- - -def rule_migrate_vertical_commas(stream: TokenStream): - TM_i = 0 - while TM_i < len(stream.tokens): - if stream.tokens[TM_i].kind == "COMMA": - is_trailing = False - next_sig = stream.next_sig_index(TM_i) - if next_sig is not None: - for TM_j in range(TM_i + 1 ,next_sig): - if stream.tokens[TM_j].kind == "NEWLINE": - is_trailing = True - break - - if is_trailing: - comma_tok = stream.tokens.pop(TM_i) - next_sig -= 1 # Shifted because of pop - stream.tokens.insert(next_sig ,comma_tok) - continue - TM_i += 1 - -def rule_format_horizontal_commas(stream: TokenStream): - for TM_i in range(len(stream.tokens) - 1 ,-1 ,-1): - if stream.tokens[TM_i].kind == "COMMA": - if stream.is_first_on_line(TM_i): - continue - - next_tok = stream.get_token(TM_i + 1) - if next_tok and next_tok.kind == "SPACE": - stream.tokens.pop(TM_i + 1) - - prev_tok = stream.get_token(TM_i - 1) - if prev_tok and prev_tok.kind == "SPACE": - if prev_tok.text != " ": - prev_tok.text = " " - else: - stream.tokens.insert(TM_i ,RT_Token("SPACE" ," ")) - -def rule_fix_closing_indent(stream: TokenStream): - for TM_i in range(len(stream.tokens) - 1 ,-1 ,-1): - if stream.tokens[TM_i].kind == "BR_CLOSE" and stream.is_first_on_line(TM_i): - target_indent = stream.indent_of_left_match(TM_i) - if target_indent is not None: - prev = stream.get_token(TM_i - 1) - if prev and prev.kind == "SPACE": - prev.text = target_indent - else: - stream.tokens.insert(TM_i ,RT_Token("SPACE" ,target_indent)) - -def rule_tighten_brackets(stream: TokenStream): - for TM_i in range(len(stream.tokens) - 1 ,-1 ,-1): - if stream.tokens[TM_i].kind == "SPACE" and not stream.is_first_on_line(TM_i): - prev_t = stream.get_token(TM_i - 1) - next_t = stream.get_token(TM_i + 1) - if (prev_t and prev_t.kind == "BR_OPEN") or (next_t and next_t.kind == "BR_CLOSE"): - stream.tokens.pop(TM_i) - -def get_bracket_spans(stream: TokenStream) -> List[Tuple[int ,int]]: - stack = [] - spans = [] - for TM_i ,tok in enumerate(stream.tokens): - if tok.kind == "BR_OPEN": - stack.append( (tok.text ,TM_i) ) - elif tok.kind == "BR_CLOSE": - if stack and REV[tok.text] == stack[-1][0]: - _ ,pos = stack.pop() - if not stack: - spans.append( (pos ,TM_i) ) - return spans - -def rule_pad_outermost(stream: TokenStream ,is_lisp: bool): - if is_lisp: - return - while True: - spans = get_bracket_spans(stream) - changed = False - for TM_start ,TM_end in reversed(spans): - has_inner = False - for TM_k in range(TM_start + 1 ,TM_end): - if stream.tokens[TM_k].kind in ("BR_OPEN" ,"BR_CLOSE"): - has_inner = True - break - - if has_inner: - left_has = (TM_start + 1 < len(stream.tokens)) and stream.tokens[TM_start + 1].kind == "SPACE" - right_has = (TM_end - 1 >= 0) and stream.tokens[TM_end - 1].kind == "SPACE" - if not left_has or not right_has: - if not right_has: - stream.tokens.insert(TM_end ,RT_Token("SPACE" ," ")) - if not left_has: - stream.tokens.insert(TM_start + 1 ,RT_Token("SPACE" ," ")) - changed = True - break - if not changed: - break - -# --------------- Public API ---------------- - -def format_tokens(tokens: List[RT_Token] ,is_lisp: bool) -> str: - stream = TokenStream(tokens) - - rule_migrate_vertical_commas(stream) - rule_format_horizontal_commas(stream) - rule_tighten_brackets(stream) - rule_fix_closing_indent(stream) - rule_pad_outermost(stream ,is_lisp) - - return "".join(t.text for t in stream.tokens) - -def rt_format_text(text: str ,is_lisp: bool) -> str: - tokens = tokenize(text) - return format_tokens(tokens ,is_lisp) - -def rt_format_stream(inp: TextIO ,out: TextIO ,is_lisp: bool) -> None: - text = inp.read() - out.write( rt_format_text(text ,is_lisp) ) - -# --------------- Self-test ---------------- - -def run_self_test() -> bool: - ok = True - def chk(src ,exp): - nonlocal ok - got = rt_format_text(src ,False) - if got != exp: - print("FAIL:\n" + src + "\n=>\n" + got + "\nexpected:\n" + exp) - ok = False - - chk("a,b,c" ,"a ,b ,c") - chk("a , b , c" ,"a ,b ,c") - chk(" ,vertical_arg" ," ,vertical_arg") - - chk("int a=0,\n b=1,\n c=2;" ,"int a=0\n ,b=1\n ,c=2;") - - chk("f ( x )" ,"f(x)") - chk("f(x) + g(y)" ,"f(x) + g(y)") - chk(" {" ," {") - - src = "int g(){int a=0,b=1,c=2; return h(a,b,c);}" - exp = "int g(){ int a=0 ,b=1 ,c=2; return h(a ,b ,c); }" - chk(src ,exp) - - chk("outer( inner(a,b) )" ,"outer( inner(a ,b) )") - - # Operator protection check - chk("for(int TM = 0; TM < count; ++TM)" ,"for(int TM = 0; TM < count; ++TM)") - - print("SELFTEST OK" if ok else "SELFTEST FAILED") - return ok - -# --------------- CLI ---------------- -def write_files(paths: List[str] ,is_lisp: bool) -> int: - for TM_path in paths: - with open(TM_path ,"r" ,encoding="utf-8") as f: - data = f.read() - - formatted = rt_format_text(data ,is_lisp) - - # Only touch the file if the content actually changed - if data != formatted: - with open(TM_path ,"w" ,encoding="utf-8") as f: - f.write(formatted) - print(f"Formatted: {TM_path}") - return 0 - -def copy_files(paths: List[str] ,is_lisp: bool) -> int: - for TM_path in paths: - shutil.copy2(TM_path ,TM_path + "~") - return write_files(paths ,is_lisp) - -def CLI(argv=None) -> int: - args = list(sys.argv[1:] if argv is None else argv) - usage_text = get_usage() - - if not args or args[0] in {"help" ,"--help" ,"-h"}: - print(usage_text) - return 0 - - is_lisp = "--lisp" in args - args = [TM_a for TM_a in args if TM_a != "--lisp"] - - if not args: - return 0 - - cmd = args[0] - rest = args[1:] - - if cmd == "version": - print(RTF_VERSION) - return 0 - if cmd == "self_test": - ok = run_self_test() - return 0 if ok else 1 - if cmd == "pipe": - rt_format_stream(sys.stdin ,sys.stdout ,is_lisp) - return 0 - if cmd == "write": - if not rest: - print("write: missing \n" + usage_text) - return 2 - return write_files(rest ,is_lisp) - if cmd == "copy": - if not rest: - print("copy: missing \n" + usage_text) - return 2 - return copy_files(rest ,is_lisp) - - print(f"Unknown command: {cmd}\n" + usage_text) - return 2 - -if __name__ == "__main__": - sys.exit( CLI() ) diff --git a/tester/RT-formatter/RT-formatter_with-compare.el b/tester/RT-formatter/RT-formatter_with-compare.el deleted file mode 100644 index 36213ae..0000000 --- a/tester/RT-formatter/RT-formatter_with-compare.el +++ /dev/null @@ -1,23 +0,0 @@ -(defun RT-formatter-buffer () - "Format the current buffer using RTfmt." - (interactive) - (if (not (executable-find "RT-formatter")) - (message "Error: RTfmt executable not found in PATH.") - (let ((temp-buffer (generate-new-buffer " *RTfmt*")) - (args (list "pipe"))) - (when (derived-mode-p 'emacs-lisp-mode 'lisp-mode) - (setq args (append args (list "--lisp")))) - (unwind-protect - (let ((exit-code (apply #'call-process-region - (point-min) (point-max) - "RT-formatter" - nil temp-buffer nil - args))) - (if (zerop exit-code) - ;; Check if the formatted text is actually different - (if (= (compare-buffer-substrings nil nil nil temp-buffer nil nil) 0) - (message "RTfmt: Already perfectly formatted.") - (replace-buffer-contents temp-buffer) - (message "RT-formatter formatting successful.")) - (message "RT-formatter failed with exit code %s. Buffer unchanged." exit-code))) - (kill-buffer temp-buffer))))) diff --git a/tester/RT-formatter/data_test-0.c b/tester/RT-formatter/data_test-0.c deleted file mode 100644 index c877406..0000000 --- a/tester/RT-formatter/data_test-0.c +++ /dev/null @@ -1,20 +0,0 @@ -// commas and simple tight brackets -int g(){ - int a=0 , - b=1 , - c=2; - return h(a ,b ,c); -} - -// balanced outermost-with-nesting -> pad inside outer () -int f(){ return outer(inner(a ,b)); } - -// strings and comments must be unchanged -int s(){ printf("x ,y ,z (still a string)"); /* a ,b ,c */ return 1; } - -// unbalanced open-right with nesting -> pad after first unmatched '(' -int u(){if(doit(foo(1 ,2) // missing )) - return 0;} - -// arrays / subscripts stay tight; commas still RT-style -int a(int i ,int j){ return M[i ,j] + V[i] + W[j]; } diff --git a/tester/RT-formatter/data_test-1.py b/tester/RT-formatter/data_test-1.py deleted file mode 100644 index 9b2fa87..0000000 --- a/tester/RT-formatter/data_test-1.py +++ /dev/null @@ -1,16 +0,0 @@ -# commas and spacing in defs / calls -def f ( x , y , z ): - return dict( a =1 , b= 2 ), [ 1, 2 ,3 ], ( (1,2) ) - -# outermost-with-nesting -> pad inside outer () -val = outer( inner( a,b ) ) - -# strings/comments untouched -s = "text, with , commas ( not to touch )" # a ,b ,c - -# unbalanced: open-left (closing without opener) -> no padding unless inner bracket before it -def g(): - return result) # likely unchanged - -# unbalanced: open-right (first unmatched opener) with inner bracket following -k = compute(x, f(y -- 2.20.1