--- /dev/null
+#!/usr/bin/env python3
+# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*-
+"""
+skeleton_update
+
+Brings a project onto the current Harmony skeleton. Skeleton files are
+copied in from a Harmony checkout, and .gitignore and shared/tool/version are
+merged with the project's own lines. The project's files are then sorted:
+
+ a file in a place Harmony has, such as developer/authored/ or a directory
+ of the skeleton, stays where it is;
+ a file the plan moves is renamed to its new place;
+ a file under a plan's duplicate tree is deleted when the project holds a
+ file with the same content elsewhere, and is otherwise deprecated;
+ every other file is deprecated: moved, with its path kept, under
+ developer/authored/deprecated/, so nothing is lost and the original place
+ of each file can be read from its new path.
+
+The plan is a Python file giving what is particular to one project. A
+project already laid out after Harmony needs no plan, and the tool then
+updates its skeleton files only. The plan's names are:
+
+ MOVE {old path: new path}; a path ending in / moves a directory tree
+ DEPRECATE paths that are deprecated even though Harmony has their place
+ DUPLICATE trees whose files are copies of files kept elsewhere
+ KEEP paths kept as they are, including skeleton files the project
+ has its own version of
+ EXECUTABLE paths, after any move, given execute permission where read
+ permission is given
+
+The files sorted are those git tracks or would track: the work tree less what
+the project's .gitignore ignores. Ignored files, such as a local Python
+environment, are left where they are and listed at the end of the log.
+Compiled Python caches are deleted. Directories left empty are removed.
+
+Commit the project first; run refuses a work tree with uncommitted changes.
+Afterward, 'git add -A' records the moves, and git recognizes each moved file
+as a rename by its content, so 'git log --follow' follows its history. Run
+again after committing, and the log reports nothing left to do.
+
+Usage: skeleton_update run <Harmony dir> <project dir> [<plan file>]
+ skeleton_update preview run <Harmony dir> <project dir> [<plan file>]
+ help | -h | --help print this message (also printed when given no arguments)
+ run bring <project dir> onto the skeleton of <Harmony dir>
+ preview run print the log of what run would do, change nothing
+"""
+
+import sys ,os ,shutil ,filecmp ,subprocess ,runpy ,hashlib
+from pathlib import Path
+
+
+# --------------------------------------------------------------------------------
+# what the tool knows about Harmony
+
+DEPRECATED_DIR = "developer/authored/deprecated"
+
+# Skeleton files a project does not take: it has its own.
+SKELETON_SKIP_FILE = {"README.md" ,"LICENSE" ,"0pus_Harmony"}
+SKELETON_SKIP_TREE = ("developer/authored/ExampleGreet.make/",)
+
+# Skeleton files merged with the project's version rather than replaced.
+MERGED_FILE = (".gitignore" ,"shared/tool/version")
+
+# Files at the top of a project that belong to the project.
+PROJECT_ROOT_FILE = {"LICENSE" ,"README.md" ,".gitattributes" ,".gitmodules"}
+
+# Trees in which every file belongs to the project.
+PROJECT_TREE = (
+ "administrator/authored/",
+ "administrator/document/",
+ "developer/authored/",
+ "developer/document/",
+ "developer/experiment/",
+ "document/",
+ "shared/authored/",
+ "tester/authored/",
+ "tester/document/",
+)
+
+# Placeholder files of the old skeleton; Harmony uses .gitkeep.
+PLACEHOLDER_NAME = {".githolder"}
+
+CACHE_DIR_NAME = {"__pycache__"}
+
+PLAN_NAME = ("MOVE" ,"DEPRECATE" ,"DUPLICATE" ,"KEEP" ,"EXECUTABLE")
+
+
+# --------------------------------------------------------------------------------
+# the log
+
+TAG_WIDTH = 18
+
+# the tag a preview prints in place of each past-tense tag
+PREVIEW_TAG = {
+ "added": "add" ,"deleted": "delete" ,"deprecated": "deprecate" ,"kept": "keep",
+ "left alone": "leave alone" ,"made executable": "make executable" ,"merged": "merge",
+ "moved": "move" ,"removed": "remove",
+ "replaced": "replace",
+}
+
+class Log:
+ def __init__(self ,preview):
+ self.preview = preview
+ self.count = {}
+ self.error_count = 0
+
+ def line(self ,tag ,rel ,reason=None):
+ if self.preview: tag = "would " + PREVIEW_TAG.get(tag ,tag)
+ text = f" {tag:<{TAG_WIDTH}}{rel}"
+ if reason: text += f" ({reason})"
+ print(text)
+
+ def tally(self ,key ,n=1):
+ self.count[key] = self.count.get(key ,0) + n
+
+ def error(self ,rel ,message):
+ self.error_count += 1
+ print(f" {'error':<{TAG_WIDTH}}{rel} ({message})")
+
+ def summary(self):
+ print("summary:")
+ order = [
+ ("moved" ,"files moved to their place by the plan"),
+ ("deprecated" ,"files moved under " + DEPRECATED_DIR),
+ ("deleted duplicate" ,"duplicate files deleted"),
+ ("deleted placeholder" ,".githolder files deleted"),
+ ("deleted cache" ,"__pycache__ directories deleted"),
+ ("skeleton added" ,"skeleton files added"),
+ ("skeleton replaced" ,"skeleton files replaced"),
+ ("skeleton current" ,"skeleton files already current"),
+ ("skeleton kept" ,"skeleton files kept in the project's own version"),
+ ("merged" ,"files merged with the skeleton"),
+ ("merged current" ,"merged files already current"),
+ ("kept" ,"project files kept in place"),
+ ("made executable" ,"files made executable"),
+ ("left alone" ,"ignored files left in place, listed above"),
+ ("directory removed" ,"empty directories removed"),
+ ]
+ for key ,label in order:
+ print(f" {self.count.get(key ,0):>4} {label}")
+ if self.error_count:
+ print(f" {self.error_count:>4} errors")
+ if self.preview:
+ print("preview only: nothing was changed.")
+
+
+# --------------------------------------------------------------------------------
+# reading git
+
+def git(directory ,*argument):
+ return subprocess.run(
+ ["git" ,"-C" ,str(directory) ,*argument],
+ capture_output=True ,text=True ,check=True
+ ).stdout
+
+def git_dirty(directory):
+ return git(directory ,"status" ,"--porcelain").strip()
+
+def project_file_list(project):
+ """
+ The files git tracks or would track, that exist: tracked files, and
+ untracked files the project's .gitignore does not ignore.
+ """
+ listing = git(project ,"ls-files" ,"-z" ,"--cached" ,"--others" ,"--exclude-standard").split("\0")
+ result = set()
+ for rel in listing:
+ if rel and ((project / rel).exists() or (project / rel).is_symlink()):
+ result.add(rel)
+ return sorted(result)
+
+def ignored_file_list(project):
+ listing = git(project ,"ls-files" ,"-z" ,"--others" ,"--ignored" ,"--exclude-standard").split("\0")
+ return sorted(rel for rel in listing if rel)
+
+def skeleton_file_list(harmony):
+ """
+ The skeleton is the set of files in Harmony's last commit, as new-project
+ takes it.
+ """
+ result = []
+ for rel in git(harmony ,"ls-tree" ,"-r" ,"--name-only" ,"-z" ,"HEAD").split("\0"):
+ if not rel or rel in SKELETON_SKIP_FILE or in_tree(rel ,SKELETON_SKIP_TREE):
+ continue
+ result.append(rel)
+ return sorted(result)
+
+
+# --------------------------------------------------------------------------------
+# paths
+
+def in_tree(rel ,tree_tuple):
+ return any(rel.startswith(t) for t in tree_tuple)
+
+def matches(rel ,path_tuple):
+ """A path names a file exactly, or, ending in /, the tree under it."""
+ return any(rel == p or (p.endswith("/") and rel.startswith(p)) for p in path_tuple)
+
+def moved_path(rel ,move):
+ for old ,new in move.items():
+ if rel == old:
+ return new
+ if old.endswith("/") and rel.startswith(old):
+ return new + rel[len(old):]
+ return None
+
+def directory_set(rel_list):
+ result = set()
+ for rel in rel_list:
+ parent = Path(rel).parent
+ while parent != Path("."):
+ result.add(parent.as_posix())
+ parent = parent.parent
+ return result
+
+def content_hash(path):
+ if path.is_symlink():
+ return "link:" + os.readlink(path)
+ h = hashlib.sha256()
+ with open(path ,"rb") as f:
+ for block in iter(lambda: f.read(1 << 16) ,b""):
+ h.update(block)
+ return h.hexdigest()
+
+
+# --------------------------------------------------------------------------------
+# file operations; each tests preview immediately before acting
+
+def same_leaf(a ,b):
+ if a.is_symlink() or b.is_symlink():
+ return a.is_symlink() and b.is_symlink() and os.readlink(a) == os.readlink(b)
+ return a.is_file() and b.is_file() and filecmp.cmp(a ,b ,shallow=False)
+
+def copy_leaf(src ,dst ,preview):
+ if preview: return
+ dst.parent.mkdir(parents=True ,exist_ok=True)
+ if dst.is_symlink() or dst.exists():
+ dst.unlink()
+ if src.is_symlink():
+ os.symlink(os.readlink(src) ,dst)
+ else:
+ shutil.copy2(src ,dst)
+
+def move_leaf(src ,dst ,preview):
+ if preview: return
+ dst.parent.mkdir(parents=True ,exist_ok=True)
+ os.rename(src ,dst)
+
+def delete_leaf(path ,preview):
+ if preview: return
+ path.unlink()
+
+def delete_tree(path ,preview):
+ if preview: return
+ shutil.rmtree(path)
+
+def write_text(path ,text ,preview):
+ if preview: return
+ path.parent.mkdir(parents=True ,exist_ok=True)
+ path.write_text(text ,encoding="utf-8")
+
+
+# --------------------------------------------------------------------------------
+# merging
+
+def merged_gitignore(project_text ,harmony_text ,project_name):
+ """
+ Harmony's lines, then a section holding the project's patterns that
+ Harmony lacks. The project's comments and blank lines are not carried.
+ """
+ section = "# " + project_name
+ harmony_line = set(harmony_text.splitlines())
+ extra = []
+ for line in project_text.splitlines():
+ pattern = line.strip()
+ if not pattern or pattern.startswith("#"):
+ continue
+ if pattern not in harmony_line and pattern not in extra:
+ extra.append(pattern)
+ text = harmony_text.rstrip("\n") + "\n"
+ if extra:
+ text += "\n" + section + "\n" + "\n".join(extra) + "\n"
+ return text
+
+def merged_version(project_text ,harmony_text ,project_name):
+ """Harmony's file, then the project's lines that do not name Harmony."""
+ project_line = [l for l in project_text.splitlines() if l.strip() and "Harmony" not in l and l not in harmony_text.splitlines()]
+ text = harmony_text if harmony_text.endswith("\n") else harmony_text + "\n"
+ return text + "".join(l + "\n" for l in project_line)
+
+MERGE_FUNCTION = {
+ ".gitignore": merged_gitignore,
+ "shared/tool/version": merged_version,
+}
+
+
+# --------------------------------------------------------------------------------
+# the work
+
+def step_cache(project ,log ,preview):
+ for dirpath ,dirnames ,filenames in os.walk(project):
+ dirnames.sort()
+ if Path(dirpath) == project and ".git" in dirnames:
+ dirnames.remove(".git")
+ for name in list(dirnames):
+ if name in CACHE_DIR_NAME:
+ rel = Path(dirpath ,name).relative_to(project).as_posix()
+ delete_tree(project / rel ,preview)
+ log.line("deleted" ,rel + "/" ,"compiled Python cache")
+ log.tally("deleted cache")
+ dirnames.remove(name)
+
+def decide(rel ,plan ,skeleton_set ,skeleton_dir ,ignored_list):
+ action = decide_place(rel ,plan ,skeleton_set ,skeleton_dir)
+ if Path(rel).name == ".gitignore" and action[0] in ("move" ,"duplicate" ,"deprecate"):
+ prefix = Path(rel).parent.as_posix() + "/"
+ n = sum(1 for i in ignored_list if i.startswith(prefix))
+ if n:
+ return ("keep" ,rel ,f"it hides {n} ignored file{'s' if n != 1 else ''}, left in place")
+ return action
+
+def decide_place(rel ,plan ,skeleton_set ,skeleton_dir):
+ """
+ What becomes of one project file: (action ,new path ,reason). The action is
+ one of placeholder, keep, keep skeleton, move, duplicate, deprecate, or
+ skeleton, which the skeleton steps handle.
+ """
+ name = Path(rel).name
+ if name in PLACEHOLDER_NAME:
+ return ("placeholder" ,None ,None)
+ if matches(rel ,plan["KEEP"]):
+ return ("keep skeleton" if rel in skeleton_set else "keep" ,rel ,None)
+ new = moved_path(rel ,plan["MOVE"])
+ if new is not None:
+ return ("move" ,new ,None)
+ if matches(rel ,plan["DUPLICATE"]):
+ return ("duplicate" ,f"{DEPRECATED_DIR}/{rel}" ,None)
+ if matches(rel ,plan["DEPRECATE"]):
+ return ("deprecate" ,f"{DEPRECATED_DIR}/{rel}" ,"by the plan")
+ if rel in skeleton_set or rel in MERGED_FILE:
+ return ("skeleton" ,rel ,None)
+ parent = Path(rel).parent.as_posix()
+ if (parent == "." and (name in PROJECT_ROOT_FILE or name.startswith("0pus_"))) \
+ or in_tree(rel ,PROJECT_TREE) or parent in skeleton_dir:
+ return ("keep" ,rel ,None)
+ return ("deprecate" ,f"{DEPRECATED_DIR}/{rel}" ,None)
+
+def step_sort(project ,plan ,skeleton_set ,file_list ,ignored_list ,log ,preview):
+ """
+ Place every project file that is not a skeleton file. Every decision is
+ made before anything moves, so a duplicate can name where its twin ends up.
+ """
+ skeleton_dir = directory_set(skeleton_set)
+ decision = {rel: decide(rel ,plan ,skeleton_set ,skeleton_dir ,ignored_list) for rel in file_list}
+
+ # where the content of each file that stays in the project ends up
+ kept_at = {}
+ for rel ,(action ,new ,reason) in decision.items():
+ if action in ("keep" ,"keep skeleton" ,"move" ,"deprecate" ,"skeleton"):
+ kept_at.setdefault(content_hash(project / rel) ,new)
+
+ for rel in file_list:
+ action ,new ,reason = decision[rel]
+ if action == "placeholder":
+ delete_leaf(project / rel ,preview)
+ log.line("deleted" ,rel ,"placeholder; Harmony uses .gitkeep")
+ log.tally("deleted placeholder")
+ elif action == "keep skeleton":
+ log.line("kept" ,rel ,"the project's own version, by the plan")
+ log.tally("skeleton kept")
+ elif action == "keep":
+ if reason: log.line("kept" ,rel ,reason)
+ log.tally("kept")
+ elif action == "move":
+ place(project ,rel ,new ,"moved" ,f"to {new}" ,log ,preview)
+ elif action == "duplicate":
+ twin = kept_at.get(content_hash(project / rel))
+ if twin:
+ delete_leaf(project / rel ,preview)
+ log.line("deleted" ,rel ,f"the same as {twin}")
+ log.tally("deleted duplicate")
+ else:
+ place(project ,rel ,new ,"deprecated" ,"no copy of it is kept" ,log ,preview)
+ elif action == "deprecate":
+ place(project ,rel ,new ,"deprecated" ,reason ,log ,preview)
+
+def place(project ,rel ,new ,tag ,reason ,log ,preview):
+ dst = project / new
+ if dst.exists() or dst.is_symlink():
+ log.error(rel ,f"{new} already exists; nothing was moved")
+ return
+ move_leaf(project / rel ,dst ,preview)
+ log.line(tag ,rel ,reason)
+ log.tally(tag)
+
+def step_skeleton(project ,harmony ,skeleton_list ,plan ,log ,preview):
+ for rel in skeleton_list:
+ if rel in MERGED_FILE or matches(rel ,plan["KEEP"]):
+ continue
+ src = harmony / rel
+ dst = project / rel
+ if dst.exists() or dst.is_symlink():
+ if same_leaf(src ,dst):
+ log.tally("skeleton current")
+ continue
+ copy_leaf(src ,dst ,preview)
+ log.line("replaced" ,rel ,"from the skeleton")
+ log.tally("skeleton replaced")
+ else:
+ copy_leaf(src ,dst ,preview)
+ log.line("added" ,rel ,"from the skeleton")
+ log.tally("skeleton added")
+
+def step_merge(project ,harmony ,project_name ,plan ,log ,preview):
+ for rel in MERGED_FILE:
+ if matches(rel ,plan["KEEP"]):
+ continue
+ project_path = project / rel
+ harmony_text = (harmony / rel).read_text(encoding="utf-8")
+ project_text = project_path.read_text(encoding="utf-8") if project_path.is_file() else ""
+ text = MERGE_FUNCTION[rel](project_text ,harmony_text ,project_name)
+ if text == project_text:
+ log.tally("merged current")
+ continue
+ if not project_path.exists():
+ copy_leaf(harmony / rel ,project_path ,preview)
+ write_text(project_path ,text ,preview)
+ log.line("merged" ,rel ,"the skeleton's lines with the project's own")
+ log.tally("merged")
+
+def step_opus(project ,harmony ,project_name ,log ,preview):
+ rel = "0pus_" + project_name
+ if (project / rel).exists():
+ return
+ write_text(project / rel ,"" ,preview)
+ if not preview: shutil.copymode(harmony / "0pus_Harmony" ,project / rel)
+ log.line("added" ,rel ,"the file that marks the project root")
+ log.tally("skeleton added")
+
+def step_executable(project ,plan ,file_list ,log ,preview):
+ """
+ Give each EXECUTABLE path execute permission wherever it has read
+ permission. In a preview the file is found at its place before the move.
+ """
+ before = {moved_path(rel ,plan["MOVE"]) or rel: rel for rel in file_list}
+ for rel in plan["EXECUTABLE"]:
+ path = project / rel
+ if not path.is_file():
+ path = project / before.get(rel ,rel)
+ if not path.is_file():
+ log.error(rel ,"EXECUTABLE names a file that does not exist")
+ continue
+ mode = path.stat().st_mode
+ wanted = mode | ((mode & 0o444) >> 2)
+ if wanted == mode:
+ continue
+ if not preview: os.chmod(path ,wanted)
+ log.line("made executable" ,rel)
+ log.tally("made executable")
+
+def step_ignored(project ,file_list ,ignored_list ,log):
+ """
+ List ignored files outside scratchpad and consumer, by the highest
+ directory holding nothing git tracks.
+ """
+ tracked_dir = directory_set(file_list)
+ group = {}
+ for rel in ignored_list:
+ part = rel.split("/")
+ if "scratchpad" in part[:-1] or part[0] == "consumer" or any(p in CACHE_DIR_NAME for p in part):
+ continue
+ top = rel
+ parent = Path(rel).parent
+ while parent != Path(".") and parent.as_posix() not in tracked_dir:
+ top = parent.as_posix() + "/"
+ parent = parent.parent
+ group[top] = group.get(top ,0) + 1
+ for top ,n in sorted(group.items()):
+ log.line("left alone" ,top ,f"ignored by git; {n} file{'s' if n != 1 else ''}")
+ log.tally("left alone" ,n)
+
+def step_newly_ignored(project ,ignored_list ,log):
+ """
+ A file the run placed where the merged .gitignore ignores it would not be
+ recorded by 'git add -A'. Such files are reported as errors.
+ """
+ before = set(ignored_list)
+ for rel in ignored_file_list(project):
+ part = rel.split("/")
+ if rel in before or "scratchpad" in part[:-1] or part[0] == "consumer" or any(p in CACHE_DIR_NAME for p in part):
+ continue
+ log.error(rel ,"git ignores it now, so 'git add -A' will not record it; add it with 'git add -f'")
+
+def step_empty_dir(project ,log ,preview):
+ for dirpath ,dirnames ,filenames in os.walk(project ,topdown=False):
+ path = Path(dirpath)
+ if path == project or ".git" in path.relative_to(project).parts:
+ continue
+ try:
+ remaining = list(path.iterdir())
+ except FileNotFoundError:
+ continue
+ if not remaining:
+ rel = path.relative_to(project).as_posix()
+ if not preview: path.rmdir()
+ log.line("removed" ,rel + "/" ,"empty directory")
+ log.tally("directory removed")
+
+def work(harmony ,project ,plan ,preview):
+ print(f"skeleton_update {'preview ' if preview else ''}run in {project}")
+ print(f" skeleton from {harmony}")
+ log = Log(preview)
+ project_name = project.name
+
+ skeleton_list = skeleton_file_list(harmony)
+ skeleton_set = set(skeleton_list)
+ file_list = [r for r in project_file_list(project) if not any(p in CACHE_DIR_NAME for p in r.split("/"))]
+ ignored_list = [r for r in ignored_file_list(project) if not any(p in CACHE_DIR_NAME for p in r.split("/"))]
+
+ step_cache(project ,log ,preview)
+ step_sort(project ,plan ,skeleton_set ,file_list ,ignored_list ,log ,preview)
+ step_skeleton(project ,harmony ,skeleton_list ,plan ,log ,preview)
+ step_merge(project ,harmony ,project_name ,plan ,log ,preview)
+ step_opus(project ,harmony ,project_name ,log ,preview)
+ step_executable(project ,plan ,file_list ,log ,preview)
+ step_ignored(project ,file_list ,ignored_list ,log)
+ if preview:
+ print(" (directories left empty are found and removed by run, not listed in a preview)")
+ else:
+ step_empty_dir(project ,log ,preview)
+ step_newly_ignored(project ,ignored_list ,log)
+
+ log.summary()
+ return 1 if log.error_count else 0
+
+
+# --------------------------------------------------------------------------------
+# command line
+
+def read_plan(path):
+ plan = {"MOVE": {} ,"DEPRECATE": () ,"DUPLICATE": () ,"KEEP": () ,"EXECUTABLE": ()}
+ if path is None:
+ return plan
+ given = runpy.run_path(str(path))
+ for name in PLAN_NAME:
+ if name in given:
+ plan[name] = given[name]
+ if not isinstance(plan["MOVE"] ,dict):
+ raise ValueError("MOVE must be a dictionary of old path to new path")
+ for name in ("DEPRECATE" ,"DUPLICATE" ,"KEEP" ,"EXECUTABLE"):
+ plan[name] = tuple(plan[name])
+ for old ,new in plan["MOVE"].items():
+ if old.endswith("/") != new.endswith("/"):
+ raise ValueError(f"MOVE '{old}' to '{new}': a directory moves to a directory, both ending in /")
+ return plan
+
+def print_help():
+ print(__doc__.strip())
+ return 0
+
+def CLI(argv=None):
+ if argv is None: argv = sys.argv[1:]
+ if not argv or argv[0] in ("help" ,"-h" ,"--help"): return print_help()
+
+ preview = argv[0] == "preview"
+ if preview: argv = argv[1:]
+ if not argv:
+ print("Error: 'preview' needs a command after it." ,file=sys.stderr)
+ return 1
+
+ command ,rest = argv[0] ,argv[1:]
+ if command != "run":
+ print(f"Error: unknown command '{command}'." ,file=sys.stderr)
+ print_help()
+ return 1
+ if len(rest) not in (2 ,3):
+ print("Error: run is given <Harmony dir> <project dir> [<plan file>]." ,file=sys.stderr)
+ return 1
+
+ harmony = Path(rest[0]).resolve()
+ project = Path(rest[1]).resolve()
+ plan_path = Path(rest[2]).resolve() if len(rest) == 3 else None
+
+ if not (harmony / "0pus_Harmony").is_file() or not (harmony / ".git").exists():
+ print(f"Error: {harmony} is not a Harmony checkout: it lacks 0pus_Harmony or .git." ,file=sys.stderr)
+ return 1
+ if not (project / ".git").exists():
+ print(f"Error: {project} is not a git work tree; the tool sorts the files git tracks." ,file=sys.stderr)
+ return 1
+ if harmony == project:
+ print("Error: the project is the Harmony checkout itself." ,file=sys.stderr)
+ return 1
+ if plan_path and not plan_path.is_file():
+ print(f"Error: plan file {plan_path} does not exist." ,file=sys.stderr)
+ return 1
+ try:
+ plan = read_plan(plan_path)
+ except Exception as error:
+ print(f"Error: plan file {plan_path}: {error}" ,file=sys.stderr)
+ return 1
+
+ dirty = git_dirty(project)
+ if dirty and not preview:
+ print("Error: the project work tree has uncommitted changes. Commit them first:" ,file=sys.stderr)
+ print(dirty ,file=sys.stderr)
+ return 1
+ dirty = git_dirty(harmony)
+ if dirty:
+ print("Error: the Harmony checkout has uncommitted changes; the skeleton is taken" ,file=sys.stderr)
+ print("from its last commit, and its files would not match it:" ,file=sys.stderr)
+ print(dirty ,file=sys.stderr)
+ return 1
+
+ return work(harmony ,project ,plan ,preview)
+
+
+if __name__ == "__main__":
+ try:
+ sys.exit(CLI())
+ except BrokenPipeError:
+ os.dup2(os.open(os.devnull ,os.O_WRONLY) ,sys.stdout.fileno())
+ sys.exit(1)
--- /dev/null
+#!/usr/bin/env python3
+# -*- mode: python; coding: utf-8; python-indent-offset: 2; indent-tabs-mode: nil -*-
+"""
+replace_from_tar
+
+Makes directories hold exactly what a tar file holds. For each directory at
+the top of the tar file, such as SubU.copy/, the directory of the same name in
+the extract dir is brought to match: files the tar file has are written when
+they are new or differ, and files and directories it does not have are
+deleted. The extract dir is the directory 'tar -x' would be run in: the one
+that holds SubU.copy, not SubU.copy itself. It is the current directory unless
+given. Extracting a tar file over a directory only adds and overwrites, so
+a file that was removed from the tar file would otherwise stay behind.
+
+The tool deletes files. Checkpoint the project with git first, and run
+preview to read what it would do.
+
+Usage: replace_from_tar <command> [argument ...]
+ replace_from_tar preview <command> [argument ...]
+ help | -h | --help print this message (also printed when given no arguments)
+ replace <tar file> [<extract dir>]
+ bring each directory at the top of the tar file,
+ in the extract dir, to match the tar file
+ preview <command> print the log of what <command> would do, change nothing
+
+Example, run in developer/authored, which holds SubU.copy:
+ replace_from_tar preview replace SubU.copy.tar
+ replace_from_tar replace SubU.copy.tar
+"""
+
+import sys ,os ,tarfile
+from pathlib import Path, PurePosixPath
+
+TAG_WIDTH = 12
+
+
+class Log:
+ PREVIEW_TAG = {"made": "would make" ,"replaced": "would replace" ,"deleted": "would delete"}
+
+ def __init__(self ,preview):
+ self.preview = preview
+ self.count = {}
+ self.error_count = 0
+
+ def line(self ,tag ,subject ,reason=None):
+ self.count[tag] = self.count.get(tag ,0) + 1
+ if tag == "unchanged": return
+ shown = self.PREVIEW_TAG.get(tag ,tag) if self.preview else tag
+ print(f" {shown:<{TAG_WIDTH + 2}}{subject}" + (f" ({reason})" if reason else ""))
+
+ def error(self ,subject ,reason):
+ self.error_count += 1
+ print(f" {'error':<{TAG_WIDTH + 2}}{subject} ({reason})")
+
+ def summary(self):
+ print("summary:")
+ for tag ,what in (("made" ,"files") ,("replaced" ,"files") ,("deleted" ,"files and directories") ,("unchanged" ,"files")):
+ n = self.count.get(tag ,0)
+ verb = f"that would be {tag}" if self.preview and tag != "unchanged" else tag
+ print(f" {n:>4} {what} {verb}")
+ if self.error_count:
+ print(f" {self.error_count:>4} errors")
+ if self.preview:
+ print("preview only: nothing was changed.")
+ return 1 if self.error_count else 0
+
+
+def member_problem(member):
+ path = PurePosixPath(member.name)
+ if path.is_absolute() or ".." in path.parts:
+ return "an absolute path or '..'"
+ if len(path.parts) < 1:
+ return "an empty name"
+ if not (member.isfile() or member.isdir() or member.issym()):
+ return "neither a file, a directory nor a symbolic link"
+ if member.issym() and (PurePosixPath(member.linkname).is_absolute() or ".." in PurePosixPath(member.linkname).parts):
+ return "a symbolic link leaving its directory"
+ return None
+
+def replace(tar_path ,extract_dir ,log ,preview):
+ extract_dir = Path(extract_dir)
+ if not extract_dir.is_dir():
+ log.error(str(extract_dir) ,"not a directory"); return
+ with tarfile.open(tar_path) as tar:
+ member_list = tar.getmembers()
+ for m in member_list:
+ problem = member_problem(m)
+ if problem:
+ log.error(m.name ,f"refused: {problem}; nothing was changed"); return
+ top_set = sorted({PurePosixPath(m.name).parts[0] for m in member_list})
+ for top in top_set:
+ if any(PurePosixPath(m.name) == PurePosixPath(top) and not m.isdir() for m in member_list):
+ log.error(top ,"refused: the top of the tar file must hold directories; nothing was changed"); return
+ target = extract_dir / top
+ if target.is_symlink():
+ log.error(str(target) ,"refused: it is a symbolic link; nothing was changed"); return
+
+ wanted = {str(PurePosixPath(m.name)): m for m in member_list}
+
+ # ---- delete what the tar file does not have, deepest first
+ for top in top_set:
+ target = extract_dir / top
+ if not target.exists(): continue
+ for dirpath ,dirname_list ,filename_list in os.walk(target ,topdown=False ,followlinks=False):
+ for name in sorted(filename_list) + sorted(d for d in dirname_list if (Path(dirpath) / d).is_symlink()):
+ path = Path(dirpath) / name
+ key = str(PurePosixPath(path.relative_to(extract_dir)))
+ if key not in wanted or wanted[key].isdir():
+ if not preview: path.unlink()
+ log.line("deleted" ,key)
+ for name in sorted(d for d in dirname_list if not (Path(dirpath) / d).is_symlink()):
+ path = Path(dirpath) / name
+ key = str(PurePosixPath(path.relative_to(extract_dir)))
+ if key not in wanted:
+ if not preview:
+ if any(path.iterdir()):
+ log.error(key ,"not empty after its files were deleted"); continue
+ path.rmdir()
+ log.line("deleted" ,key + "/")
+
+ # ---- write what is new or differs
+ for m in sorted(member_list ,key=lambda m: m.name):
+ key = str(PurePosixPath(m.name))
+ path = extract_dir / key
+ if m.isdir():
+ if not path.is_dir():
+ if not preview: path.mkdir(parents=True ,exist_ok=True)
+ log.line("made" ,key + "/")
+ continue
+ if m.issym():
+ if path.is_symlink() and os.readlink(path) == m.linkname:
+ log.line("unchanged" ,key); continue
+ if not preview:
+ if path.is_symlink() or path.exists(): path.unlink()
+ path.parent.mkdir(parents=True ,exist_ok=True)
+ os.symlink(m.linkname ,path)
+ log.line("replaced" if path.is_symlink() else "made" ,key); continue
+ data = tar.extractfile(m).read()
+ mode = m.mode & 0o777
+ if path.is_file() and not path.is_symlink():
+ if path.read_bytes() == data and (path.stat().st_mode & 0o777) == mode:
+ log.line("unchanged" ,key); continue
+ tag = "replaced"
+ else:
+ tag = "made"
+ if not preview:
+ path.parent.mkdir(parents=True ,exist_ok=True)
+ if path.is_symlink(): path.unlink()
+ path.write_bytes(data)
+ os.chmod(path ,mode)
+ log.line(tag ,key)
+
+
+def print_help():
+ print(__doc__.strip())
+ return 0
+
+def CLI(argv=None):
+ if argv is None: argv = sys.argv[1:]
+ if not argv or argv[0] in ("help" ,"-h" ,"--help"): return print_help()
+ preview = argv[0] == "preview"
+ if preview: argv = argv[1:]
+ if not argv:
+ print("Error: 'preview' needs a command after it." ,file=sys.stderr)
+ return 1
+ command ,rest = argv[0] ,argv[1:]
+ if command == "replace" and len(rest) in (1 ,2):
+ if len(rest) == 1: rest = [rest[0] ,"."]
+ if not Path(rest[0]).is_file():
+ print(f"Error: no tar file at {rest[0]}." ,file=sys.stderr)
+ return 1
+ log = Log(preview)
+ print(f"replace_from_tar {'preview ' if preview else ''}replace {rest[0]} in {os.path.abspath(rest[1])}")
+ replace(rest[0] ,rest[1] ,log ,preview)
+ return log.summary()
+ print(f"Error: unknown command or wrong arguments: '{command}'." ,file=sys.stderr)
+ print_help()
+ return 1
+
+
+if __name__ == "__main__":
+ sys.exit(CLI())
projects / Harmony / commitdiff