From: Thomas Walker Lynch Date: Sat, 28 Sep 2024 11:09:55 +0000 (+0000) Subject: DAG build first draft. Proposed DAG clean algorithm. X-Git-Url: https://git.reasoningtechnology.com/fossil/timeline.rss?a=commitdiff_plain;h=5ca2e7a594b6d7557516cdd64afdfe00b501f1d0;p=GQL-to-Cypher DAG build first draft. Proposed DAG clean algorithm. --- diff --git a/developer/#build.gradle# b/developer/#build.gradle# deleted file mode 100644 index 35450ad..0000000 --- a/developer/#build.gradle# +++ /dev/null @@ -1,696 +0,0 @@ -//--------------------------------------------------------------------------------- -// globals - -// string comprehension includes for regular expressions -def base = "[a-zA-Z0-9_-]+" -def ext = "[a-zA-Z0-9_-]+$" -def name = "${base}\\.${ext}" -def path = ".+/${name}" - -//-------------------------------------------------------------------------------- -// Import variables from the environment -// - -def env = [:] - -// Required shell environment variables -def varName_List = [ - 'REPO_HOME', - 'PROJECT', - 'ENV_BUILD_VERSION', - 'DEVELOPER_HOME' -] -varName_List.each { varName -> - def value = System.getenv(varName) ?: "" - env[varName] = value -} - -// Optional shell environment variables -def varNameOptional_List = [ - 'CLASSPATH' -] -varNameOptional_List.each { varName -> - def value = System.getenv(varName) ?: "" - env[varName] = value -} - -env.CLASSPATH += ":${env.ANTLR_JAR}" - -//-------------------------------------------------------------------------------- -// PM installed tools to be used -// these should be added to the project object by the installer, and taken -// from the project object here. -// -// Tools used (set by project manager: JAVA_HOME, ANTLR_JAR, DEVELOPER_HOME) -def JAVA_COMP_FP = "${env.JAVA_HOME}/bin/javac" // Java compiler path -def JAVA_INTERP_FP = "${env.JAVA_HOME}/bin/java" // Java interpreter path -def JAVA_ARCHIVE_FP = "${env.JAVA_HOME}/bin/jar" // Java archive tool path - - -//-------------------------------------------------------------------------------- -// Directory structure -// - -def dir_map = [ - ,'EXECUTOR_IN' : 'executor/' - ,'ANTLR_IN_LEAF' : 'ANTLR/' - ,'ANTLR_OUT' : 'javac/ANTLR/' - ,'ANTLR_OUT_PARENT' : 'javac/' - ,'JAVA_COMP_IN' : 'javac/' - ,'JAVA_COMP_IN_LEAF': 'javac/leaf/' - ,'JAVA_COMP_IN_ANTLR' : 'javac/ANTLR/' - ,'JAVA_COMP_IN_SYN' : 'javac/synthesized/' - ,'JAVA_COMP_OUT' : 'jvm/' - ,'JVM_IN' : 'jvm/' - ,'TEMP' : 'Erebus/' -] - -// Update CLASSPATH -env.CLASSPATH += ":${dir_map.JVM_IN}" - -// Construct JAVA_COMP_IN_LIST dynamically and add to dp map -dir_map.JAVA_COMP_IN_LIST = - "${dir_map.JAVA_COMP_IN_LEAF}" + - ":${dir_map.JAVA_COMP_IN_ANTLR}" + - ":${dir_map.JAVA_COMP_IN_SYN}" - -println "CLASSPATH: ${env.CLASSPATH}" -println "JAVA_COMP_IN_LIST: ${dir_map.JAVA_COMP_IN_LIST}" - - -// Subroutine to print missing environment variables list message -def printMissingVars(missingVars) { - if (missingVars.isEmpty()) { - // Print nothing if the list is empty - return - } else if (missingVars.size() == 1) { - println "This environment variable was not set: ${missingVars[0]}" - } else { - println "These environment variables were not set: ${missingVars.join(', ')}" - } -} - -task preface { - dependsOn ':installTools' - doFirst { - - // Environment variable checks - def error_missing = false - def error_project = false - def missingVars = [] // To collect missing environment variables - varName_List.each { varName -> - if (!env[varName]) { - error_missing = true - missingVars << varName - } - } - printMissingVars(missingVars) - if (env.PROJECT != "GQL_to_Cypher") { - error_project = true - println "Expected project 'GQL_to_Cypher', but found '${env.PROJECT}'." - } - if (error_missing || error_project) { - throw new GradleException("Bailing due to missing environment variables.") - } - } - doLast { - println "================================================================================" - println "Building project .." - } -} - -/*-------------------------------------------------------------------------------- - Dependency Graph - - A node is a property list. - - A node label is a relative path to a file, or a symbolic value. - - We either a) put nodes in a map that key on the node label then return the node, - or b) we use functions that recognize the node label and return a property list. - The `lookup(node-label)` function combines all methods to find the node. - - A will formed node will have these properties: label and type. - - A node without a 'neighbor' property is said to be a 'leaf_node'. - - `path` type nodes have a `build` property. -*/ - -//--------------------------------------- -// some helpers - -def node_type_set = ['symbol' ,'path'] as Set -def node_mark_set = ['cycle_member' ,'malformed' ] as Set - -/* - */ - -def well_formed_q = { node -> - def form_error_set = [] as Set - - if( - !node.label || node.label.length() == 0 - ) - form_error_set << 'no_node_label' - - if( - !node.type || !(node.type in node_type_set) - ) - form_error_set << 'no_node_type' - - if( - node.neighbor - && !(node.neighbor instanceof List) - ) - form_error_set << 'neighbor_not_a_list' - - if( - node.type == "path" - && (!node.build || !(node.build instanceof Closure)) - ) - form_error_set << 'no_build_code' - - if( - node.type == "symbol" - && node.build - ) - form_error_set << 'symbol_has_build_code' - - if( - node.mark - && (node.mark instance of Set) - && ! (node.mark.every {it in node_mark_set}) - ) - form_error_set << 'unregistered_mark' - - return form_error_set -} -def is_leaf_q = { node -> !node.neighbor || node.neighbor.isEmpty() } -def is_marked_q = { node -> node.mark && !node.mark.isEmpty() } - - -/*---------------------------------------- - maps node label -> node - - The keys are the node labels. They are formally added as the value to the - `label` property by the lookup function. - - Each leaf node must have a node definition. All path labels that lead to the - `javac/leaf` directory are recognized as leaf nodes. A leaf node has a - `type` property value of `leaf`, and they can be added to the map at different - file paths. - -*/ - -def node_map = [ - - "ANTLR_OUT_FL" : [ - ,type: "symbol" - ,neighbor: ["${dir_map.EXECUTOR_IN}/ANTLR_OUT_FL"] - ] - - ,"RuleNameList" : [ - ,type: "symbol" - ,neighbor: ["${dir_map.EXECUTOR_IN}/RuleNameList"] - ] - - ,"RuleNameListRegx" : [ - ,type: "symbol" - ,neighbor: ["${dir_map.EXECUTOR_IN}/RuleNameListRegx"] - ] - - ,"Synthesize_SyntaxAnnotate" : [ - ,type: "symbol" - ,neighbor: [ - "${dir_map.JAVA_COMP_IN_LEAF}/StringUtils.java" - ,"${dir_map.EXECUTOR_IN}/Synthesize_SyntaxAnnotate" - ] - ] - - ,"Synthesize_SyntaxAnnotate_PrintVisitor" : [ - ,type: "symbol" - ,neighbor: [ - "${dir_map.JAVA_COMP_IN_LEAF}/StringUtils.java" - ,"${dir_map.JAVA_COMP_IN_LEAF}/Synthesize_SyntaxAnnotate_PrintVisitorMethod.java" - ,"${dir_map.EXECUTOR_IN}/Synthesize_SyntaxAnnotate_PrintVisitor" - ] - ] - - ,"Synthesize_SyntaxAnnotate_PrintVisitorMethod" : [ - ,type: "symbol" - ,neighbor: [ - "${dir_map.JAVA_COMP_IN_LEAF}/StringUtils.java" - ,"${dir_map.EXECUTOR_IN}/Synthesize_SyntaxAnnotate_PrintVisitorMethod" - ] - ] -] - -//---------------------------------------- -// the following are recognizer functions that return node definitions. - -// any leaf node located in javac/leaf -def node_leaf_f(node_label) { - def leafNodePattern = ~/${dir_map['JAVA_COMP_IN_LEAF']}(.*)/ - def match = node_label =~ leafNodePattern - if (!match) { - return [status: "no_match"] - } - def baseName = match[0][1] - - def leafFilePath = "${dir_map['JAVA_COMP_IN_LEAF']}${baseName}" - def leafFile = new File(leafFilePath) - if (!leafFile.exists()) { - return [status: "no_match"] - } - - return [ - status: "matched" - ,label: node_label - ,type: "leaf" - ,neighbor: [] - ] -} - -// any shell wrapper (for wrapping a name corresponding .jar file) -def node_executor_f(node) { - - def match = node =~ /^(executor\/)(${base})$/ - if (!match) { - return [status: "no_match"] - } - def baseName = match[0][2] - - def jarFilePath = "${dir_map['JVM_IN']}${baseName}.jar" - def wrapperFilePath = "${dir_map['EXECUTOR_IN']}${baseName}" - - def jarFile = new File(jarFilePath) - if (!jarFile.exists()) { - return [status: "no_match"] - } - - return [ - status: "matched" - ,label: node - ,type: "path" - ,neighbor: [jarFilePath] - ,build: { node, neighbor -> - - // The script for wrapping the jar file: - def wrapper = """ - #!/usr/bin/env bash - ${dir_map['JAVA_INTERP']} -cp \${CLASSPATH}:${dir_map['JVM_IN']}:${dir_map['JVM_IN']}/${baseName}.jar ${baseName} \\\$\\@ - """ - - new File(wrapperFilePath).withWriter('UTF-8') { writer -> - writer.write(wrapper) - } - - println "Creating executable wrapper script for ${baseName} in executor directory." - "chmod +x ${wrapperFilePath}".execute().text - } - ] -} - -// any antlr output java file -def node_grammar_f(node) { - - def match = node =~ /^(${dir_map['ANTLR_OUT']})(${base})(Lexer|Parser|BaseListener|Listener|BaseVisitor|Visitor)\.java$/ - if( !match ){ - return [status: "no_match"] - } - - def grammarName = match[0][2] - def outputType = match[0][3] - - def grammarFilePath = "${dir_map['ANTLR_IN_LEAF']}${grammarName}.g4" - def grammarFile = new File(grammarFilePath) - - if( !grammarFile.exists() ){ - return [status: "no_match"] - } - - return [ - status: "matched" - ,label: node - ,type: "path" - ,neighbor: [grammarFilePath] - ] -} - -// any class file (built from a name corresponding .java file) -def node_class_f(node) { - - def match = node =~ /^(${dir_map['JAVA_COMP_OUT']})(${base})\.class$/ - if( !match ){ - return [status: "no_match"] - } - - def baseName = match[0][2] - def javaFilePath = "${dir_map['JAVA_COMP_IN_LEAF']}${baseName}.java" - def javaFile = new File(javaFilePath) - - if( !javaFile.exists() ){ - return [status: "no_match"] - } - - return [ - status: "matched" - ,label: node - ,type: "path" - ,neighbor: [javaFilePath] - ] -} - -// any jar file (built from a corresponding class file) -def node_jar_f(node) { - - // Use the symbolic name and base patterns - def match = node =~ /^(${dir_map['JAVA_COMP_OUT']})(${base})\.jar$/ - - if( !match ){ - return [status: "no_match"] - } - - def baseName = match[0][2] - def classFilePath = "${dir_map['JAVA_COMP_OUT']}${baseName}.class" - def classFile = new File(classFilePath) - - if( !classFile.exists() ){ - return [status: "no_match"] - } - - return [ - status: "matched" - ,label: node - ,type: "path" - ,neighbor: [classFilePath] - ,build: { node, neighbor -> - println "Building jar for ${baseName}" - def command = "${ext.javaHome}/bin/jar cf ${baseName}.jar -C ${dir_map['JAVA_COMP_OUT']} ${baseName}.class" - return command.execute().text; - } - ] -} - -// list of the recognizer functions -def node_f_list = [ - node_leaf_f - ,node)executor_f - ,node_grammar_f - ,node_class_f - ,node_jar_f -] - - -// combines the map and recognizer functions into a single lookup -def lookup(node_label, verbose = false){ - // the actual lookup - def lookup_node = node_map[node_label] - if( lookup_node ){ - lookup_node.label = node_label - } else { - def match_result - for( func in node_f_list ){ - match_result = func(node_label) - if( match_result.status == "matched" ){ - lookup_node = match_result - break - } - } - } - - if( !lookup_node ){ - if( verbose ) println "Lookup error: Node ${node_label} could not be found." - return null - } - - // If we already marked this one as malformed, do not check it again - // This is done primarily to avoid duplicate error messages, as it isn't a big - // performance win. - if( lookup_node.mark && ('malformed' in lookup_node.mark) ) return null - - // Check if the node is well formed - def form_errors = well_formed_q(lookup_node) - if( !form_errors.isEmpty() ){ - // Mark the node as malformed - if( !lookup_node.mark ) lookup_node.mark = [] as Set - lookup_node.mark << 'malformed' - - // Tell the user - if( verbose ){ - println "Lookup error: Node ${node_label} is malformed due to: ${form_errors.join(', ')}" - } - - // Refuse to return a malformed node - return null - } - - return lookup_node -} - -/*-------------------------------------------------------------------------------- - is_acyclic_q checks a dependency graph for cycles - - given a dependency graph. Returns either 'given_argument_not_accepted', - `cycles` or `acyclic`. - - marks the nodes that are in cycles. - - There is a call function and a helper function - -*/ - -def is_acyclic_q_descend(path_stack ,boolean verbose = true){ - def local_path = path_stack.collect{ node_label -> lookup(node_label) } - def local_node = local_path[-1] - def cycle_start_index - - do{ - // Check for a cycle in the local path - cycle_start_index = local_path[0..-2].findIndexOf{ node -> node.label == local_node.label } - if( cycle_start_index != -1 ){ // Cycle detected - if( verbose ) print "is_acyclic_q:: dependency cycle found:" - local_path[cycle_start_index..-1].each{ cycle_node -> - if( verbose ) print " ${cycle_node.label}" - cycle_node.mark = cycle_node.mark ?: [] as Set // Initialize mark set if needed - cycle_node.mark << 'cycle_member' - } - if( verbose ) println "" - // Pop off the stack above the cycle start, so that search can continue if desired - path_stack = path_stack[0..cycle_start_index] - return 'cycle_found' - } - - // Put this test after the cycle detector in case another definition with - // the same label had children and participated in a cycle. - if( local_node.neighbor.isEmpty() ) return 'leaf_node' - - // Descend further into the tree. - path_stack << local_node.neighbor.clone() - local_node = lookup(local_node.neighbor[0]) - local_path << local_node - }while( true ) -} - -def is_acyclic_q(root_node_labels ,boolean verbose = true){ - // We are optimistic. - def return_value = 'acyclic' - - // Initialize the DFS tree iterator. - def path_stack = [] - path_stack << root_node_labels.clone() - - // iterate over left side tree descent, not ideal as it starts at the - // root each time, but avoids complexity in the cycle detection logic. - do{ - def result = is_acyclic_q_descend(path_stack ,verbose) - if(result == 'cycle_found') return_value = result - - // increment the iterator to the next leftmost path - def top_list = path_stack[-1] - top_list.remove(0) - if( top_list.isEmpty() ) path_stack.pop() - - }while( !path_stack.isEmpty() ) - - return return_value -} - -// LocalWords: FN FPL DN DNL RuleNameListRegx RuleNameList PrintVisitorMethod -// LocalWords: PrintVisitor SyntaxAnnotate - -/*-------------------------------------------------------------------------------- - run the build scripts - depends upon is_acyclic having already marked up the graph. - -Initialization: -* visited: A set to keep track of visited nodes. -* build_order: A list to store the nodes in the order they should be built. - -Depth-First Search (DFS): -* The dfs function performs a depth-first traversal of the graph. -* It looks up the node using the lookup function. -* If the node has already been visited, it returns. -* Otherwise, it marks the node as visited and recursively visits its neighbors. -* After visiting all neighbors, it adds the node to the build_order list. - -Build Script Execution: -* After the DFS traversal, the build_order list is reversed to ensure that leaf nodes are built first. -* The build scripts for each node are executed in the correct order. - --- add date checks - -*/ - -def run_build_scripts(root_node_labels ,boolean verbose = false){ - def visited = [] as Set - def build_order = [] - - def dfs(node_label){ - def node = lookup(node_label ,verbose) - if( !node ) return - - if( node.label in visited ) return - visited << node.label - - node.neighbor.each{ neighbor_label -> - dfs(neighbor_label) - } - - build_order << node - } - - root_node_labels.each{ root_label -> - dfs(root_label) - } - - build_order.reverse().each{ node -> - if( node.build ){ - println "Running build script for ${node.label}" - node.build(node ,node.neighbor) - } - } -} - -// Example usage -def root_node_labels = ["ANTLR_OUT_FL" ,"RuleNameList"] -run_build_scripts(root_node_labels ,true) - - -def run_build_scripts(root_node_labels ,boolean verbose = false){ - if( root_node_labels.isEmpty() ) return - - def visited = [] as Set - def build_order = [] - def stack = [] - - root_node_labels.each{ root_label -> - stack << root_label - } - - do{ - def node_label = stack.pop() - def node = lookup(node_label ,verbose) - if( !node ) continue - - if( node.label in visited ) continue - visited << node.label - - node.neighbor.each{ neighbor_label -> - stack << neighbor_label - } - - build_order << node - }while( !stack.isEmpty() ) - - build_order.reverse().each{ node -> - if( node.build ){ - println "Running build script for ${node.label}" - node.build(node ,node.neighbor) - } - } -} - --------------------- - -def should_build(node, verbose = false){ - if( node.mark && ('malformed' in node.mark || 'cycle_member' in node.mark) ){ - if( verbose ) println "Skipping build for ${node.label} due to errors or cycle membership." - return false - } - - def nodeFile = new File(node.label) - if( !nodeFile.exists() ){ - if( verbose ) println "Node file ${node.label} does not exist." - return false - } - - def nodeLastModified = nodeFile.lastModified() - def shouldBuild = false - - node.neighbor.each{ neighbor_label -> - def neighborNode = lookup(neighbor_label, verbose) - if( !neighborNode || neighborNode.type != 'path' ){ - if( verbose ) println "Dependency ${neighbor_label} is missing or not a path type." - return false - } - - def neighborFile = new File(neighbor_label) - if( !neighborFile.exists() ){ - if( verbose ) println "Dependency file ${neighbor_label} does not exist." - return false - } - - def neighborLastModified = neighborFile.lastModified() - if( neighborLastModified > nodeLastModified ){ - shouldBuild = true - } - } - - return shouldBuild -} - -def run_build_scripts(root_node_labels ,boolean verbose = false){ - if( root_node_labels.isEmpty() ) return - - def visited = [] as Set - def build_order = [] - def stack = [] - - root_node_labels.each{ root_label -> - stack << root_label - } - - do{ - def node_label = stack.pop() - def node = lookup(node_label ,verbose) - if( !node ) continue - - if( node.label in visited ) continue - visited << node.label - - node.neighbor.each{ neighbor_label -> - stack << neighbor_label - } - - build_order << node - }while( !stack.isEmpty() ) - - build_order.reverse().each{ node -> - if( node.build && should_build(node ,verbose) ){ - println "Running build script for ${node.label}" - node.build(node ,node.neighbor) - } - } -} - -// Example usage -def root_node_labels = ["ANTLR_OUT_FL" ,"RuleNameList"] -run_build_scripts(root_node_labels ,true) - - - -// Example usage -def root_node_labels = ["ANTLR_OUT_FL" ,"RuleNameList"] -run_build_scripts(root_node_labels ,true) diff --git a/developer/build.gradle b/developer/build.gradle index ffc383d..d8d82d0 100644 --- a/developer/build.gradle +++ b/developer/build.gradle @@ -140,13 +140,29 @@ task preface { //--------------------------------------- // some helpers -def node_type_set = ['symbol' ,'path'] as Set -def node_mark_set = ['cycle_member' ,'malformed' ] as Set +def all_node_type_set = ['symbol' ,'path' ,'leaf'] as Set +def all_node_mark_set = ['cycle_member' ,'wellformed' ] as Set -/* - */ +def is_leaf_q = { node -> node.type && node.type == 'leaf' } +def is_marked_q = { node -> node.mark && !node.mark.isEmpty() } + +def is_marked_good_q(node){ + return node.mark && ('wellformed' in node.mark) && !('cycle_member' in node.mark) +} -def well_formed_q = { node -> +def all_form_error_set = [ + 'no_node_label' + ,'no_such_node_type' + ,'neighbor_property_value_not_a_list' + ,'unregistered_mark' + ,'symbol_has_build_code' + ,'symbol_has_no_dependencies' + ,'missing_required_build_code' + ,'build_target_has_no_dependencies' + ,'leaf_has_neighbor_property' + ,'leaf_has_neighbor' +] +def wellformed_q = { node -> def form_error_set = [] as Set if( @@ -155,39 +171,64 @@ def well_formed_q = { node -> form_error_set << 'no_node_label' if( - !node.type || !(node.type in node_type_set) + !node.type || !(node.type in all_node_type_set) ) - form_error_set << 'no_node_type' + form_error_set << 'no_such_node_type' if( node.neighbor && !(node.neighbor instanceof List) ) - form_error_set << 'neighbor_not_a_list' + form_error_set << 'neighbor_property_value_not_a_list' + + if( + node.mark + && (node.mark instance of Set) + && ! (node.mark.every {it in all_node_mark_set}) + ) + form_error_set << 'unregistered_mark' + + if( + node.type == "symbol" + && node.build + ) + form_error_set << 'symbol_has_build_code' + + if( + node.type == "symbol" + && ( !node.neighbor || node.neighbor.isEmpty() ) + ) + form_error_set << 'symbol_has_no_dependencies' if( node.type == "path" && (!node.build || !(node.build instanceof Closure)) ) - form_error_set << 'no_build_code' + form_error_set << 'missing_required_build_code' + + if( + node.type == "path" + && ( !node.neighbor || node.neighbor.isEmpty() ) + ) + form_error_set << 'build_target_has_no_dependencies' if( node.type == "symbol" - && node.build + && ( node.neighbor || node.neighbor.isEmpty() ) ) - form_error_set << 'symbol_has_build_code' + form_error_set << 'symbol_has_no_dependencies' if( - node.mark - && (node.mark instance of Set) - && ! (node.mark.every {it in node_mark_set}) - ) - form_error_set << 'unregistered_mark' + node.type == "leaf" + && node.neighbor + ){ + form_error_set << 'leaf_has_neighbor_property' + if( !node.neighbor.isEmpty() ) + form_error_set << 'leaf_has_neighbor' + } return form_error_set } -def is_leaf_q = { node -> !node.neighbor || node.neighbor.isEmpty() } -def is_marked_q = { node -> node.mark && !node.mark.isEmpty() } /*---------------------------------------- @@ -393,7 +434,7 @@ def node_jar_f(node) { ] } -// list of the recognizer functions +// list of the pattern recognizer functions def node_f_list = [ node_leaf_f ,node)executor_f @@ -403,17 +444,16 @@ def node_f_list = [ ] -// combines the map and recognizer functions into a single lookup +// Given a node_label, returns the node or null. def lookup(node_label, verbose = false){ - // the actual lookup def lookup_node = node_map[node_label] if( lookup_node ){ lookup_node.label = node_label } else { def match_result - for( func in node_f_list ){ + for(func in node_f_list){ match_result = func(node_label) - if( match_result.status == "matched" ){ + if(match_result.status == "matched"){ lookup_node = match_result break } @@ -425,77 +465,118 @@ def lookup(node_label, verbose = false){ return null } - // If we already marked this one as malformed, do not check it again - // This is done primarily to avoid duplicate error messages, as it isn't a big - // performance win. - if( lookup_node.mark && ('malformed' in lookup_node.mark) ) return null - - // Check if the node is well formed - def form_errors = well_formed_q(lookup_node) - if( !form_errors.isEmpty() ){ - // Mark the node as malformed - if( !lookup_node.mark ) lookup_node.mark = [] as Set - lookup_node.mark << 'malformed' - - // Tell the user - if( verbose ){ - println "Lookup error: Node ${node_label} is malformed due to: ${form_errors.join(', ')}" - } - - // Refuse to return a malformed node - return null - } - return lookup_node } +// Given a node_label, if found, returns node if it is marked good. +def lookup_marked_good(node_label, verbose = false){ + def node = lookup(node_label, verbose) + if( node && is_marked_good_q(node) ) return node; + return null; +} + + + + /*-------------------------------------------------------------------------------- - is_acyclic_q checks a dependency graph for cycles + do_markup_graph checks given a graph, marks nodes in the graph, and returns a set + potentially with the marks 'given_argument_not_accepted', `wellformed`, + `acyclic`. - given a dependency graph. Returns either 'given_argument_not_accepted', - `cycles` or `acyclic`. + Well formed nodes are marked, 'wellformed'. Nodes that are part of a cycle + are marked 'cycle_member'. - marks the nodes that are in cycles. + */ - There is a call function and a helper function +// Given a node label list. Looks up and marks the nodes in the list. Returns +// 'all_wellformed' if all nodes are wellformed, otherwise it returns +// 'exists_malformed'. +def mark_the_wellformed_f(node_labels, boolean verbose = true){ + def all_wellformed = true + + def neighbors = node_labels.collect{ neighbor_label -> + def neighbor_node = lookup(neighbor_label) + def form_errors = wellformed_q(neighbor_node) + if(form_errors.isEmpty()){ + neighbor_node.mark = neighbor_node.mark ?: [] as Set + neighbor_node.mark << 'wellformed' + } else { + all_wellformed = false + if(verbose){ + if(neighbor_node.label && neighbor_node.label.length() > 0){ + print("node ${neighbor_node.label} is malformed due to:") + } else { + print("anonymous node is malformed due to:") + } + form_errors.each { error -> print(" ${error}") } + println("") + } + } + neighbor_label + } -*/ + return all_wellformed ? 'all_wellformed' : 'exists_malformed' +} -def is_acyclic_q_descend(path_stack ,boolean verbose = true){ - def local_path = path_stack.collect{ node_label -> lookup(node_label) } - def local_node = local_path[-1] +def markup_graph_f_descend(path_stack, boolean verbose = true){ + def ret_value = [] as Set + def local_path = path_stack.collect{ it[0] } + def local_node_label = local_path[-1] def cycle_start_index do{ // Check for a cycle in the local path - cycle_start_index = local_path[0..-2].findIndexOf{ node -> node.label == local_node.label } - if( cycle_start_index != -1 ){ // Cycle detected - if( verbose ) print "is_acyclic_q:: dependency cycle found:" - local_path[cycle_start_index..-1].each{ cycle_node -> - if( verbose ) print " ${cycle_node.label}" + cycle_start_index = local_path[0..-2].findIndexOf{ it == local_node_label } + if(cycle_start_index != -1){ // Cycle detected + ret_value << 'cycle_found' + if(verbose) print "markup_graph_f_descend:: dependency cycle found:" + local_path[cycle_start_index..-1].each{ cycle_node_label -> + def cycle_node = lookup(cycle_node_label) + if(verbose) print " ${cycle_node.label}" cycle_node.mark = cycle_node.mark ?: [] as Set // Initialize mark set if needed cycle_node.mark << 'cycle_member' } - if( verbose ) println "" - // Pop off the stack above the cycle start, so that search can continue if desired + if(verbose) println "" + // we can not continue searching after the loop so, we pop back to treat + // the first node in the loop as though a leaf node. path_stack = path_stack[0..cycle_start_index] - return 'cycle_found' + return ret_value } - // Put this test after the cycle detector in case another definition with - // the same label had children and participated in a cycle. - if( local_node.neighbor.isEmpty() ) return 'leaf_node' + // a 'de-facto' leaf node test subtleties here because we have not yet + // determined if the nodes we are wellformed. This is purposeful, as + // this function does not know about the relationships between the + // possible error marks. + def local_node = lookup(local_node_label) + if(local_node.neighbor.isEmpty()){ + ret_value << 'defacto_leaf_node' + return ret_value + } + // Mark the wellformed nodes and get the result + def result = mark_the_wellformed_f(local_node.neighbor, verbose) + if(result == 'exists_malformed'){ + ret_value << 'exists_malformed' + } + // Descend further into the tree. path_stack << local_node.neighbor.clone() - local_node = lookup(local_node.neighbor[0]) - local_path << local_node - }while( true ) + local_node_label = local_node.neighbor[0] + local_path << local_node_label + }while(true) } -def is_acyclic_q(root_node_labels ,boolean verbose = true){ - // We are optimistic. - def return_value = 'acyclic' +// given root_node_labels, marks up graph, returns a set possibly +// containing 'all_wellformed' and 'cycles_exist'. +def markup_graph_f(root_node_labels, boolean verbose = true){ + def ret_value = [] as Set + def exists_malformed = false; + + // check the root nodes + def result = mark_the_wellformed_f(root_node_labels, verbose) + if(result == 'exists_malformed'){ + ret_value << 'exists_malformed' + } // Initialize the DFS tree iterator. def path_stack = [] @@ -504,26 +585,35 @@ def is_acyclic_q(root_node_labels ,boolean verbose = true){ // iterate over left side tree descent, not ideal as it starts at the // root each time, but avoids complexity in the cycle detection logic. do{ - def result = is_acyclic_q_descend(path_stack ,verbose) - if(result == 'cycle_found') return_value = result + def result = markup_graph_f_descend(path_stack, verbose) + if('cycle_found' in result) ret_value << 'cycle_exists' + if('exists_malformed' in result) exists_malformed = true; // increment the iterator to the next leftmost path def top_list = path_stack[-1] top_list.remove(0) - if( top_list.isEmpty() ) path_stack.pop() + if(top_list.isEmpty()) path_stack.pop() - }while( !path_stack.isEmpty() ) + }while(!path_stack.isEmpty()) - return return_value + if(!exists_malformed) ret_value << 'all_wellformed' + if( verbose ){ + if(exists_malformed) println("one or more malformed nodes were found") + def exists_cycle = 'cycle_found' in ret_value + if(exists_cycle) println("one or more cyclic dependency loop found") + if( exists_malformed || exists_cycle ) println("will attempt to build unaffected nodes") + } + + return ret_value } // LocalWords: FN FPL DN DNL RuleNameListRegx RuleNameList PrintVisitorMethod -// LocalWords: PrintVisitor SyntaxAnnotate +// LocalWords: PrintVisitor SyntaxAnnotate wellformed defacto /*-------------------------------------------------------------------------------- run the build scripts depends upon is_acyclic having already marked up the graph. - +o Initialization: * visited: A set to keep track of visited nodes. * build_order: A list to store the nodes in the order they should be built. @@ -543,46 +633,140 @@ Build Script Execution: */ -def run_build_scripts(root_node_labels ,boolean verbose = false){ - def visited = [] as Set - def build_order = [] +import java.nio.file.Files +import java.nio.file.Paths - def dfs(node_label){ - def node = lookup(node_label ,verbose) - if( !node ) return +def good_dependency_q( node ){ + if( node.type in ['path' ,'leaf'] ){ + def node_path = Paths.get( node.label ) + if( !Files.exists( node_path ) ) return false + } + return ( + node.mark + && ( 'wellformed' in node.mark ) + && !( 'build_failed' in node.mark ) + && !( 'cycle_member' in node.mark ) + ) +} - if( node.label in visited ) return - visited << node.label +def build_status_q( node, verbose = true ){ + if( node.type == 'leaf' ){ + def node_path = Paths.get( node.label ) + if( !Files.exists( node_path ) ){ + node.mark.add( 'build_failed' ) // so that it will not be a good_dependency + if( verbose ) println( "Leaf node ${node.label} is missing." ) + } + return 'leaf' + } - node.neighbor.each{ neighbor_label -> - dfs(neighbor_label) + if( node.type == 'symbol' ){ + def dependencies = node.neighbor.findAll{ neighbor_label -> + def neighbor_node = lookup( neighbor_label ) + return good_dependency_q( neighbor_node ) } + if( dependencies.size() != node.neighbor.size() ) return 'dependency_problem' + return 'should_build' + } - build_order << node + if( node.type == 'path' ){ + def node_path = Paths.get( node.label ) + if( !Files.exists( node_path ) ) return 'should_build' + + def node_last_modified = Files.getLastModifiedTime( node_path ).toMillis() + + def dependencies = node.neighbor.findAll{ neighbor_label -> + def neighbor_node = lookup( neighbor_label ) + return good_dependency_q( neighbor_node ) + } + if( dependencies.size() != node.neighbor.size() ) return 'dependency_problem' + + def dependency_last_modified = dependencies.collect{ neighbor_label -> + def neighbor_node = lookup( neighbor_label ) + def neighbor_path = Paths.get( neighbor_node.label ) + return Files.getLastModifiedTime( neighbor_path ).toMillis() + } + + if( dependency_last_modified.any{ it > node_last_modified } ) return 'should_build' } + return 'up_to_date' +} + +def run_build_scripts_f( root_node_labels, boolean verbose = true ){ + if( root_node_labels.isEmpty() ) return + + def visited = [] as Set + def build_order = [] + def stack = [] + root_node_labels.each{ root_label -> - dfs(root_label) + stack << root_label } + do{ + def node_label = stack.pop() + def node = lookup( node_label, verbose ) + if( !node ) continue + + if( node.label in visited ) continue + visited << node.label + + if( !is_marked_good_q( node ) ){ + if( verbose ) println( "Skipping malformed node ${node.label}" ) + continue + } + + // Clear the build_failed mark + node.mark.remove( 'build_failed' ) + + // reverse to be consistent in using the left subtree first + node.neighbor.reverse().each{ neighbor_label -> + stack << neighbor_label + } + + build_order << node + }while( !stack.isEmpty() ) + build_order.reverse().each{ node -> - if( node.build ){ - println "Running build script for ${node.label}" - node.build(node ,node.neighbor) + def build_status = build_status_q( node, verbose ) + if( build_status == 'should_build' ){ + println( "Running build script for ${node.label}" ) + node.build( node, node.neighbor ) + + // Check if the build updated the target file + if( build_status_q( node, verbose ) == 'should_build' ){ + println( "Build failed for ${node.label}" ) + node.mark = node.mark ?: [] as Set + node.mark << 'build_failed' + } + } else if( build_status == 'dependency_problem' ){ + println( "Skipping build for ${node.label} due to dependency problems" ) + } else if( build_status == 'leaf' ){ + println( "Skipping build for leaf node ${node.label}" ) } } } -// Example usage -def root_node_labels = ["ANTLR_OUT_FL" ,"RuleNameList"] -run_build_scripts(root_node_labels ,true) +def mark_for_clean_q( node ){ + if( node.type == 'leaf' ) return false -def run_build_scripts(root_node_labels ,boolean verbose = false){ + if( node.type == 'symbol' || node.type == 'path' ){ + def dependencies = node.neighbor.findAll{ neighbor_label -> + def neighbor_node = lookup( neighbor_label ) + return good_dependency_q( neighbor_node ) + } + return dependencies.size() == node.neighbor.size() + } + + return false +} + +def clean_nodes( root_node_labels, boolean verbose = true ){ if( root_node_labels.isEmpty() ) return def visited = [] as Set - def build_order = [] + def clean_order = [] def stack = [] root_node_labels.each{ root_label -> @@ -591,27 +775,45 @@ def run_build_scripts(root_node_labels ,boolean verbose = false){ do{ def node_label = stack.pop() - def node = lookup(node_label ,verbose) + def node = lookup( node_label, verbose ) if( !node ) continue if( node.label in visited ) continue visited << node.label - node.neighbor.each{ neighbor_label -> + if( !is_marked_good_q( node ) ){ + if( verbose ) println( "Skipping malformed node ${node.label}" ) + continue + } + + // reverse to be consistent in using the left subtree first + node.neighbor.reverse().each{ neighbor_label -> stack << neighbor_label } - build_order << node + clean_order << node }while( !stack.isEmpty() ) - build_order.reverse().each{ node -> - if( node.build ){ - println "Running build script for ${node.label}" - node.build(node ,node.neighbor) + clean_order.reverse().each{ node -> + if( mark_for_clean_q( node ) ){ + def can_clean = true + node.uses?.each{ used_label -> + def used_node = lookup( used_label ) + if( used_node && !mark_for_clean_q( used_node ) ){ + can_clean = false + if( verbose ) println( "Cannot clean ${node.label} because it uses ${used_node.label}" ) + } + } + if( can_clean ){ + println( "Cleaning node ${node.label}" ) + node.clean( node, node.neighbor ) + } else { + if( verbose ) println( "Skipping node ${node.label} due to dependency problems" ) + } + } else if( node.type == 'leaf' ){ + if( verbose ) println( "Skipping leaf node ${node.label}" ) + } else { + if( verbose ) println( "Skipping node ${node.label} due to dependency problems" ) } } } - -// Example usage -def root_node_labels = ["ANTLR_OUT_FL" ,"RuleNameList"] -run_build_scripts(root_node_labels ,true) diff --git a/developer/ologist/node_labels_unique_q.txt b/developer/ologist/node_labels_unique_q.txt new file mode 100644 index 0000000..b978a74 --- /dev/null +++ b/developer/ologist/node_labels_unique_q.txt @@ -0,0 +1,15 @@ + +predicate == is_well_formed_q + +We can not check that the node labels are unique because the given value +is a single node, and the code is stateless. Besides there is no contract with +the programmer on how to use the predicated, so the programmer could call the +predicate multiple times on the same node. Now can we test this condition in +our do_markup_graph routine because of the way lookup works, it will always +return the same node for the same label. This would be a truly difficult check +to perform because the map does not given an error but just takes the second of +the duplicate key definitions (is this really true?) besides, it would require +a formal proof of the recognizer functions that they do not return different +definitions for different keys to match regexprs against. I've been mulling +this over. As we currently the programmer provides the map and function +definitions, we don't even know which nodes will be in the graph...