From ee70f46012ceb9b5bcf979fade19336e5d0fe5a0 Mon Sep 17 00:00:00 2001 From: Daniel Jones Date: Sun, 3 Mar 2013 17:50:00 -0800 Subject: [PATCH] Concurrent rule execution. --- expand.go | 38 +++++++++- graph.go | 206 +++++++++++++++++++++++++++++++++++++++++++++++++++++ mk.go | 171 +++++++++++++++++++++++++++++++++++++++++--- parse.go | 6 +- recipe.go | 110 +++++++++++++++++++++++++--- ruleset.go | 18 ++--- 6 files changed, 518 insertions(+), 31 deletions(-) create mode 100644 graph.go diff --git a/expand.go b/expand.go index 494da80..7f4a58f 100644 --- a/expand.go +++ b/expand.go @@ -142,7 +142,7 @@ func expandSigil(input string, vars map[string][]string) ([]string, int) { i := 0 j := i for j < len(input) { - c, w = utf8.DecodeRuneInString(input) + c, w = utf8.DecodeRuneInString(input[j:]) if !(isalpha(c) || c == '_' || (j > i && isdigit(c))) { break } @@ -196,6 +196,39 @@ func expandSigils(input string, vars map[string][]string) []string { } +// Find and expand all sigils in a recipe, producing a flat string. +func expandRecipeSigils(input string, vars map[string][]string) string { + expanded := "" + for i := 0; i < len(input); { + j := strings.IndexAny(input[i:], "$\\") + if j < 0 { + expanded += input[i:] + break + } + + expanded += input[i:j] + i = j + c, w := utf8.DecodeRuneInString(input[i:]) + if c == '$' { + i += w + ex, k := expandSigil(input[i:], vars) + expanded += strings.Join(ex, " ") + i += k + } else if c == '\\' { + i += w + c, w := utf8.DecodeRuneInString(input[i:]) + if c == '$' { + expanded += "$" + } else { + expanded += "\\" + string(c) + } + i += w + } + } + + return expanded +} + // Expand all unescaped '%' characters. func expandSuffixes(input string, stem string) string { @@ -236,7 +269,8 @@ func expandBackQuoted(input string, vars map[string][]string) (string, int) { return input, len(input) } - output := executeRecipe("sh", nil, input[:j], false, false, true) + // TODO: handle errors + output, _ := subprocess("sh", nil, input[:j], false, false, true) return output, (j + 1) } diff --git a/graph.go b/graph.go new file mode 100644 index 0000000..3be5663 --- /dev/null +++ b/graph.go @@ -0,0 +1,206 @@ + +package main + +import ( + "fmt" + "io" + "os" + "sync" + "time" +) + +// A dependency graph +type graph struct { + root *node // the intial target's node + nodes map[string]*node // map targets to their nodes +} + +// An edge in the graph. +type edge struct { + v *node // node this edge directs to + stem string // stem matched for meta-rule applications + matches []string // regular expression matches + r *rule +} + + +// Current status of a node in the build. +type nodeStatus int + +const ( + nodeStatusReady nodeStatus = iota + nodeStatusStarted + nodeStatusDone + nodeStatusFailed +) + + +// A node in the dependency graph +type node struct { + r *rule // rule to be applied + name string // target name + prog string // custom program to compare times + t time.Time // file modification time + exists bool // does a non-virtual target exist + prereqs []*edge // prerequisite rules + status nodeStatus // current state of the node in the build + mutex sync.Mutex // exclusivity for the status variable + listeners []chan nodeStatus // channels to notify of completion +} + +// Create a new node +func (g *graph) newnode(name string) *node { + u := &node{name: name} + info, err := os.Stat(name) + if err == nil { + u.t = info.ModTime() + u.exists = true + } else { + _, ok := err.(*os.PathError) + if ok { + u.exists = false + } else { + mkError(err.Error()) + } + } + g.nodes[name] = u + return u +} + +// Print a graph in graphviz format. +func (g *graph) visualize(w io.Writer) { + fmt.Fprintln(w, "digraph mk {") + for t, u := range g.nodes { + for i := range u.prereqs { + if u.prereqs[i].v != nil { + fmt.Fprintf(w, " \"%s\" -> \"%s\";\n", t, u.prereqs[i].v.name) + } + } + } + fmt.Fprintln(w, "}") +} + + +// Create a new arc. +func (u *node) newedge(v *node, r *rule) *edge { + e := &edge{v: v, r: r} + u.prereqs = append(u.prereqs, e) + return e +} + + +// Create a dependency graph for the given target. +func buildgraph(rs *ruleSet, target string) *graph { + g := &graph{nil, make(map[string]*node)} + + // keep track of how many times each rule is visited, to avoid cycles. + rulecnt := make([]int, len(rs.rules)) + g.root = applyrules(rs, g, target, rulecnt) + + return g +} + +// Recursively match the given target to a rule in the rule set to construct the +// full graph. +func applyrules(rs *ruleSet, g *graph, target string, rulecnt []int) *node { + u, ok := g.nodes[target] + if ok { + return u + } + u = g.newnode(target) + + // does the target match a concrete rule? + + ks, ok := rs.targetrules[target] + if ok { + for ki := range ks { + k := ks[ki] + if rulecnt[k] > max_rule_cnt { + continue + } + + r := &rs.rules[k] + + // skip meta-rules + if r.ismeta { + continue + } + + // skip rules that have no effect + if r.recipe == "" && len(r.prereqs) == 0 { + continue + } + + rulecnt[k] += 1 + if len(r.prereqs) == 0 { + u.newedge(nil, r) + } else { + for i := range r.prereqs { + u.newedge(applyrules(rs, g, r.prereqs[i], rulecnt), r) + } + } + rulecnt[k] -= 1 + } + } + + // find applicable metarules + for k := range rs.rules { + if rulecnt[k] > max_rule_cnt { + continue + } + + r := &rs.rules[k] + + if !r.ismeta { + continue + } + + // skip rules that have no effect + if r.recipe == "" && len(r.prereqs) == 0 { + continue + } + + for j := range r.targets { + mat := r.targets[j].match(target) + if mat == nil { + continue + } + + var stem string + var matches []string + + if r.attributes.regex { + matches = mat + } else { + stem = mat[1] + } + + rulecnt[k] += 1 + if len(r.prereqs) == 0 { + e := u.newedge(nil, r) + e.stem = stem + e.matches = matches + } else { + for i := range r.prereqs { + var prereq string + if r.attributes.regex { + // TODO: write substituteRegexpRefs and use that here + prereq = r.prereqs[i] + } else { + prereq = expandSuffixes(r.prereqs[i], stem) + } + + e := u.newedge(applyrules(rs, g, prereq, rulecnt), r) + e.stem = stem + e.matches = matches + } + } + rulecnt[k] -= 1 + } + } + + return u +} + + + diff --git a/mk.go b/mk.go index 0d32770..8eed45c 100644 --- a/mk.go +++ b/mk.go @@ -8,25 +8,175 @@ import ( ) +// True if messages should be printed without fancy colors. +var nocolor bool = false + +// True if we are no actualyl executing any recipes or updating any timestamps. +var dryrun bool = false + // The maximum number of times an rule may be applied. const max_rule_cnt = 3 +// Ansi color codes. +const ( + colorDefault string = "\033[0m" + colorBlack string = "\033[30m" + colorRed string = "\033[31m" + colorGreen string = "\033[32m" + colorYellow string = "\033[33m" + colorBlue string = "\033[34m" + colorMagenta string = "\033[35m" +) + func mk(rs *ruleSet, target string, dryrun bool) { - - // Build a graph - - - - // 1. Introduce special variables into the ruleSet + g := buildgraph(rs, target) + //g.visualize(os.Stdout) + mkNode(g, g.root) } +// Build a target in the graph. +// +// This selects an appropriate rule (edge) and builds all prerequisites +// concurrently. +// +// TODO: control the number of concurrent rules being built. This would involve +// some sort of global counter protected by a mutex, so we wait our turn to +// execute our rule. +// +func mkNode(g *graph, u *node) { + // try to claim on this node + u.mutex.Lock() + if u.status != nodeStatusReady { + u.mutex.Unlock() + return + } else { + u.status = nodeStatusStarted + } + u.mutex.Unlock() + + // when finished, notify the listeners + finalstatus := nodeStatusDone + defer func () { + u.mutex.Lock() + u.status = finalstatus + u.mutex.Unlock() + for i := range u.listeners { + u.listeners[i] <- u.status + } + }() + + // there's no fucking rules, dude + if len(u.prereqs) == 0 { + if !u.r.attributes.virtual && !u.exists { + wd, _ := os.Getwd() + mkError(fmt.Sprintf("don't know how to make %s in %s", u.name, wd)) + } + return + } + + // there should otherwise be exactly one edge with an associated rule + prereqs := make([]*node, 0) + var e *edge = nil + for i := range u.prereqs { + if u.prereqs[i].r != nil { + e = u.prereqs[i] + } + if u.prereqs[i].v != nil { + prereqs = append(prereqs, u.prereqs[i].v) + } + } + + // this should have been caught during graph building + if e == nil { + wd, _ := os.Getwd() + mkError(fmt.Sprintf("don't know how to make %s in %s", u.name, wd)) + } + + prereqstat := make(chan nodeStatus) + pending := 0 + + // build prereqs that need building + e.r.mutex.Lock() + for i := range prereqs { + prereqs[i].mutex.Lock() + // needs to be built? + if !prereqs[i].exists || e.r.attributes.virtual || (u.exists && u.t.Before(prereqs[i].t)) { + switch prereqs[i].status { + case nodeStatusReady: + go mkNode(g, prereqs[i]) + fallthrough + case nodeStatusStarted: + prereqs[i].listeners = append(prereqs[i].listeners, prereqstat) + pending++ + } + } + prereqs[i].mutex.Unlock() + } + e.r.mutex.Unlock() + + // wait until all the prereqs are built + //fmt.Printf("%s: %d\n", u.name, pending) + for pending > 0 { + //for i := range prereqs { + //fmt.Println(prereqs[i].name) + //} + + s := <-prereqstat + pending-- + if s == nodeStatusFailed { + finalstatus = nodeStatusFailed + } + } + + // execute the recipe, unless the prereqs failed + if finalstatus != nodeStatusFailed { + mkPrintMessage("mking " + u.name) + if !dorecipe(u.name, u, e) { + finalstatus = nodeStatusFailed + } + } + + mkPrintSuccess("finished mking " + u.name) +} func mkError(msg string) { + if !nocolor { + os.Stderr.WriteString(colorRed) + } fmt.Fprintf(os.Stderr, "mk: %s\n", msg) + if !nocolor { + os.Stderr.WriteString(colorDefault) + } os.Exit(1) } +func mkPrintSuccess(msg string) { + if nocolor { + fmt.Println(msg) + } else { + fmt.Printf("%s%s%s\n", colorGreen, msg, colorDefault) + } +} + +func mkPrintMessage(msg string) { + if nocolor { + fmt.Println(msg) + } else { + fmt.Printf("%s%s%s\n", colorBlue, msg, colorDefault) + } +} + +func mkPrintRecipe(msg string) { + if !nocolor { + os.Stdout.WriteString(colorYellow) + } + printIndented(os.Stdout, msg) + if !nocolor { + os.Stdout.WriteString(colorDefault) + } +} + func main() { var mkfilepath string @@ -51,6 +201,7 @@ func main() { for j := range rs.rules[i].targets { targets = append(targets, rs.rules[i].targets[j].spat) } + break } } @@ -59,11 +210,9 @@ func main() { return } + // TODO: For multiple targets, we should add a dummy rule that depends on + // all let mk handle executing each. for _, target := range targets { - //fmt.Printf("building: %q\n", target) - g := buildgraph(rs, target) - g.visualize(os.Stdout) - //mk(rs, target, dryrun) + mk(rs, target, dryrun) } - } diff --git a/parse.go b/parse.go index 6c950b5..68d88b5 100644 --- a/parse.go +++ b/parse.go @@ -114,7 +114,11 @@ func parsePipeInclude(p *parser, t token) parserStateFun { args[i-1] = p.tokenbuf[i].val } - output := executeRecipe("sh", args, "", false, false, true) + output, success := subprocess("sh", args, "", false, false, true) + if !success { + p.basicErrorAtToken("subprocess include failed", t) + } + parseInto(output, fmt.Sprintf("%s:sh", p.name), p.rules) p.clear() diff --git a/recipe.go b/recipe.go index ed32325..994f6a5 100644 --- a/recipe.go +++ b/recipe.go @@ -1,3 +1,6 @@ + +// Various function for dealing with recipes. + package main import ( @@ -5,27 +8,111 @@ import ( "log" "os" "os/exec" + "bufio" + "fmt" + "strings" ) -// A monolithic function for executing recipes. -func executeRecipe(program string, + +// Try to unindent a recipe, so that it begins an column 0. (This is mainly for +// recipes in python, or other indentation-significant languages.) +//func stripIndentation(s string) string { + +//} + +// Indent each line of a recipe. +func printIndented(out io.Writer, s string) { + reader := bufio.NewReader(strings.NewReader(s)) + for { + line, err := reader.ReadString('\n') + if len(line) > 0 { + io.WriteString(out, " ") + io.WriteString(out, line) + } + + if (err != nil) { + break + } + } +} + +// Execute a recipe. +func dorecipe(target string, u *node, e *edge) bool { + vars := make(map[string][]string) + vars["target"] = []string{target} + if e.r.ismeta { + if e.r.attributes.regex { + for i := range e.matches { + vars[fmt.Sprintf("stem%d", i)] = e.matches[i:i+1] + } + } else { + vars["stem"] = []string{e.stem} + } + } + + // TODO: other variables to set + // alltargets + // newprereq + + prereqs := make([]string, 0) + for i := range u.prereqs { + if u.prereqs[i].r == e.r && u.prereqs[i].v != nil { + prereqs = append(prereqs, u.prereqs[i].v.name) + } + } + vars["prereqs"] = prereqs + + input := expandRecipeSigils(e.r.recipe, vars) + sh := "sh" + args := []string{} + + if len(e.r.shell) > 0 { + sh = e.r.shell[0] + args = e.r.shell[1:] + } + + if !e.r.attributes.quiet { + mkPrintRecipe(input) + } + + if dryrun { + return true + } + + _, success := subprocess( + sh, + args, + input, + true, + true, + false) + + + // TODO: update the timestamps of each target + + return success +} + + +// A monolithic function for executing subprocesses +func subprocess(program string, args []string, input string, echo_out bool, echo_err bool, - capture_out bool) string { + capture_out bool) (string, bool) { cmd := exec.Command(program, args...) if echo_out { cmdout, err := cmd.StdoutPipe() - if err != nil { + if err == nil { go io.Copy(os.Stdout, cmdout) } } if echo_err { - cmderr, err := cmd.StdoutPipe() - if err != nil { + cmderr, err := cmd.StderrPipe() + if err == nil { go io.Copy(os.Stderr, cmderr) } } @@ -49,11 +136,16 @@ func executeRecipe(program string, } else { err = cmd.Run() } + success := true if err != nil { - // TODO: better error output - log.Fatal("Recipe failed") + exiterr, ok := err.(*exec.ExitError) + if ok { + success = exiterr.ProcessState.Success() + } else { + log.Fatal(err) + } } - return output + return output, success } diff --git a/ruleset.go b/ruleset.go index b781d0d..caf3402 100644 --- a/ruleset.go +++ b/ruleset.go @@ -8,6 +8,7 @@ import ( "fmt" "unicode/utf8" "regexp" + "sync" ) type attribSet struct { @@ -52,13 +53,14 @@ func (p *pattern) match(target string) []string { // A single rule. type rule struct { - targets []pattern // non-empty array of targets - attributes attribSet // rule attributes - prereqs []string // possibly empty prerequesites - shell []string // command used to execute the recipe - recipe string // recipe source - command []string // command attribute - ismeta bool // is this a meta rule + targets []pattern // non-empty array of targets + attributes attribSet // rule attributes + prereqs []string // possibly empty prerequesites + shell []string // command used to execute the recipe + recipe string // recipe source + command []string // command attribute + ismeta bool // is this a meta rule + mutex sync.Mutex // prevent the rule from being executed multiple times } @@ -122,7 +124,7 @@ func (r *rule) parseAttribs(inputs []string) *attribError { // Add a rule to the rule set. func (rs *ruleSet) add(r rule) { - rs.rules = append(rs.rules, r) + rs.rules = append(rs.rules, r) k := len(rs.rules) - 1 for i := range r.targets { if r.targets[i].rpat == nil {