Concurrent rule execution.

expand.go

@@ -142,7 +142,7 @@ func expandSigil(input string, vars map[string][]string) ([]string, int) {
         i := 0
         j := i
         for j < len(input) {
-            c, w = utf8.DecodeRuneInString(input)
+            c, w = utf8.DecodeRuneInString(input[j:])
             if !(isalpha(c) || c == '_' || (j > i && isdigit(c))) {
                 break
             }
@@ -196,6 +196,39 @@ func expandSigils(input string, vars map[string][]string) []string {
 }
 
 
+// Find and expand all sigils in a recipe, producing a flat string.
+func expandRecipeSigils(input string, vars map[string][]string) string {
+    expanded := ""
+    for i := 0; i < len(input); {
+        j := strings.IndexAny(input[i:], "$\\")
+        if j < 0 {
+            expanded += input[i:]
+            break
+        }
+
+        expanded += input[i:j]
+        i = j
+        c, w := utf8.DecodeRuneInString(input[i:])
+        if c == '$' {
+            i += w
+            ex, k := expandSigil(input[i:], vars)
+            expanded += strings.Join(ex, " ")
+            i += k
+        } else if c == '\\' {
+            i += w
+            c, w := utf8.DecodeRuneInString(input[i:])
+            if c == '$' {
+                expanded += "$"
+            } else {
+                expanded += "\\" + string(c)
+            }
+            i += w
+        }
+    }
+
+    return expanded
+}
+
 
 // Expand all unescaped '%' characters.
 func expandSuffixes(input string, stem string) string {
@@ -236,7 +269,8 @@ func expandBackQuoted(input string, vars map[string][]string) (string, int) {
 		return input, len(input)
 	}
 
-	output := executeRecipe("sh", nil, input[:j], false, false, true)
+    // TODO: handle errors
+	output, _ := subprocess("sh", nil, input[:j], false, false, true)
 	return output, (j + 1)
 }
 

graph.go

@@ -0,0 +1,206 @@
+
+package main
+
+import (
+    "fmt"
+    "io"
+    "os"
+    "sync"
+    "time"
+)
+
+// A dependency graph
+type graph struct {
+    root *node             // the intial target's node
+    nodes map[string]*node // map targets to their nodes
+}
+
+// An edge in the graph.
+type edge struct {
+    v *node          // node this edge directs to
+    stem string      // stem matched for meta-rule applications
+    matches []string // regular expression matches
+    r *rule
+}
+
+
+// Current status of a node in the build.
+type nodeStatus int
+
+const (
+    nodeStatusReady nodeStatus = iota
+    nodeStatusStarted
+    nodeStatusDone
+    nodeStatusFailed
+)
+
+
+// A node in the dependency graph
+type node struct {
+    r    *rule        // rule to be applied
+    name string       // target name
+    prog string       // custom program to compare times
+    t time.Time       // file modification time
+    exists bool       // does a non-virtual target exist
+    prereqs []*edge   // prerequisite rules
+    status nodeStatus // current state of the node in the build
+    mutex sync.Mutex  // exclusivity for the status variable
+    listeners []chan nodeStatus // channels to notify of completion
+}
+
+// Create a new node
+func (g *graph) newnode(name string) *node {
+    u := &node{name: name}
+    info, err := os.Stat(name)
+    if err == nil {
+        u.t = info.ModTime()
+        u.exists = true
+    } else {
+        _, ok := err.(*os.PathError)
+        if ok {
+            u.exists = false
+        } else {
+            mkError(err.Error())
+        }
+    }
+    g.nodes[name] = u
+    return u
+}
+
+// Print a graph in graphviz format.
+func (g *graph) visualize(w io.Writer) {
+    fmt.Fprintln(w, "digraph mk {")
+    for t, u := range g.nodes {
+        for i := range u.prereqs {
+            if u.prereqs[i].v != nil {
+                fmt.Fprintf(w, "    \"%s\" -> \"%s\";\n", t, u.prereqs[i].v.name)
+            }
+        }
+    }
+    fmt.Fprintln(w, "}")
+}
+
+
+// Create a new arc.
+func (u *node) newedge(v *node, r *rule) *edge {
+    e := &edge{v: v, r: r}
+    u.prereqs = append(u.prereqs, e)
+    return e
+}
+
+
+// Create a dependency graph for the given target.
+func buildgraph(rs *ruleSet, target string) *graph {
+    g := &graph{nil, make(map[string]*node)}
+
+    // keep track of how many times each rule is visited, to avoid cycles.
+    rulecnt := make([]int, len(rs.rules))
+    g.root = applyrules(rs, g, target, rulecnt)
+
+    return g
+}
+
+// Recursively match the given target to a rule in the rule set to construct the
+// full graph.
+func applyrules(rs *ruleSet, g *graph, target string, rulecnt []int) *node {
+    u, ok := g.nodes[target]
+    if ok {
+        return u
+    }
+    u = g.newnode(target)
+
+    // does the target match a concrete rule?
+
+    ks, ok := rs.targetrules[target]
+    if ok {
+        for ki := range ks {
+            k := ks[ki]
+            if rulecnt[k] > max_rule_cnt {
+                continue
+            }
+
+            r := &rs.rules[k]
+
+            // skip meta-rules
+            if r.ismeta {
+                continue
+            }
+
+            // skip rules that have no effect
+            if r.recipe == "" && len(r.prereqs) == 0 {
+                continue
+            }
+
+            rulecnt[k] += 1
+            if len(r.prereqs) == 0 {
+                u.newedge(nil, r)
+            } else {
+                for i := range r.prereqs {
+                    u.newedge(applyrules(rs, g, r.prereqs[i], rulecnt), r)
+                }
+            }
+            rulecnt[k] -= 1
+        }
+    }
+
+    // find applicable metarules
+    for k := range rs.rules {
+        if rulecnt[k] > max_rule_cnt {
+            continue
+        }
+
+        r := &rs.rules[k]
+
+        if !r.ismeta {
+            continue
+        }
+
+        // skip rules that have no effect
+        if r.recipe == "" && len(r.prereqs) == 0 {
+            continue
+        }
+
+        for j := range r.targets {
+            mat := r.targets[j].match(target)
+            if mat == nil {
+                continue
+            }
+
+            var stem string
+            var matches []string
+
+            if r.attributes.regex {
+                matches = mat
+            } else {
+                stem = mat[1]
+            }
+
+            rulecnt[k] += 1
+            if len(r.prereqs) == 0 {
+                e := u.newedge(nil, r)
+                e.stem = stem
+                e.matches = matches
+            } else {
+                for i := range r.prereqs {
+                    var prereq string
+                    if r.attributes.regex {
+                        // TODO: write substituteRegexpRefs and use that here
+                        prereq = r.prereqs[i]
+                    } else {
+                        prereq = expandSuffixes(r.prereqs[i], stem)
+                    }
+
+                    e := u.newedge(applyrules(rs, g, prereq, rulecnt), r)
+                    e.stem = stem
+                    e.matches = matches
+                }
+            }
+            rulecnt[k] -= 1
+        }
+    }
+
+    return u
+}
+
+
+

mk.go

@@ -8,25 +8,175 @@ import (
 )
 
 
+// True if messages should be printed without fancy colors.
+var nocolor bool = false
+
+// True if we are no actualyl executing any recipes or updating any timestamps.
+var dryrun bool = false
+
 // The maximum number of times an rule may be applied.
 const max_rule_cnt = 3
 
+// Ansi color codes.
+const (
+    colorDefault string = "\033[0m"
+    colorBlack   string = "\033[30m"
+    colorRed     string = "\033[31m"
+    colorGreen   string = "\033[32m"
+    colorYellow  string = "\033[33m"
+    colorBlue    string = "\033[34m"
+    colorMagenta string = "\033[35m"
+)
+
 
 func mk(rs *ruleSet, target string, dryrun bool) {
-
+    g := buildgraph(rs, target)
-    // Build a graph
+    //g.visualize(os.Stdout)
-
+    mkNode(g, g.root)
-
-
-    // 1. Introduce special variables into the ruleSet
 }
 
+// Build a target in the graph.
+//
+// This selects an appropriate rule (edge) and builds all prerequisites
+// concurrently.
+//
+// TODO: control the number of concurrent rules being built. This would involve
+// some sort of global counter protected by a mutex, so we wait our turn to
+// execute our rule.
+//
+func mkNode(g *graph, u *node) {
+    // try to claim on this node
+    u.mutex.Lock()
+    if u.status != nodeStatusReady {
+        u.mutex.Unlock()
+        return
+    } else {
+        u.status = nodeStatusStarted
+    }
+    u.mutex.Unlock()
+
+    // when finished, notify the listeners
+    finalstatus := nodeStatusDone
+    defer func () {
+        u.mutex.Lock()
+        u.status = finalstatus
+        u.mutex.Unlock()
+        for i := range u.listeners {
+            u.listeners[i] <- u.status
+        }
+    }()
+
+    // there's no fucking rules, dude
+    if len(u.prereqs) == 0 {
+        if !u.r.attributes.virtual && !u.exists {
+            wd, _ := os.Getwd()
+            mkError(fmt.Sprintf("don't know how to make %s in %s", u.name, wd))
+        }
+        return
+    }
+
+    // there should otherwise be exactly one edge with an associated rule
+    prereqs := make([]*node, 0)
+    var e *edge = nil
+    for i := range u.prereqs {
+        if u.prereqs[i].r != nil {
+            e = u.prereqs[i]
+        }
+        if u.prereqs[i].v != nil {
+            prereqs = append(prereqs, u.prereqs[i].v)
+        }
+    }
+
+    // this should have been caught during graph building
+    if e == nil {
+        wd, _ := os.Getwd()
+        mkError(fmt.Sprintf("don't know how to make %s in %s", u.name, wd))
+    }
+
+    prereqstat := make(chan nodeStatus)
+    pending := 0
+
+    // build prereqs that need building
+    e.r.mutex.Lock()
+    for i := range prereqs {
+        prereqs[i].mutex.Lock()
+        // needs to be built?
+        if !prereqs[i].exists || e.r.attributes.virtual || (u.exists && u.t.Before(prereqs[i].t)) {
+            switch prereqs[i].status {
+            case nodeStatusReady:
+                go mkNode(g, prereqs[i])
+                fallthrough
+            case nodeStatusStarted:
+                prereqs[i].listeners = append(prereqs[i].listeners, prereqstat)
+                pending++
+            }
+        }
+        prereqs[i].mutex.Unlock()
+    }
+    e.r.mutex.Unlock()
+
+    // wait until all the prereqs are built
+    //fmt.Printf("%s: %d\n", u.name, pending)
+    for pending > 0 {
+        //for i := range prereqs {
+            //fmt.Println(prereqs[i].name)
+        //}
+
+        s := <-prereqstat
+        pending--
+        if s == nodeStatusFailed {
+            finalstatus = nodeStatusFailed
+        }
+    }
+
+    // execute the recipe, unless the prereqs failed
+    if finalstatus != nodeStatusFailed {
+        mkPrintMessage("mking " + u.name)
+        if !dorecipe(u.name, u, e) {
+            finalstatus = nodeStatusFailed
+        }
+    }
+
+    mkPrintSuccess("finished mking " + u.name)
+}
 
 func mkError(msg string) {
+    if !nocolor {
+        os.Stderr.WriteString(colorRed)
+    }
     fmt.Fprintf(os.Stderr, "mk: %s\n", msg)
+    if !nocolor {
+        os.Stderr.WriteString(colorDefault)
+    }
     os.Exit(1)
 }
 
+func mkPrintSuccess(msg string) {
+    if nocolor {
+        fmt.Println(msg)
+    } else {
+        fmt.Printf("%s%s%s\n", colorGreen, msg, colorDefault)
+    }
+}
+
+func mkPrintMessage(msg string) {
+    if nocolor {
+        fmt.Println(msg)
+    } else {
+        fmt.Printf("%s%s%s\n", colorBlue, msg, colorDefault)
+    }
+}
+
+func mkPrintRecipe(msg string) {
+    if !nocolor {
+        os.Stdout.WriteString(colorYellow)
+    }
+    printIndented(os.Stdout, msg)
+    if !nocolor {
+        os.Stdout.WriteString(colorDefault)
+    }
+}
+
 
 func main() {
     var mkfilepath string
@@ -51,6 +201,7 @@ func main() {
             for j := range rs.rules[i].targets {
                 targets = append(targets, rs.rules[i].targets[j].spat)
             }
+            break
         }
     }
 
@@ -59,11 +210,9 @@ func main() {
         return
     }
 
+    // TODO: For multiple targets, we should add a dummy rule that depends on
+    // all let mk handle executing each.
     for _, target := range targets {
-        //fmt.Printf("building: %q\n", target)
+        mk(rs, target, dryrun)
-        g := buildgraph(rs, target)
-        g.visualize(os.Stdout)
-        //mk(rs, target, dryrun)
     }
-
 }

parse.go

@@ -114,7 +114,11 @@ func parsePipeInclude(p *parser, t token) parserStateFun {
 			args[i-1] = p.tokenbuf[i].val
 		}
 
-		output := executeRecipe("sh", args, "", false, false, true)
+		output, success := subprocess("sh", args, "", false, false, true)
+        if !success {
+            p.basicErrorAtToken("subprocess include failed", t)
+        }
+
 		parseInto(output, fmt.Sprintf("%s:sh", p.name), p.rules)
 
 		p.clear()

recipe.go

@@ -1,3 +1,6 @@
+
+// Various function for dealing with recipes.
+
 package main
 
 import (
@@ -5,27 +8,111 @@ import (
 	"log"
 	"os"
 	"os/exec"
+    "bufio"
+    "fmt"
+    "strings"
 )
 
-// A monolithic function for executing recipes.
+
-func executeRecipe(program string,
+// Try to unindent a recipe, so that it begins an column 0. (This is mainly for
+// recipes in python, or other indentation-significant languages.)
+//func stripIndentation(s string) string {
+
+//}
+
+// Indent each line of a recipe.
+func printIndented(out io.Writer, s string) {
+    reader := bufio.NewReader(strings.NewReader(s))
+    for {
+        line, err := reader.ReadString('\n')
+        if len(line) > 0 {
+            io.WriteString(out, "    ")
+            io.WriteString(out, line)
+        }
+
+        if (err != nil) {
+            break
+        }
+    }
+}
+
+// Execute a recipe.
+func dorecipe(target string, u *node, e *edge) bool {
+    vars := make(map[string][]string)
+    vars["target"] = []string{target}
+    if e.r.ismeta {
+        if e.r.attributes.regex {
+            for i := range e.matches {
+                vars[fmt.Sprintf("stem%d", i)] = e.matches[i:i+1]
+            }
+        } else {
+            vars["stem"] = []string{e.stem}
+        }
+    }
+
+    // TODO: other variables to set
+    // alltargets
+    // newprereq
+
+    prereqs := make([]string, 0)
+    for i := range u.prereqs {
+        if u.prereqs[i].r == e.r && u.prereqs[i].v != nil {
+            prereqs = append(prereqs, u.prereqs[i].v.name)
+        }
+    }
+    vars["prereqs"] = prereqs
+
+    input := expandRecipeSigils(e.r.recipe, vars)
+    sh := "sh"
+    args := []string{}
+
+    if len(e.r.shell) > 0 {
+        sh = e.r.shell[0]
+        args = e.r.shell[1:]
+    }
+
+    if !e.r.attributes.quiet {
+        mkPrintRecipe(input)
+    }
+
+    if dryrun {
+        return true
+    }
+
+    _, success := subprocess(
+        sh,
+        args,
+        input,
+        true,
+        true,
+        false)
+
+
+    // TODO: update the timestamps of each target
+
+    return success
+}
+
+
+// A monolithic function for executing subprocesses
+func subprocess(program string,
 	args []string,
 	input string,
 	echo_out bool,
 	echo_err bool,
-	capture_out bool) string {
+	capture_out bool) (string, bool) {
 	cmd := exec.Command(program, args...)
 
 	if echo_out {
 		cmdout, err := cmd.StdoutPipe()
-		if err != nil {
+		if err == nil {
 			go io.Copy(os.Stdout, cmdout)
 		}
 	}
 
 	if echo_err {
-		cmderr, err := cmd.StdoutPipe()
+		cmderr, err := cmd.StderrPipe()
-		if err != nil {
+		if err == nil {
 			go io.Copy(os.Stderr, cmderr)
 		}
 	}
@@ -49,11 +136,16 @@ func executeRecipe(program string,
 	} else {
 		err = cmd.Run()
 	}
+    success := true
 
 	if err != nil {
-		// TODO: better error output
+        exiterr, ok := err.(*exec.ExitError)
-		log.Fatal("Recipe failed")
+        if ok {
+            success = exiterr.ProcessState.Success()
+        } else {
+            log.Fatal(err)
+        }
 	}
 
-	return output
+	return output, success
 }

ruleset.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"unicode/utf8"
     "regexp"
+    "sync"
 )
 
 type attribSet struct {
@@ -59,6 +60,7 @@ type rule struct {
 	recipe     string     // recipe source
 	command    []string   // command attribute
     ismeta     bool       // is this a meta rule
+    mutex      sync.Mutex // prevent the rule from being executed multiple times
 }