Work on graph build and metarule matching.

expand.go

@@ -0,0 +1,214 @@
+
+// String substitution and expansion.
+
+package main
+
+import (
+    "strings"
+    "unicode/utf8"
+)
+
+
+// Expand a word. This includes substituting variables and handling quotes.
+func expand(input string, vars map[string][]string, expandBackticks bool) string {
+	expanded := make([]byte, 0)
+	var i, j int
+	for i = 0; i < len(input); {
+		j = i + strings.IndexAny(input[i:], "\"'`$\\")
+
+		if j < 0 {
+			expanded = append(expanded, input[i:]...)
+			break
+		}
+
+		expanded = append(expanded, input[i:j]...)
+		c, w := utf8.DecodeRuneInString(input[j:])
+		i = j + w
+
+		var off int
+		var out string
+		switch c {
+		case '\\':
+			out, off = expandEscape(input[i:])
+
+		case '"':
+			out, off = expandDoubleQuoted(input[i:], vars, expandBackticks)
+
+		case '\'':
+			out, off = expandSingleQuoted(input[i:])
+
+		case '`':
+            if expandBackticks {
+                out, off = expandBackQuoted(input[i:], vars)
+            } else {
+                out = input
+                off = len(input)
+            }
+
+		case '$':
+            out, off = expandSigil(input[i:], vars)
+		}
+
+		expanded = append(expanded, out...)
+		i += off
+	}
+
+	return string(expanded)
+}
+
+// Expand following a '\\'
+func expandEscape(input string) (string, int) {
+	c, w := utf8.DecodeRuneInString(input)
+	return string(c), w
+}
+
+// Expand a double quoted string starting after a '\"'
+func expandDoubleQuoted(input string, vars map[string][]string, expandBackticks bool) (string, int) {
+	// find the first non-escaped "
+	j := 0
+	for {
+		j = strings.IndexAny(input[j:], "\"\\")
+		if j < 0 {
+			break
+		}
+
+		_, w := utf8.DecodeRuneInString(input[j:])
+		j += w
+
+		c, w := utf8.DecodeRuneInString(input[j:])
+		j += w
+
+		if c == '"' {
+			return expand(input[:j], vars, expandBackticks), (j + w)
+		}
+
+		if c == '\\' {
+			if j+w < len(input) {
+				j += w
+				_, w := utf8.DecodeRuneInString(input[j:])
+				j += w
+			} else {
+				break
+			}
+		}
+	}
+
+	return input, len(input)
+}
+
+// Expand a single quoted string starting after a '\''
+func expandSingleQuoted(input string) (string, int) {
+	j := strings.Index(input, "'")
+	if j < 0 {
+		return input, len(input)
+	}
+	return input[:j], (j + 1)
+}
+
+// Expand something starting with at '$'.
+func expandSigil(input string, vars map[string][]string) (string, int) {
+    c, w := utf8.DecodeRuneInString(input)
+    var offset int
+    var varname string
+    if c == '{' {
+        j := strings.IndexRune(input[w:], '}')
+        if j < 0 {
+            return input, len(input)
+        }
+
+        varname = input[w:j]
+        offset = j + 1
+    } else {
+        // try to match a variable name
+        i := w
+        j := i
+        for j < len(input) {
+            c, w = utf8.DecodeRuneInString(input)
+            if !(isalpha(c) || c == '_' || (j > i && isdigit(c))) {
+                break
+            }
+            j += w
+        }
+
+        if j > i {
+            varname = input[i:j]
+        } else {
+            return input, len(input)
+        }
+    }
+
+    if isValidVarName(varname) {
+        varvals, ok := vars[varname]
+        if ok {
+            return strings.Join(varvals, " "), offset
+        }
+    }
+
+    return input, len(input)
+}
+
+
+// Find and expand all sigils.
+func expandSigils(input string, vars map[string][]string) string {
+    expanded := make([]byte, 0)
+    for i := 0; i < len(input); {
+        j := strings.IndexRune(input[i:], '$')
+        if j < 0 {
+            expanded = append(expanded, input[i:]...)
+            break
+        }
+
+        ex, k := expandSigil(input[j+1:], vars)
+        expanded = append(expanded, ex...)
+        i = k
+    }
+
+    return string(expanded)
+}
+
+
+
+// Expand all unescaped '%' characters.
+func expandSuffixes(input string, stem string) string {
+    expanded := make([]byte, 0)
+    for i := 0; i < len(input); {
+        j := strings.IndexAny(input[i:], "\\%")
+        if j < 0 {
+            expanded = append(expanded, input[i:]...)
+            break
+        }
+
+        c, w := utf8.DecodeRuneInString(input[j:])
+        if c == '%' {
+            expanded = append(expanded, stem...) 
+            i += w
+        } else {
+            j += w
+            c, w := utf8.DecodeRuneInString(input[j:])
+            if c == '%' {
+                expanded = append(expanded, '%')
+                i = j + w
+            }
+        }
+    }
+
+    return string(expanded)
+}
+
+
+// TODO: expand RegexpRefs
+
+
+// Expand a backtick quoted string, by executing the contents.
+func expandBackQuoted(input string, vars map[string][]string) (string, int) {
+    // TODO: expand sigils?
+	j := strings.Index(input, "`")
+	if j < 0 {
+		return input, len(input)
+	}
+
+	output := executeRecipe("sh", nil, input[:j], false, false, true)
+	return output, (j + 1)
+}
+
+

mk.go

@@ -1,25 +1,69 @@
 package main
 
 import (
+    "flag"
     "fmt"
     "io/ioutil"
     "os"
 )
 
-func main() {
-    input, _ := ioutil.ReadAll(os.Stdin)
 
-	// TEST LEXING
+// The maximum number of times an rule may be applied.
-	//_, tokens := lex(string(input))
+const max_rule_cnt = 3
-	//for t := range tokens {
-	//fmt.Printf("%s %s\n", t.typ, t.val)
-	//}
 
-	// TEST PARSING
-    rs := parse(string(input), "<stdin>")
-    fmt.Println(rs)
 
-	// TEST STRING EXPANSION
+func mk(rs *ruleSet, target string, dryrun bool) {
-	//rules := &ruleSet{make(map[string][]string), make([]rule, 0)}
+
-	//println(rules.expand("\"This is a quote: \\\"\""))
+    // Build a graph
+
+
+
+    // 1. Introduce special variables into the ruleSet
+}
+
+
+func mkError(msg string) {
+    fmt.Fprintf(os.Stderr, "mk: %s\n", msg)
+    os.Exit(1)
+}
+
+
+func main() {
+    var mkfilepath string
+    var dryrun bool
+    flag.StringVar(&mkfilepath, "f", "mkfile", "use the given file as mkfile")
+    flag.BoolVar(&dryrun, "n", false, "print commands without actually executing")
+    flag.Parse()
+
+    mkfile, err := os.Open(mkfilepath)
+    if err != nil {
+        mkError("no mkfile found")
+    }
+    input, _ := ioutil.ReadAll(mkfile)
+    mkfile.Close()
+
+    rs := parse(string(input), mkfilepath)
+    targets := flag.Args()
+
+    // build the first non-meta rule in the makefile, if none are given explicitly
+    for i := range rs.rules {
+        if !rs.rules[i].ismeta {
+            for j := range rs.rules[i].targets {
+                targets = append(targets, rs.rules[i].targets[j].spat)
+            }
+        }
+    }
+
+    if len(targets) == 0 {
+        fmt.Println("mk: nothing to mk")
+        return
+    }
+
+    for _, target := range targets {
+        //fmt.Printf("building: %q\n", target)
+        g := buildgraph(rs, target)
+        g.visualize(os.Stdout)
+        //mk(rs, target, dryrun)
+    }
+
 }

parse.go

@@ -6,6 +6,8 @@ package main
 import (
 	"fmt"
 	"os"
+    "regexp"
+    "strings"
 )
 
 type parser struct {
@@ -50,7 +52,9 @@ type parserStateFun func(*parser, token) parserStateFun
 
 // Parse a mkfile, returning a new ruleSet.
 func parse(input string, name string) *ruleSet {
-	rules := &ruleSet{make(map[string][]string), make([]rule, 0)}
+	rules := &ruleSet{make(map[string][]string),
+                      make([]rule, 0),
+                      make(map[string][]int)}
 	parseInto(input, name, rules)
 	return rules
 }
@@ -155,14 +159,12 @@ func parseRedirInclude(p *parser, t token) parserStateFun {
 
 // Encountered a bare string at the beginning of the line.
 func parseAssignmentOrTarget(p *parser, t token) parserStateFun {
-	fmt.Println("assignment or target")
 	p.push(t)
 	return parseEqualsOrTarget
 }
 
 // Consumed one bare string ot the beginning of the line.
 func parseEqualsOrTarget(p *parser, t token) parserStateFun {
-	fmt.Println("equals or target")
 	switch t.typ {
 	case tokenAssign:
 		return parseAssignment
@@ -220,7 +222,6 @@ func parseTargets(p *parser, t token) parserStateFun {
 
 // Consumed one or more strings followed by a first ':'.
 func parseAttributesOrPrereqs(p *parser, t token) parserStateFun {
-	fmt.Println("attributes or prereqs")
 	switch t.typ {
 	case tokenNewline:
 		return parseRecipe
@@ -239,7 +240,6 @@ func parseAttributesOrPrereqs(p *parser, t token) parserStateFun {
 
 // Targets and attributes and the second ':' have been consumed.
 func parsePrereqs(p *parser, t token) parserStateFun {
-	fmt.Println("prereqs")
 	switch t.typ {
 	case tokenNewline:
 		return parseRecipe
@@ -256,8 +256,6 @@ func parsePrereqs(p *parser, t token) parserStateFun {
 
 // An entire rule has been consumed.
 func parseRecipe(p *parser, t token) parserStateFun {
-	fmt.Println("recipe")
-
 	// Assemble the rule!
 	r := rule{}
 
@@ -269,39 +267,73 @@ func parseRecipe(p *parser, t token) parserStateFun {
 	for ; j < len(p.tokenbuf) && p.tokenbuf[j].typ != tokenColon; j++ {
 	}
 
-	// targets
-	r.targets = make([]string, i)
-	for k := 0; k < i; k++ {
-		r.targets[k] = p.rules.expand(p.tokenbuf[k].val, true)
-	}
-
 	// rule has attributes
-    // TODO: should we be expanding the attribute strings?
 	if j < len(p.tokenbuf) {
 		attribs := make([]string, j-i-1)
 		for k := i + 1; k < j; k++ {
-			attribs[k-i-1] = p.tokenbuf[k].val
+			attribs[k-i-1] = expand(p.tokenbuf[k].val, p.rules.vars, true)
 		}
 		err := r.parseAttribs(attribs)
 		if err != nil {
 			msg := fmt.Sprintf("while reading a rule's attributes expected an attribute but found \"%c\".", err.found)
 			p.basicErrorAtToken(msg, p.tokenbuf[i+1])
 		}
+
+        if r.attributes.regex {
+            r.ismeta = true
+        }
 	} else {
 		j = i
 	}
 
+	// targets
+	r.targets = make([]pattern, i)
+	for k := 0; k < i; k++ {
+        targetstr := expand(p.tokenbuf[k].val, p.rules.vars, true)
+        r.targets[k].spat = targetstr
+
+        if r.attributes.regex {
+            rpat, err := regexp.Compile(targetstr)
+            if err != nil {
+                msg := fmt.Sprintf("invalid regular expression: %q", err)
+                p.basicErrorAtToken(msg, p.tokenbuf[k])
+            }
+            r.targets[k].rpat = rpat
+        } else {
+            idx := strings.IndexRune(targetstr, '%')
+            if idx >= 0 {
+                var left, right string
+                if idx > 0 {
+                    left = regexp.QuoteMeta(targetstr[:idx])
+                }
+                if idx < len(targetstr) - 1 {
+                    right = regexp.QuoteMeta(targetstr[idx+1:])
+                }
+
+                patstr := fmt.Sprintf("^%s(.*)%s$", left, right)
+                rpat, err := regexp.Compile(patstr)
+                if err != nil {
+                    msg := fmt.Sprintf("error compiling suffix rule. This is a bug.", err)
+                    p.basicErrorAtToken(msg, p.tokenbuf[k])
+                }
+                r.targets[k].rpat = rpat
+                r.targets[k].issuffix = true
+                r.ismeta = true
+            }
+        }
+	}
+
 	// prereqs
 	r.prereqs = make([]string, len(p.tokenbuf)-j-1)
 	for k := j + 1; k < len(p.tokenbuf); k++ {
-		r.prereqs[k-j-1] = p.rules.expand(p.tokenbuf[k].val, true)
+        r.prereqs[k-j-1] = expand(p.tokenbuf[k].val, p.rules.vars, true)
 	}
 
 	if t.typ == tokenRecipe {
-		r.recipe = p.rules.expand(t.val, false)
+		r.recipe = t.val
 	}
 
-	p.rules.push(r)
+	p.rules.add(r)
 	p.clear()
 
 	// the current token doesn't belong to this rule

ruleset.go

@@ -6,8 +6,8 @@ package main
 
 import (
 	"fmt"
-	"strings"
 	"unicode/utf8"
+    "regexp"
 )
 
 type attribSet struct {
@@ -26,15 +26,51 @@ type attribError struct {
 	found rune
 }
 
+
+// target and rereq patterns
+type pattern struct {
+    issuffix bool       // is a suffix '%' rule, so we should define $stem.
+    spat string         // simple string pattern
+    rpat *regexp.Regexp // non-nil if this is a regexp pattern
+}
+
+
+// Match a pattern, returning an array of submatches, or nil if it doesn'm
+// match.
+func (p *pattern) match(target string) []string {
+    if p.rpat != nil {
+        return p.rpat.FindStringSubmatch(target)
+    }
+
+    if target == p.spat {
+        return make([]string, 0)
+    }
+
+    return nil
+}
+
+
+// A single rule.
 type rule struct {
-	targets    []string  // non-empty array of targets
+	targets    []pattern // non-empty array of targets
 	attributes attribSet // rule attributes
 	prereqs    []string  // possibly empty prerequesites
 	shell      []string  // command used to execute the recipe
 	recipe     string    // recipe source
 	command    []string  // command attribute
+    ismeta     bool      // is this a meta rule
 }
 
+
+// A set of rules.
+type ruleSet struct {
+	vars  map[string][]string
+	rules []rule
+    // map a target to an array of indexes into rules
+    targetrules map[string][]int
+}
+
+
 // Read attributes for an array of strings, updating the rule.
 func (r *rule) parseAttribs(inputs []string) *attribError {
 	for i := 0; i < len(inputs); i++ {
@@ -84,164 +120,18 @@ func (r *rule) parseAttribs(inputs []string) *attribError {
 	return nil
 }
 
-type ruleSet struct {
-	vars  map[string][]string
-	rules []rule
-}
-
 // Add a rule to the rule set.
-func (rs *ruleSet) push(r rule) {
+func (rs *ruleSet) add(r rule) {
 	rs.rules = append(rs.rules, r)
-}
+    k := len(rs.rules) - 1
-
+    for i := range r.targets {
-// Expand a word. This includes substituting variables and handling quotes.
+        if r.targets[i].rpat == nil {
-func (rs *ruleSet) expand(input string, expandBackticks bool) string {
+            rs.targetrules[r.targets[i].spat] =
-	expanded := make([]byte, 0)
+                append(rs.targetrules[r.targets[i].spat], k)
-	var i, j int
-	for i = 0; i < len(input); {
-		j = i + strings.IndexAny(input[i:], "\"'`$\\")
-
-		if j < 0 {
-			expanded = append(expanded, []byte(input[i:])...)
-			break
-		}
-
-		expanded = append(expanded, []byte(input[i:j])...)
-		c, w := utf8.DecodeRuneInString(input[j:])
-		i = j + w
-
-		var off int
-		var out string
-		switch c {
-		case '\\':
-			out, off = rs.expandEscape(input[i:])
-
-		case '"':
-			out, off = rs.expandDoubleQuoted(input[i:], expandBackticks)
-
-		case '\'':
-			out, off = rs.expandSingleQuoted(input[i:])
-
-		case '`':
-            if expandBackticks {
-                out, off = rs.expandBackQuoted(input[i:])
-            } else {
-                out = input
-                off = len(input)
-            }
-
-		case '$':
-            out, off = rs.expandSigil(input[i:])
-		}
-
-		expanded = append(expanded, []byte(out)...)
-		i += off
-	}
-
-	return string(expanded)
-}
-
-// Expand following a '\\'
-func (rs *ruleSet) expandEscape(input string) (string, int) {
-	c, w := utf8.DecodeRuneInString(input)
-	return string(c), w
-}
-
-// Expand a double quoted string starting after a '\"'
-func (rs *ruleSet) expandDoubleQuoted(input string, expandBackticks bool) (string, int) {
-	// find the first non-escaped "
-	j := 0
-	for {
-		j = strings.IndexAny(input[j:], "\"\\")
-		if j < 0 {
-			break
-		}
-
-		_, w := utf8.DecodeRuneInString(input[j:])
-		j += w
-
-		c, w := utf8.DecodeRuneInString(input[j:])
-		j += w
-
-		if c == '"' {
-			return rs.expand(input[:j], expandBackticks), (j + w)
-		}
-
-		if c == '\\' {
-			if j+w < len(input) {
-				j += w
-				_, w := utf8.DecodeRuneInString(input[j:])
-				j += w
-			} else {
-				break
         }
     }
 }
 
-	return input, len(input)
-}
-
-// Expand a single quoted string starting after a '\''
-func (rs *ruleSet) expandSingleQuoted(input string) (string, int) {
-	j := strings.Index(input, "'")
-	if j < 0 {
-		return input, len(input)
-	}
-	return input[:j], (j + 1)
-}
-
-// Expand something starting with at '$'.
-func (rs *ruleSet) expandSigil(input string) (string, int) {
-    c, w := utf8.DecodeRuneInString(input)
-    var offset int
-    var varname string
-    if c == '{' {
-        j := strings.Index(input[w:], "}")
-        if j < 0 {
-            return input, len(input)
-        }
-
-        varname = input[w:j]
-        offset = j + 1
-    } else {
-        // try to match a variable name
-        i := w
-        j := i
-        for j < len(input) {
-            c, w = utf8.DecodeRuneInString(input)
-            if !(isalpha(c) || c == '_' || (j > i && isdigit(c))) {
-                break
-            }
-            j += w
-        }
-
-        if j > i {
-            varname = input[i:j]
-        } else {
-            return input, len(input)
-        }
-    }
-
-    if isValidVarName(varname) {
-        varvals, ok := rs.vars[varname]
-        if ok {
-            return strings.Join(varvals, " "), offset
-        }
-    }
-
-    return input, len(input)
-}
-
-// Expand a backtick quoted string, by executing the contents.
-func (rs *ruleSet) expandBackQuoted(input string) (string, int) {
-	j := strings.Index(input, "`")
-	if j < 0 {
-		return input, len(input)
-	}
-
-	output := executeRecipe("sh", nil, input[:j], false, false, true)
-	return output, (j + 1)
-}
 
 func isValidVarName(v string) bool {
 	for i := 0; i < len(v); {
@@ -286,7 +176,7 @@ func (rs *ruleSet) executeAssignment(ts []token) *assignmentError {
 	// expanded variables
 	vals := make([]string, len(ts)-1)
 	for i := 0; i < len(vals); i++ {
-		vals[i] = rs.expand(ts[i+1].val, true)
+		vals[i] = expand(ts[i+1].val, rs.vars, true)
 	}
 
 	rs.vars[assignee] = vals