From 207e0523721921d9cb4c0964fb463ff2f3976358 Mon Sep 17 00:00:00 2001 From: Daniel Jones Date: Sat, 2 Mar 2013 00:34:31 -0800 Subject: [PATCH] Work on graph build and metarule matching. --- expand.go | 214 +++++++++++++++++++++++++++++++++++++++++++++++++++++ mk.go | 76 +++++++++++++++---- parse.go | 68 ++++++++++++----- ruleset.go | 200 +++++++++++-------------------------------------- 4 files changed, 369 insertions(+), 189 deletions(-) create mode 100644 expand.go diff --git a/expand.go b/expand.go new file mode 100644 index 0000000..eb06dc5 --- /dev/null +++ b/expand.go @@ -0,0 +1,214 @@ + +// String substitution and expansion. + +package main + +import ( + "strings" + "unicode/utf8" +) + + +// Expand a word. This includes substituting variables and handling quotes. +func expand(input string, vars map[string][]string, expandBackticks bool) string { + expanded := make([]byte, 0) + var i, j int + for i = 0; i < len(input); { + j = i + strings.IndexAny(input[i:], "\"'`$\\") + + if j < 0 { + expanded = append(expanded, input[i:]...) + break + } + + expanded = append(expanded, input[i:j]...) + c, w := utf8.DecodeRuneInString(input[j:]) + i = j + w + + var off int + var out string + switch c { + case '\\': + out, off = expandEscape(input[i:]) + + case '"': + out, off = expandDoubleQuoted(input[i:], vars, expandBackticks) + + case '\'': + out, off = expandSingleQuoted(input[i:]) + + case '`': + if expandBackticks { + out, off = expandBackQuoted(input[i:], vars) + } else { + out = input + off = len(input) + } + + case '$': + out, off = expandSigil(input[i:], vars) + } + + expanded = append(expanded, out...) + i += off + } + + return string(expanded) +} + +// Expand following a '\\' +func expandEscape(input string) (string, int) { + c, w := utf8.DecodeRuneInString(input) + return string(c), w +} + +// Expand a double quoted string starting after a '\"' +func expandDoubleQuoted(input string, vars map[string][]string, expandBackticks bool) (string, int) { + // find the first non-escaped " + j := 0 + for { + j = strings.IndexAny(input[j:], "\"\\") + if j < 0 { + break + } + + _, w := utf8.DecodeRuneInString(input[j:]) + j += w + + c, w := utf8.DecodeRuneInString(input[j:]) + j += w + + if c == '"' { + return expand(input[:j], vars, expandBackticks), (j + w) + } + + if c == '\\' { + if j+w < len(input) { + j += w + _, w := utf8.DecodeRuneInString(input[j:]) + j += w + } else { + break + } + } + } + + return input, len(input) +} + +// Expand a single quoted string starting after a '\'' +func expandSingleQuoted(input string) (string, int) { + j := strings.Index(input, "'") + if j < 0 { + return input, len(input) + } + return input[:j], (j + 1) +} + +// Expand something starting with at '$'. +func expandSigil(input string, vars map[string][]string) (string, int) { + c, w := utf8.DecodeRuneInString(input) + var offset int + var varname string + if c == '{' { + j := strings.IndexRune(input[w:], '}') + if j < 0 { + return input, len(input) + } + + varname = input[w:j] + offset = j + 1 + } else { + // try to match a variable name + i := w + j := i + for j < len(input) { + c, w = utf8.DecodeRuneInString(input) + if !(isalpha(c) || c == '_' || (j > i && isdigit(c))) { + break + } + j += w + } + + if j > i { + varname = input[i:j] + } else { + return input, len(input) + } + } + + if isValidVarName(varname) { + varvals, ok := vars[varname] + if ok { + return strings.Join(varvals, " "), offset + } + } + + return input, len(input) +} + + +// Find and expand all sigils. +func expandSigils(input string, vars map[string][]string) string { + expanded := make([]byte, 0) + for i := 0; i < len(input); { + j := strings.IndexRune(input[i:], '$') + if j < 0 { + expanded = append(expanded, input[i:]...) + break + } + + ex, k := expandSigil(input[j+1:], vars) + expanded = append(expanded, ex...) + i = k + } + + return string(expanded) +} + + + +// Expand all unescaped '%' characters. +func expandSuffixes(input string, stem string) string { + expanded := make([]byte, 0) + for i := 0; i < len(input); { + j := strings.IndexAny(input[i:], "\\%") + if j < 0 { + expanded = append(expanded, input[i:]...) + break + } + + c, w := utf8.DecodeRuneInString(input[j:]) + if c == '%' { + expanded = append(expanded, stem...) + i += w + } else { + j += w + c, w := utf8.DecodeRuneInString(input[j:]) + if c == '%' { + expanded = append(expanded, '%') + i = j + w + } + } + } + + return string(expanded) +} + + +// TODO: expand RegexpRefs + + +// Expand a backtick quoted string, by executing the contents. +func expandBackQuoted(input string, vars map[string][]string) (string, int) { + // TODO: expand sigils? + j := strings.Index(input, "`") + if j < 0 { + return input, len(input) + } + + output := executeRecipe("sh", nil, input[:j], false, false, true) + return output, (j + 1) +} + + diff --git a/mk.go b/mk.go index 0ea7449..0d32770 100644 --- a/mk.go +++ b/mk.go @@ -1,25 +1,69 @@ package main import ( -"fmt" -"io/ioutil" -"os" + "flag" + "fmt" + "io/ioutil" + "os" ) -func main() { - input, _ := ioutil.ReadAll(os.Stdin) - // TEST LEXING - //_, tokens := lex(string(input)) - //for t := range tokens { - //fmt.Printf("%s %s\n", t.typ, t.val) - //} +// The maximum number of times an rule may be applied. +const max_rule_cnt = 3 - // TEST PARSING - rs := parse(string(input), "") - fmt.Println(rs) - // TEST STRING EXPANSION - //rules := &ruleSet{make(map[string][]string), make([]rule, 0)} - //println(rules.expand("\"This is a quote: \\\"\"")) +func mk(rs *ruleSet, target string, dryrun bool) { + + // Build a graph + + + + // 1. Introduce special variables into the ruleSet +} + + +func mkError(msg string) { + fmt.Fprintf(os.Stderr, "mk: %s\n", msg) + os.Exit(1) +} + + +func main() { + var mkfilepath string + var dryrun bool + flag.StringVar(&mkfilepath, "f", "mkfile", "use the given file as mkfile") + flag.BoolVar(&dryrun, "n", false, "print commands without actually executing") + flag.Parse() + + mkfile, err := os.Open(mkfilepath) + if err != nil { + mkError("no mkfile found") + } + input, _ := ioutil.ReadAll(mkfile) + mkfile.Close() + + rs := parse(string(input), mkfilepath) + targets := flag.Args() + + // build the first non-meta rule in the makefile, if none are given explicitly + for i := range rs.rules { + if !rs.rules[i].ismeta { + for j := range rs.rules[i].targets { + targets = append(targets, rs.rules[i].targets[j].spat) + } + } + } + + if len(targets) == 0 { + fmt.Println("mk: nothing to mk") + return + } + + for _, target := range targets { + //fmt.Printf("building: %q\n", target) + g := buildgraph(rs, target) + g.visualize(os.Stdout) + //mk(rs, target, dryrun) + } + } diff --git a/parse.go b/parse.go index 8298b0b..16f336d 100644 --- a/parse.go +++ b/parse.go @@ -6,6 +6,8 @@ package main import ( "fmt" "os" + "regexp" + "strings" ) type parser struct { @@ -50,7 +52,9 @@ type parserStateFun func(*parser, token) parserStateFun // Parse a mkfile, returning a new ruleSet. func parse(input string, name string) *ruleSet { - rules := &ruleSet{make(map[string][]string), make([]rule, 0)} + rules := &ruleSet{make(map[string][]string), + make([]rule, 0), + make(map[string][]int)} parseInto(input, name, rules) return rules } @@ -155,14 +159,12 @@ func parseRedirInclude(p *parser, t token) parserStateFun { // Encountered a bare string at the beginning of the line. func parseAssignmentOrTarget(p *parser, t token) parserStateFun { - fmt.Println("assignment or target") p.push(t) return parseEqualsOrTarget } // Consumed one bare string ot the beginning of the line. func parseEqualsOrTarget(p *parser, t token) parserStateFun { - fmt.Println("equals or target") switch t.typ { case tokenAssign: return parseAssignment @@ -220,7 +222,6 @@ func parseTargets(p *parser, t token) parserStateFun { // Consumed one or more strings followed by a first ':'. func parseAttributesOrPrereqs(p *parser, t token) parserStateFun { - fmt.Println("attributes or prereqs") switch t.typ { case tokenNewline: return parseRecipe @@ -239,7 +240,6 @@ func parseAttributesOrPrereqs(p *parser, t token) parserStateFun { // Targets and attributes and the second ':' have been consumed. func parsePrereqs(p *parser, t token) parserStateFun { - fmt.Println("prereqs") switch t.typ { case tokenNewline: return parseRecipe @@ -256,8 +256,6 @@ func parsePrereqs(p *parser, t token) parserStateFun { // An entire rule has been consumed. func parseRecipe(p *parser, t token) parserStateFun { - fmt.Println("recipe") - // Assemble the rule! r := rule{} @@ -269,39 +267,73 @@ func parseRecipe(p *parser, t token) parserStateFun { for ; j < len(p.tokenbuf) && p.tokenbuf[j].typ != tokenColon; j++ { } - // targets - r.targets = make([]string, i) - for k := 0; k < i; k++ { - r.targets[k] = p.rules.expand(p.tokenbuf[k].val, true) - } - // rule has attributes - // TODO: should we be expanding the attribute strings? if j < len(p.tokenbuf) { attribs := make([]string, j-i-1) for k := i + 1; k < j; k++ { - attribs[k-i-1] = p.tokenbuf[k].val + attribs[k-i-1] = expand(p.tokenbuf[k].val, p.rules.vars, true) } err := r.parseAttribs(attribs) if err != nil { msg := fmt.Sprintf("while reading a rule's attributes expected an attribute but found \"%c\".", err.found) p.basicErrorAtToken(msg, p.tokenbuf[i+1]) } + + if r.attributes.regex { + r.ismeta = true + } } else { j = i } + // targets + r.targets = make([]pattern, i) + for k := 0; k < i; k++ { + targetstr := expand(p.tokenbuf[k].val, p.rules.vars, true) + r.targets[k].spat = targetstr + + if r.attributes.regex { + rpat, err := regexp.Compile(targetstr) + if err != nil { + msg := fmt.Sprintf("invalid regular expression: %q", err) + p.basicErrorAtToken(msg, p.tokenbuf[k]) + } + r.targets[k].rpat = rpat + } else { + idx := strings.IndexRune(targetstr, '%') + if idx >= 0 { + var left, right string + if idx > 0 { + left = regexp.QuoteMeta(targetstr[:idx]) + } + if idx < len(targetstr) - 1 { + right = regexp.QuoteMeta(targetstr[idx+1:]) + } + + patstr := fmt.Sprintf("^%s(.*)%s$", left, right) + rpat, err := regexp.Compile(patstr) + if err != nil { + msg := fmt.Sprintf("error compiling suffix rule. This is a bug.", err) + p.basicErrorAtToken(msg, p.tokenbuf[k]) + } + r.targets[k].rpat = rpat + r.targets[k].issuffix = true + r.ismeta = true + } + } + } + // prereqs r.prereqs = make([]string, len(p.tokenbuf)-j-1) for k := j + 1; k < len(p.tokenbuf); k++ { - r.prereqs[k-j-1] = p.rules.expand(p.tokenbuf[k].val, true) + r.prereqs[k-j-1] = expand(p.tokenbuf[k].val, p.rules.vars, true) } if t.typ == tokenRecipe { - r.recipe = p.rules.expand(t.val, false) + r.recipe = t.val } - p.rules.push(r) + p.rules.add(r) p.clear() // the current token doesn't belong to this rule diff --git a/ruleset.go b/ruleset.go index 74b735a..562cb98 100644 --- a/ruleset.go +++ b/ruleset.go @@ -6,8 +6,8 @@ package main import ( "fmt" - "strings" "unicode/utf8" + "regexp" ) type attribSet struct { @@ -26,15 +26,51 @@ type attribError struct { found rune } + +// target and rereq patterns +type pattern struct { + issuffix bool // is a suffix '%' rule, so we should define $stem. + spat string // simple string pattern + rpat *regexp.Regexp // non-nil if this is a regexp pattern +} + + +// Match a pattern, returning an array of submatches, or nil if it doesn'm +// match. +func (p *pattern) match(target string) []string { + if p.rpat != nil { + return p.rpat.FindStringSubmatch(target) + } + + if target == p.spat { + return make([]string, 0) + } + + return nil +} + + +// A single rule. type rule struct { - targets []string // non-empty array of targets + targets []pattern // non-empty array of targets attributes attribSet // rule attributes prereqs []string // possibly empty prerequesites shell []string // command used to execute the recipe recipe string // recipe source command []string // command attribute + ismeta bool // is this a meta rule } + +// A set of rules. +type ruleSet struct { + vars map[string][]string + rules []rule + // map a target to an array of indexes into rules + targetrules map[string][]int +} + + // Read attributes for an array of strings, updating the rule. func (r *rule) parseAttribs(inputs []string) *attribError { for i := 0; i < len(inputs); i++ { @@ -84,164 +120,18 @@ func (r *rule) parseAttribs(inputs []string) *attribError { return nil } -type ruleSet struct { - vars map[string][]string - rules []rule -} - // Add a rule to the rule set. -func (rs *ruleSet) push(r rule) { +func (rs *ruleSet) add(r rule) { rs.rules = append(rs.rules, r) -} - -// Expand a word. This includes substituting variables and handling quotes. -func (rs *ruleSet) expand(input string, expandBackticks bool) string { - expanded := make([]byte, 0) - var i, j int - for i = 0; i < len(input); { - j = i + strings.IndexAny(input[i:], "\"'`$\\") - - if j < 0 { - expanded = append(expanded, []byte(input[i:])...) - break - } - - expanded = append(expanded, []byte(input[i:j])...) - c, w := utf8.DecodeRuneInString(input[j:]) - i = j + w - - var off int - var out string - switch c { - case '\\': - out, off = rs.expandEscape(input[i:]) - - case '"': - out, off = rs.expandDoubleQuoted(input[i:], expandBackticks) - - case '\'': - out, off = rs.expandSingleQuoted(input[i:]) - - case '`': - if expandBackticks { - out, off = rs.expandBackQuoted(input[i:]) - } else { - out = input - off = len(input) - } - - case '$': - out, off = rs.expandSigil(input[i:]) - } - - expanded = append(expanded, []byte(out)...) - i += off - } - - return string(expanded) -} - -// Expand following a '\\' -func (rs *ruleSet) expandEscape(input string) (string, int) { - c, w := utf8.DecodeRuneInString(input) - return string(c), w -} - -// Expand a double quoted string starting after a '\"' -func (rs *ruleSet) expandDoubleQuoted(input string, expandBackticks bool) (string, int) { - // find the first non-escaped " - j := 0 - for { - j = strings.IndexAny(input[j:], "\"\\") - if j < 0 { - break - } - - _, w := utf8.DecodeRuneInString(input[j:]) - j += w - - c, w := utf8.DecodeRuneInString(input[j:]) - j += w - - if c == '"' { - return rs.expand(input[:j], expandBackticks), (j + w) - } - - if c == '\\' { - if j+w < len(input) { - j += w - _, w := utf8.DecodeRuneInString(input[j:]) - j += w - } else { - break - } - } - } - - return input, len(input) -} - -// Expand a single quoted string starting after a '\'' -func (rs *ruleSet) expandSingleQuoted(input string) (string, int) { - j := strings.Index(input, "'") - if j < 0 { - return input, len(input) - } - return input[:j], (j + 1) -} - -// Expand something starting with at '$'. -func (rs *ruleSet) expandSigil(input string) (string, int) { - c, w := utf8.DecodeRuneInString(input) - var offset int - var varname string - if c == '{' { - j := strings.Index(input[w:], "}") - if j < 0 { - return input, len(input) - } - - varname = input[w:j] - offset = j + 1 - } else { - // try to match a variable name - i := w - j := i - for j < len(input) { - c, w = utf8.DecodeRuneInString(input) - if !(isalpha(c) || c == '_' || (j > i && isdigit(c))) { - break - } - j += w - } - - if j > i { - varname = input[i:j] - } else { - return input, len(input) + k := len(rs.rules) - 1 + for i := range r.targets { + if r.targets[i].rpat == nil { + rs.targetrules[r.targets[i].spat] = + append(rs.targetrules[r.targets[i].spat], k) } } - - if isValidVarName(varname) { - varvals, ok := rs.vars[varname] - if ok { - return strings.Join(varvals, " "), offset - } - } - - return input, len(input) } -// Expand a backtick quoted string, by executing the contents. -func (rs *ruleSet) expandBackQuoted(input string) (string, int) { - j := strings.Index(input, "`") - if j < 0 { - return input, len(input) - } - - output := executeRecipe("sh", nil, input[:j], false, false, true) - return output, (j + 1) -} func isValidVarName(v string) bool { for i := 0; i < len(v); { @@ -286,7 +176,7 @@ func (rs *ruleSet) executeAssignment(ts []token) *assignmentError { // expanded variables vals := make([]string, len(ts)-1) for i := 0; i < len(vals); i++ { - vals[i] = rs.expand(ts[i+1].val, true) + vals[i] = expand(ts[i+1].val, rs.vars, true) } rs.vars[assignee] = vals