From f812efe3ad454d2e7c1993010d4008ce73331bb9 Mon Sep 17 00:00:00 2001 From: Daniel Jones Date: Thu, 28 Feb 2013 22:49:34 -0800 Subject: [PATCH] Some more work on string expansion. --- README.md | 34 ++++++++++++++------------- mk.go | 16 ++++++------- parse.go | 7 +++--- ruleset.go | 68 +++++++++++++++++++++++++++++++++++++++++++++++------- 4 files changed, 90 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 8e7baea..d61451b 100644 --- a/README.md +++ b/README.md @@ -11,42 +11,44 @@ of it. Put simply, mk takes make, keeps its simple direct syntax, but fixes basically everything that's annoyed you over the years. To name a few things: 1. Recipes are delimited by any indentation, not tab characters in particular. - 2. Phony targets are handled separately from file targets. Your mkfile won't + 1. Phony targets are handled separately from file targets. Your mkfile won't be broken by having a file named 'clean'. - 2. Attributes instead of weird special targets like `.SECONDARY:`. - 5. Special variables like `$target`, `$prereq`, and `$stem` in place of + 1. Attributes instead of weird special targets like `.SECONDARY:`. + 1. Special variables like `$target`, `$prereq`, and `$stem` in place of make's pointlessly cryptic `$@`, `$^`, and `$*`. - 3. In addition to suffix rules (e.g. `%.o: %.c`), mk has more powerful regular + 1. In addition to suffix rules (e.g. `%.o: %.c`), mk has more powerful regular expression rules. - 4. Sane handling of rules with multiple targets. - 5. An optional attribute to delete targets when a recipe fails, so you aren't + 1. Sane handling of rules with multiple targets. + 1. An optional attribute to delete targets when a recipe fails, so you aren't left with corrupt output. - 6. Plan 9 mkfiles can not only include other mkfiles, but pipe in the output of + 1. Plan 9 mkfiles can not only include other mkfiles, but pipe in the output of recipes. Your mkfile can configure itself by doing something like `<|sh config.sh`. - 7. A generalized mechanism to determine if a target is out of date, for when + 1. A generalized mechanism to determine if a target is out of date, for when timestamps won't cut it. + 1. Variables are expanded in recipes only if they are defined. They way you + usually don't have to escape `$`. And much more! For more, read the original mk paper: ["Mk: a successor to make"](#). # Improvements over Plan 9 mk -This mk stays mostly faithful to Plan 9, but makes a few minor (in my opinion) +This mk stays mostly faithful to Plan 9, but makes a few (in my opinion) improvements. + 1. A clean, modern implementation in go, that doesn't depend on the whole plan + 9 for userspace stack. + 1. Use go regular expressions, which are perl-like. The original mk used plan9 + regex, which few people know or care to learn. 1. Allow blank lines in recipes. A recipe is any indented block of text, and continues until a non-indented character or the end of the file. - 2. Add an 'S' attribute to execute recipes with programs other than sh. This + 1. Add an 'S' attribute to execute recipes with programs other than sh. This way, you don't have to separate your six line python script into its own file. Just stick it in the mkfile. - 3. Use a perl-compatible regular expressions. The original mk used plan9 - regex, which few people know or care to learn. - 4. A clean, modern implementation in go, that doesn't depend on the whole plan - 9 for userspace stack. + 1. Use sh syntax for command insertion (i.e. backticks) rather than rc shell + syntax. -Most Plan 9 mkfiles should remain backwards compatible, but strict backwards -compatibility isn't the goal. # Current State diff --git a/mk.go b/mk.go index 5f6cd4f..0ea7449 100644 --- a/mk.go +++ b/mk.go @@ -1,13 +1,13 @@ package main import ( -//"fmt" -//"io/ioutil" -//"os" +"fmt" +"io/ioutil" +"os" ) func main() { - //input, _ := ioutil.ReadAll(os.Stdin) + input, _ := ioutil.ReadAll(os.Stdin) // TEST LEXING //_, tokens := lex(string(input)) @@ -16,10 +16,10 @@ func main() { //} // TEST PARSING - //rs := parse(string(input), "") - //fmt.Println(rs) + rs := parse(string(input), "") + fmt.Println(rs) // TEST STRING EXPANSION - rules := &ruleSet{make(map[string][]string), make([]rule, 0)} - println(rules.expand("\"This is a quote: \\\"\"")) + //rules := &ruleSet{make(map[string][]string), make([]rule, 0)} + //println(rules.expand("\"This is a quote: \\\"\"")) } diff --git a/parse.go b/parse.go index e793bf1..8298b0b 100644 --- a/parse.go +++ b/parse.go @@ -272,10 +272,11 @@ func parseRecipe(p *parser, t token) parserStateFun { // targets r.targets = make([]string, i) for k := 0; k < i; k++ { - r.targets[k] = p.tokenbuf[k].val + r.targets[k] = p.rules.expand(p.tokenbuf[k].val, true) } // rule has attributes + // TODO: should we be expanding the attribute strings? if j < len(p.tokenbuf) { attribs := make([]string, j-i-1) for k := i + 1; k < j; k++ { @@ -293,11 +294,11 @@ func parseRecipe(p *parser, t token) parserStateFun { // prereqs r.prereqs = make([]string, len(p.tokenbuf)-j-1) for k := j + 1; k < len(p.tokenbuf); k++ { - r.prereqs[k-j-1] = p.tokenbuf[k].val + r.prereqs[k-j-1] = p.rules.expand(p.tokenbuf[k].val, true) } if t.typ == tokenRecipe { - r.recipe = t.val + r.recipe = p.rules.expand(t.val, false) } p.rules.push(r) diff --git a/ruleset.go b/ruleset.go index f7e1eeb..74b735a 100644 --- a/ruleset.go +++ b/ruleset.go @@ -95,7 +95,7 @@ func (rs *ruleSet) push(r rule) { } // Expand a word. This includes substituting variables and handling quotes. -func (rs *ruleSet) expand(input string) string { +func (rs *ruleSet) expand(input string, expandBackticks bool) string { expanded := make([]byte, 0) var i, j int for i = 0; i < len(input); { @@ -117,16 +117,21 @@ func (rs *ruleSet) expand(input string) string { out, off = rs.expandEscape(input[i:]) case '"': - out, off = rs.expandDoubleQuoted(input[i:]) + out, off = rs.expandDoubleQuoted(input[i:], expandBackticks) case '\'': out, off = rs.expandSingleQuoted(input[i:]) case '`': - out, off = rs.expandBackQuoted(input[i:]) + if expandBackticks { + out, off = rs.expandBackQuoted(input[i:]) + } else { + out = input + off = len(input) + } case '$': - // TODO: recursive call: expandSigil + out, off = rs.expandSigil(input[i:]) } expanded = append(expanded, []byte(out)...) @@ -143,7 +148,7 @@ func (rs *ruleSet) expandEscape(input string) (string, int) { } // Expand a double quoted string starting after a '\"' -func (rs *ruleSet) expandDoubleQuoted(input string) (string, int) { +func (rs *ruleSet) expandDoubleQuoted(input string, expandBackticks bool) (string, int) { // find the first non-escaped " j := 0 for { @@ -159,7 +164,7 @@ func (rs *ruleSet) expandDoubleQuoted(input string) (string, int) { j += w if c == '"' { - return rs.expand(input[:j]), (j + w) + return rs.expand(input[:j], expandBackticks), (j + w) } if c == '\\' { @@ -185,6 +190,48 @@ func (rs *ruleSet) expandSingleQuoted(input string) (string, int) { return input[:j], (j + 1) } +// Expand something starting with at '$'. +func (rs *ruleSet) expandSigil(input string) (string, int) { + c, w := utf8.DecodeRuneInString(input) + var offset int + var varname string + if c == '{' { + j := strings.Index(input[w:], "}") + if j < 0 { + return input, len(input) + } + + varname = input[w:j] + offset = j + 1 + } else { + // try to match a variable name + i := w + j := i + for j < len(input) { + c, w = utf8.DecodeRuneInString(input) + if !(isalpha(c) || c == '_' || (j > i && isdigit(c))) { + break + } + j += w + } + + if j > i { + varname = input[i:j] + } else { + return input, len(input) + } + } + + if isValidVarName(varname) { + varvals, ok := rs.vars[varname] + if ok { + return strings.Join(varvals, " "), offset + } + } + + return input, len(input) +} + // Expand a backtick quoted string, by executing the contents. func (rs *ruleSet) expandBackQuoted(input string) (string, int) { j := strings.Index(input, "`") @@ -209,12 +256,17 @@ func isValidVarName(v string) bool { return true } +func isdigit(c rune) bool { + return '0' <= c && c <= '9' +} + + func isalpha(c rune) bool { return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') } func isalnum(c rune) bool { - return isalpha(c) || ('0' <= c && c <= '9') + return isalpha(c) || isdigit(c) } type assignmentError struct { @@ -234,7 +286,7 @@ func (rs *ruleSet) executeAssignment(ts []token) *assignmentError { // expanded variables vals := make([]string, len(ts)-1) for i := 0; i < len(vals); i++ { - vals[i] = rs.expand(ts[i+1].val) + vals[i] = rs.expand(ts[i+1].val, true) } rs.vars[assignee] = vals