Work on graph build and metarule matching.

This commit is contained in:
Daniel Jones 2013-03-02 00:34:31 -08:00
parent f812efe3ad
commit 207e052372
4 changed files with 369 additions and 189 deletions

214
expand.go Normal file
View file

@ -0,0 +1,214 @@
// String substitution and expansion.
package main
import (
"strings"
"unicode/utf8"
)
// Expand a word. This includes substituting variables and handling quotes.
func expand(input string, vars map[string][]string, expandBackticks bool) string {
expanded := make([]byte, 0)
var i, j int
for i = 0; i < len(input); {
j = i + strings.IndexAny(input[i:], "\"'`$\\")
if j < 0 {
expanded = append(expanded, input[i:]...)
break
}
expanded = append(expanded, input[i:j]...)
c, w := utf8.DecodeRuneInString(input[j:])
i = j + w
var off int
var out string
switch c {
case '\\':
out, off = expandEscape(input[i:])
case '"':
out, off = expandDoubleQuoted(input[i:], vars, expandBackticks)
case '\'':
out, off = expandSingleQuoted(input[i:])
case '`':
if expandBackticks {
out, off = expandBackQuoted(input[i:], vars)
} else {
out = input
off = len(input)
}
case '$':
out, off = expandSigil(input[i:], vars)
}
expanded = append(expanded, out...)
i += off
}
return string(expanded)
}
// Expand following a '\\'
func expandEscape(input string) (string, int) {
c, w := utf8.DecodeRuneInString(input)
return string(c), w
}
// Expand a double quoted string starting after a '\"'
func expandDoubleQuoted(input string, vars map[string][]string, expandBackticks bool) (string, int) {
// find the first non-escaped "
j := 0
for {
j = strings.IndexAny(input[j:], "\"\\")
if j < 0 {
break
}
_, w := utf8.DecodeRuneInString(input[j:])
j += w
c, w := utf8.DecodeRuneInString(input[j:])
j += w
if c == '"' {
return expand(input[:j], vars, expandBackticks), (j + w)
}
if c == '\\' {
if j+w < len(input) {
j += w
_, w := utf8.DecodeRuneInString(input[j:])
j += w
} else {
break
}
}
}
return input, len(input)
}
// Expand a single quoted string starting after a '\''
func expandSingleQuoted(input string) (string, int) {
j := strings.Index(input, "'")
if j < 0 {
return input, len(input)
}
return input[:j], (j + 1)
}
// Expand something starting with at '$'.
func expandSigil(input string, vars map[string][]string) (string, int) {
c, w := utf8.DecodeRuneInString(input)
var offset int
var varname string
if c == '{' {
j := strings.IndexRune(input[w:], '}')
if j < 0 {
return input, len(input)
}
varname = input[w:j]
offset = j + 1
} else {
// try to match a variable name
i := w
j := i
for j < len(input) {
c, w = utf8.DecodeRuneInString(input)
if !(isalpha(c) || c == '_' || (j > i && isdigit(c))) {
break
}
j += w
}
if j > i {
varname = input[i:j]
} else {
return input, len(input)
}
}
if isValidVarName(varname) {
varvals, ok := vars[varname]
if ok {
return strings.Join(varvals, " "), offset
}
}
return input, len(input)
}
// Find and expand all sigils.
func expandSigils(input string, vars map[string][]string) string {
expanded := make([]byte, 0)
for i := 0; i < len(input); {
j := strings.IndexRune(input[i:], '$')
if j < 0 {
expanded = append(expanded, input[i:]...)
break
}
ex, k := expandSigil(input[j+1:], vars)
expanded = append(expanded, ex...)
i = k
}
return string(expanded)
}
// Expand all unescaped '%' characters.
func expandSuffixes(input string, stem string) string {
expanded := make([]byte, 0)
for i := 0; i < len(input); {
j := strings.IndexAny(input[i:], "\\%")
if j < 0 {
expanded = append(expanded, input[i:]...)
break
}
c, w := utf8.DecodeRuneInString(input[j:])
if c == '%' {
expanded = append(expanded, stem...)
i += w
} else {
j += w
c, w := utf8.DecodeRuneInString(input[j:])
if c == '%' {
expanded = append(expanded, '%')
i = j + w
}
}
}
return string(expanded)
}
// TODO: expand RegexpRefs
// Expand a backtick quoted string, by executing the contents.
func expandBackQuoted(input string, vars map[string][]string) (string, int) {
// TODO: expand sigils?
j := strings.Index(input, "`")
if j < 0 {
return input, len(input)
}
output := executeRecipe("sh", nil, input[:j], false, false, true)
return output, (j + 1)
}

76
mk.go
View file

@ -1,25 +1,69 @@
package main
import (
"fmt"
"io/ioutil"
"os"
"flag"
"fmt"
"io/ioutil"
"os"
)
func main() {
input, _ := ioutil.ReadAll(os.Stdin)
// TEST LEXING
//_, tokens := lex(string(input))
//for t := range tokens {
//fmt.Printf("%s %s\n", t.typ, t.val)
//}
// The maximum number of times an rule may be applied.
const max_rule_cnt = 3
// TEST PARSING
rs := parse(string(input), "<stdin>")
fmt.Println(rs)
// TEST STRING EXPANSION
//rules := &ruleSet{make(map[string][]string), make([]rule, 0)}
//println(rules.expand("\"This is a quote: \\\"\""))
func mk(rs *ruleSet, target string, dryrun bool) {
// Build a graph
// 1. Introduce special variables into the ruleSet
}
func mkError(msg string) {
fmt.Fprintf(os.Stderr, "mk: %s\n", msg)
os.Exit(1)
}
func main() {
var mkfilepath string
var dryrun bool
flag.StringVar(&mkfilepath, "f", "mkfile", "use the given file as mkfile")
flag.BoolVar(&dryrun, "n", false, "print commands without actually executing")
flag.Parse()
mkfile, err := os.Open(mkfilepath)
if err != nil {
mkError("no mkfile found")
}
input, _ := ioutil.ReadAll(mkfile)
mkfile.Close()
rs := parse(string(input), mkfilepath)
targets := flag.Args()
// build the first non-meta rule in the makefile, if none are given explicitly
for i := range rs.rules {
if !rs.rules[i].ismeta {
for j := range rs.rules[i].targets {
targets = append(targets, rs.rules[i].targets[j].spat)
}
}
}
if len(targets) == 0 {
fmt.Println("mk: nothing to mk")
return
}
for _, target := range targets {
//fmt.Printf("building: %q\n", target)
g := buildgraph(rs, target)
g.visualize(os.Stdout)
//mk(rs, target, dryrun)
}
}

View file

@ -6,6 +6,8 @@ package main
import (
"fmt"
"os"
"regexp"
"strings"
)
type parser struct {
@ -50,7 +52,9 @@ type parserStateFun func(*parser, token) parserStateFun
// Parse a mkfile, returning a new ruleSet.
func parse(input string, name string) *ruleSet {
rules := &ruleSet{make(map[string][]string), make([]rule, 0)}
rules := &ruleSet{make(map[string][]string),
make([]rule, 0),
make(map[string][]int)}
parseInto(input, name, rules)
return rules
}
@ -155,14 +159,12 @@ func parseRedirInclude(p *parser, t token) parserStateFun {
// Encountered a bare string at the beginning of the line.
func parseAssignmentOrTarget(p *parser, t token) parserStateFun {
fmt.Println("assignment or target")
p.push(t)
return parseEqualsOrTarget
}
// Consumed one bare string ot the beginning of the line.
func parseEqualsOrTarget(p *parser, t token) parserStateFun {
fmt.Println("equals or target")
switch t.typ {
case tokenAssign:
return parseAssignment
@ -220,7 +222,6 @@ func parseTargets(p *parser, t token) parserStateFun {
// Consumed one or more strings followed by a first ':'.
func parseAttributesOrPrereqs(p *parser, t token) parserStateFun {
fmt.Println("attributes or prereqs")
switch t.typ {
case tokenNewline:
return parseRecipe
@ -239,7 +240,6 @@ func parseAttributesOrPrereqs(p *parser, t token) parserStateFun {
// Targets and attributes and the second ':' have been consumed.
func parsePrereqs(p *parser, t token) parserStateFun {
fmt.Println("prereqs")
switch t.typ {
case tokenNewline:
return parseRecipe
@ -256,8 +256,6 @@ func parsePrereqs(p *parser, t token) parserStateFun {
// An entire rule has been consumed.
func parseRecipe(p *parser, t token) parserStateFun {
fmt.Println("recipe")
// Assemble the rule!
r := rule{}
@ -269,39 +267,73 @@ func parseRecipe(p *parser, t token) parserStateFun {
for ; j < len(p.tokenbuf) && p.tokenbuf[j].typ != tokenColon; j++ {
}
// targets
r.targets = make([]string, i)
for k := 0; k < i; k++ {
r.targets[k] = p.rules.expand(p.tokenbuf[k].val, true)
}
// rule has attributes
// TODO: should we be expanding the attribute strings?
if j < len(p.tokenbuf) {
attribs := make([]string, j-i-1)
for k := i + 1; k < j; k++ {
attribs[k-i-1] = p.tokenbuf[k].val
attribs[k-i-1] = expand(p.tokenbuf[k].val, p.rules.vars, true)
}
err := r.parseAttribs(attribs)
if err != nil {
msg := fmt.Sprintf("while reading a rule's attributes expected an attribute but found \"%c\".", err.found)
p.basicErrorAtToken(msg, p.tokenbuf[i+1])
}
if r.attributes.regex {
r.ismeta = true
}
} else {
j = i
}
// targets
r.targets = make([]pattern, i)
for k := 0; k < i; k++ {
targetstr := expand(p.tokenbuf[k].val, p.rules.vars, true)
r.targets[k].spat = targetstr
if r.attributes.regex {
rpat, err := regexp.Compile(targetstr)
if err != nil {
msg := fmt.Sprintf("invalid regular expression: %q", err)
p.basicErrorAtToken(msg, p.tokenbuf[k])
}
r.targets[k].rpat = rpat
} else {
idx := strings.IndexRune(targetstr, '%')
if idx >= 0 {
var left, right string
if idx > 0 {
left = regexp.QuoteMeta(targetstr[:idx])
}
if idx < len(targetstr) - 1 {
right = regexp.QuoteMeta(targetstr[idx+1:])
}
patstr := fmt.Sprintf("^%s(.*)%s$", left, right)
rpat, err := regexp.Compile(patstr)
if err != nil {
msg := fmt.Sprintf("error compiling suffix rule. This is a bug.", err)
p.basicErrorAtToken(msg, p.tokenbuf[k])
}
r.targets[k].rpat = rpat
r.targets[k].issuffix = true
r.ismeta = true
}
}
}
// prereqs
r.prereqs = make([]string, len(p.tokenbuf)-j-1)
for k := j + 1; k < len(p.tokenbuf); k++ {
r.prereqs[k-j-1] = p.rules.expand(p.tokenbuf[k].val, true)
r.prereqs[k-j-1] = expand(p.tokenbuf[k].val, p.rules.vars, true)
}
if t.typ == tokenRecipe {
r.recipe = p.rules.expand(t.val, false)
r.recipe = t.val
}
p.rules.push(r)
p.rules.add(r)
p.clear()
// the current token doesn't belong to this rule

View file

@ -6,8 +6,8 @@ package main
import (
"fmt"
"strings"
"unicode/utf8"
"regexp"
)
type attribSet struct {
@ -26,15 +26,51 @@ type attribError struct {
found rune
}
// target and rereq patterns
type pattern struct {
issuffix bool // is a suffix '%' rule, so we should define $stem.
spat string // simple string pattern
rpat *regexp.Regexp // non-nil if this is a regexp pattern
}
// Match a pattern, returning an array of submatches, or nil if it doesn'm
// match.
func (p *pattern) match(target string) []string {
if p.rpat != nil {
return p.rpat.FindStringSubmatch(target)
}
if target == p.spat {
return make([]string, 0)
}
return nil
}
// A single rule.
type rule struct {
targets []string // non-empty array of targets
targets []pattern // non-empty array of targets
attributes attribSet // rule attributes
prereqs []string // possibly empty prerequesites
shell []string // command used to execute the recipe
recipe string // recipe source
command []string // command attribute
ismeta bool // is this a meta rule
}
// A set of rules.
type ruleSet struct {
vars map[string][]string
rules []rule
// map a target to an array of indexes into rules
targetrules map[string][]int
}
// Read attributes for an array of strings, updating the rule.
func (r *rule) parseAttribs(inputs []string) *attribError {
for i := 0; i < len(inputs); i++ {
@ -84,164 +120,18 @@ func (r *rule) parseAttribs(inputs []string) *attribError {
return nil
}
type ruleSet struct {
vars map[string][]string
rules []rule
}
// Add a rule to the rule set.
func (rs *ruleSet) push(r rule) {
func (rs *ruleSet) add(r rule) {
rs.rules = append(rs.rules, r)
}
// Expand a word. This includes substituting variables and handling quotes.
func (rs *ruleSet) expand(input string, expandBackticks bool) string {
expanded := make([]byte, 0)
var i, j int
for i = 0; i < len(input); {
j = i + strings.IndexAny(input[i:], "\"'`$\\")
if j < 0 {
expanded = append(expanded, []byte(input[i:])...)
break
}
expanded = append(expanded, []byte(input[i:j])...)
c, w := utf8.DecodeRuneInString(input[j:])
i = j + w
var off int
var out string
switch c {
case '\\':
out, off = rs.expandEscape(input[i:])
case '"':
out, off = rs.expandDoubleQuoted(input[i:], expandBackticks)
case '\'':
out, off = rs.expandSingleQuoted(input[i:])
case '`':
if expandBackticks {
out, off = rs.expandBackQuoted(input[i:])
} else {
out = input
off = len(input)
}
case '$':
out, off = rs.expandSigil(input[i:])
}
expanded = append(expanded, []byte(out)...)
i += off
}
return string(expanded)
}
// Expand following a '\\'
func (rs *ruleSet) expandEscape(input string) (string, int) {
c, w := utf8.DecodeRuneInString(input)
return string(c), w
}
// Expand a double quoted string starting after a '\"'
func (rs *ruleSet) expandDoubleQuoted(input string, expandBackticks bool) (string, int) {
// find the first non-escaped "
j := 0
for {
j = strings.IndexAny(input[j:], "\"\\")
if j < 0 {
break
}
_, w := utf8.DecodeRuneInString(input[j:])
j += w
c, w := utf8.DecodeRuneInString(input[j:])
j += w
if c == '"' {
return rs.expand(input[:j], expandBackticks), (j + w)
}
if c == '\\' {
if j+w < len(input) {
j += w
_, w := utf8.DecodeRuneInString(input[j:])
j += w
} else {
break
}
}
}
return input, len(input)
}
// Expand a single quoted string starting after a '\''
func (rs *ruleSet) expandSingleQuoted(input string) (string, int) {
j := strings.Index(input, "'")
if j < 0 {
return input, len(input)
}
return input[:j], (j + 1)
}
// Expand something starting with at '$'.
func (rs *ruleSet) expandSigil(input string) (string, int) {
c, w := utf8.DecodeRuneInString(input)
var offset int
var varname string
if c == '{' {
j := strings.Index(input[w:], "}")
if j < 0 {
return input, len(input)
}
varname = input[w:j]
offset = j + 1
} else {
// try to match a variable name
i := w
j := i
for j < len(input) {
c, w = utf8.DecodeRuneInString(input)
if !(isalpha(c) || c == '_' || (j > i && isdigit(c))) {
break
}
j += w
}
if j > i {
varname = input[i:j]
} else {
return input, len(input)
k := len(rs.rules) - 1
for i := range r.targets {
if r.targets[i].rpat == nil {
rs.targetrules[r.targets[i].spat] =
append(rs.targetrules[r.targets[i].spat], k)
}
}
if isValidVarName(varname) {
varvals, ok := rs.vars[varname]
if ok {
return strings.Join(varvals, " "), offset
}
}
return input, len(input)
}
// Expand a backtick quoted string, by executing the contents.
func (rs *ruleSet) expandBackQuoted(input string) (string, int) {
j := strings.Index(input, "`")
if j < 0 {
return input, len(input)
}
output := executeRecipe("sh", nil, input[:j], false, false, true)
return output, (j + 1)
}
func isValidVarName(v string) bool {
for i := 0; i < len(v); {
@ -286,7 +176,7 @@ func (rs *ruleSet) executeAssignment(ts []token) *assignmentError {
// expanded variables
vals := make([]string, len(ts)-1)
for i := 0; i < len(vals); i++ {
vals[i] = rs.expand(ts[i+1].val, true)
vals[i] = expand(ts[i+1].val, rs.vars, true)
}
rs.vars[assignee] = vals