Tokenize output from shell expansions. (Fixes #1)

expand.go

@@ -42,12 +42,18 @@ func expand(input string, vars map[string][]string, expandBackticks bool) []stri
 
 		case '`':
 			if expandBackticks {
-				out, off = expandBackQuoted(input[i:], vars)
+				var outparts []string
+				outparts, off = expandBackQuoted(input[i:], vars)
+				if len(outparts) > 0 {
+					outparts[0] = expanded + outparts[0]
+					expanded = outparts[len(outparts)-1]
+					parts = append(parts, outparts[:len(outparts)-1]...)
+				}
 			} else {
 				out = input
 				off = len(input)
+				expanded += out
 			}
-			expanded += out
 
 		case '$':
 			var outparts []string
@@ -251,46 +257,21 @@ func expandSuffixes(input string, stem string) string {
 // TODO: expand RegexpRefs
 
 // Expand a backtick quoted string, by executing the contents.
-func expandBackQuoted(input string, vars map[string][]string) (string, int) {
+func expandBackQuoted(input string, vars map[string][]string) ([]string, int) {
 	// TODO: expand sigils?
 	j := strings.Index(input, "`")
 	if j < 0 {
-		return input, len(input)
+		return []string{input}, len(input)
 	}
 
 	// TODO: handle errors
 	output, _ := subprocess("sh", nil, input[:j], true)
-	return output, (j + 1)
+
+	parts := make([]string, 0)
+	_, tokens := lexWords(output)
+	for t := range tokens {
+		parts = append(parts, t.val)
+	}
+
+	return parts, (j + 1)
 }
-
-// Split a string on whitespace taking into account escaping and quoting.
-//func splitQuoted(input string) []string {
-//parts := make([]string, 0)
-//var i, j int
-//i = 0
-//for {
-//// skip all unescaped whitespace
-//for i < len(input) {
-//c, w := utf8.DecodeRuneInString(input[i:])
-//if strings.IndexRune(" \t", c) < 0 {
-//break
-//}
-//i += w
-//}
-
-//if i >= len(input) {
-//break
-//}
-
-//// Ugh. Will this take into account quoting in variables?
-
-//switch c {
-//case '"':
-//case '\'':
-//default:
-
-//}
-//}
-
-//return parts
-//}

lex.go

@@ -69,15 +69,16 @@ func (t *token) String() string {
 }
 
 type lexer struct {
-	input    string     // input string to be lexed
-	output   chan token // channel on which tokens are sent
-	start    int        // token beginning
-	startcol int        // column on which the token begins
-	pos      int        // position within input
-	line     int        // line within input
-	col      int        // column within input
-	errmsg   string     // set to an appropriate error message when necessary
-	indented bool       // true if the only whitespace so far on this line
+	input     string     // input string to be lexed
+	output    chan token // channel on which tokens are sent
+	start     int        // token beginning
+	startcol  int        // column on which the token begins
+	pos       int        // position within input
+	line      int        // line within input
+	col       int        // column within input
+	errmsg    string     // set to an appropriate error message when necessary
+	indented  bool       // true if the only whitespace so far on this line
+	barewords bool       // lex only a sequence of words
 }
 
 // A lexerStateFun is simultaneously the the state of the lexer and the next
@@ -214,6 +215,12 @@ func lex(input string) (*lexer, chan token) {
 	return l, l.output
 }
 
+func lexWords(input string) (*lexer, chan token) {
+	l := &lexer{input: input, output: make(chan token), line: 1, col: 0, indented: true, barewords: true}
+	go l.run()
+	return l, l.output
+}
+
 func (l *lexer) run() {
 	for state := lexTopLevel; state != nil; {
 		state = state(l)
@@ -221,17 +228,17 @@ func (l *lexer) run() {
 	close(l.output)
 }
 
-// What do we need?
-// A function that consumes non-newline whitespace.
-// A way of determining if the current line might be a recipe.
-
 func lexTopLevel(l *lexer) lexerStateFun {
 	for {
 		l.skipRun(" \t\r")
 		// emit a newline token if we are ending a non-empty line.
 		if l.peek() == '\n' && !l.indented {
 			l.next()
-			l.emit(tokenNewline)
+			if l.barewords {
+				return nil
+			} else {
+				l.emit(tokenNewline)
+			}
 		}
 		l.skipRun(" \t\r\n")