Sketching out the parser.

README.md

@@ -0,0 +1,55 @@
+
+# Mk
+
+Mk is a reboot of the Plan 9 mk command, which itself is a replacement for make.
+This tool is for anyone who loves make, but hates all its stupid bullshit.
+
+# Why Plan 9 mk is better than make
+
+Plan 9 mk blows make out of the water. Yet tragically, few use or have even heard
+of it. Put simply, mk takes make, keeps its simple direct syntax, but fixes
+basically everything that's annoyed you over the years. To name a few things:
+
+  1. Recipes are delimited by any indentation, not tab characters in particular.
+  2. Phony targets are handled separately from file targets. Your mkfile won't
+     be broken by having a file named 'clean'.
+  2. Attributes instead of weird special targets like `.SECONDARY:`.
+  5. Special variables like `$target`, `$prereq`, and `$stem` in place of
+     make's pointlessly cryptic `$@`, `$^`, and `$*`.
+  3. In addition to suffix rules (e.g. `%.o: %.c`), mk has more powerful regular
+     expression rules.
+  4. Sane handling of rules with multiple targets.
+  5. An optional attribute to delete targets when a recipe fails, so you aren't
+     left with corrupt output.
+  6. Plan 9 mkfiles can not only include other mkfiles, but pipe in the output of
+     recipes. Your mkfile can configure itself by doing something like
+     `<|sh config.sh`.
+  7. A generalized mechanism to determine if a target is out of date, for when
+     timestamps won't cut it.
+
+And much more! For more, read the original mk paper: ["Mk: a successor to
+make"](#).
+
+# Improvements over Plan 9 mk
+
+This mk stays mostly faithful to Plan 9, but makes a few minor (in my opinion)
+improvements.
+
+  1. Allow blank lines in recipes. A recipe is any indented block of text, and
+     continues until a non-indented character or the end of the file.
+  2. Add an 'S' attribute to execute recipes with programs other than sh. This
+     way, you don't have to separate your six line python script into its own
+     file. Just stick it in the mkfile.
+  3. Use a perl-compatible regular expressions. The original mk used plan9
+     regex, which few people know or care to learn.
+  4. A clean, modern implementation in go, that doesn't depend on the whole plan
+     9 for userspace stack.
+
+Most Plan 9 mkfiles should remain backwards compatible, but strict backwards
+compatibility isn't the goal.
+
+# Current State
+
+Totally non-functional. Check back later!
+
+

lex.go

@@ -1,4 +1,6 @@
 
+// TODO: Backquoted strings.
+
 package main
 
 import (
@@ -15,7 +17,8 @@ const (
     tokenError tokenType = iota
     tokenBareString
     tokenQuotedString
-    tokenInclude
+    tokenPipeInclude
+    tokenRedirInclude
     tokenColon
     tokenAssign
     tokenRecipe
@@ -27,7 +30,8 @@ func (typ tokenType) String() string {
     case tokenError:        return "[Error]"
     case tokenBareString:   return "[BareString]"
     case tokenQuotedString: return "[QuotedString]"
-    case tokenInclude:      return "[Include]"
+    case tokenPipeInclude:  return "[PipeInclude]"
+    case tokenRedirInclude: return "[RedirInclude]"
     case tokenColon:        return "[Colon]"
     case tokenAssign:       return "[Assign]"
     case tokenRecipe:       return "[Recipe]"
@@ -144,6 +148,17 @@ func (l *lexer) accept(valid string) bool {
 }
 
 
+// Skip the next rune if it is in the valid string. Return true if it was
+// skipped.
+func (l *lexer) ignore(valid string) bool {
+    if strings.IndexRune(valid, l.peek()) >= 0 {
+        l.skip()
+        return true
+    }
+    return false
+}
+
+
 // Consume characters from the valid string until the next is not.
 func (l *lexer) acceptRun(valid string) int {
     prevpos := l.pos
@@ -256,9 +271,15 @@ func lexComment (l* lexer) lexerStateFun {
 
 func lexInclude (l* lexer) lexerStateFun {
     l.skip() // '<'
+    var typ tokenType
+    if l.ignore("|") {
+        typ = tokenPipeInclude
+    } else {
+        typ = tokenRedirInclude
+    }
+
     l.skipRun(" \t\n\r")
-    l.acceptUntil("\n\r")
+    l.emit(typ)
-    l.emit(tokenInclude)
     return lexTopLevel
 }
 

mk.go

@@ -2,23 +2,9 @@
 package main
 
 import (
-    "fmt"
-    "os"
-    "io/ioutil"
 )
 
 func main() {
-    input, _ := ioutil.ReadAll(os.Stdin)
-    l, tokens := lex(string(input))
-
-    for t := range tokens {
-        if t.typ == tokenError {
-            fmt.Printf("Error: %s", l.errmsg)
-            break
-        }
-
-        fmt.Println(t.String())
-    }
 }
 
 

parse.go

@@ -0,0 +1,121 @@
+
+package main
+
+import (
+    "fmt"
+    "os"
+    "os/exec"
+)
+
+/* Grammar, to the best of my knowledge:
+
+Should we deviate at all from mk?
+
+Yes! I want to simplify things by saying recipes have nonzero indentation and
+everything else has zero.
+
+rule ::= targets ':' attributes ':' prereqs NEWLINE RECIPE |
+         targets ':' prereqs NEWLINE RECIPE
+
+targets ::= string | string "," targets
+
+attributes ::= SCALAR | SCALAR attributes
+
+prereqs ::= string | string "," prereqs
+
+include ::= '<' string NEWLINE
+
+string ::= SCALAR | QSTRING
+
+assignment ::= SCALAR '=' string
+
+How do we handle escaping new lines?
+Is newline a token that's emitted?
+
+*/
+
+
+// The parser for mk files is terribly simple. There are only three sorts of
+// statements in mkfiles: variable assignments, rules (possibly with
+// accompanying recipes), and includes.
+
+
+
+//
+// Maybe this is the wrong way to organize things.
+// We should perhaps have a type for a parsed mkfile that includes every
+// assignment as well as every rule.
+//
+// Rule order should not matter.
+//
+// Includes are tricky. If they were straight up includes, the could be
+// evaluated in place, but they could contain shell script, etc.
+//
+// No...we still have to evaluate them in place. That means figuring out how to
+// spawn shells from go.
+//
+
+
+type parser struct {
+    l *lexer         // underlying lexer
+    tokenbuf []token // tokens consumed on the current statement
+    rules *ruleSet   // current ruleSet
+}
+
+
+// A parser state function takes a parser and the next token and returns a new
+// state function, or nil if there was a parse error.
+type parserStateFun func (*parser, token) parserStateFun
+
+
+// Parse a mkfile, returning a new ruleSet.
+func parse(input string) *ruleSet {
+    rules := &ruleSet{}
+    parseInto(input, rules)
+    return rules
+}
+
+
+// Parse a mkfile inserting rules and variables into a given ruleSet.
+func parseInto(input string, rules *ruleSet) {
+    l, tokens := lex(input)
+    p := &parser{l, []token{}, rules}
+    state := parseTopLevel
+    for t := range tokens {
+        if t.typ == tokenError {
+            // TODO: fancier error messages
+            fmt.Fprintf(os.Stderr, "Error: %s", l.errmsg)
+            break
+        }
+
+        state = state(p, t)
+    }
+
+    // TODO: Handle the case when state is not top level.
+}
+
+
+func parseTopLevel(p *parser, t token) parserStateFun {
+    switch t.typ {
+        case tokenPipeInclude: return parsePipeInclude(p, t)
+        // TODO: all others
+    }
+
+    return parseTopLevel
+}
+
+
+func parsePipeInclude(p *parser, t token) parserStateFun {
+    // TODO: We need to split this up into arguments so we can feed it into
+    // executeRecipe.
+    return parseTopLevel
+}
+
+
+func parseRedirInclude(p *parser, t token) parserStateFun {
+    // TODO: Open the file, read its context, call parseInto recursively.
+    return parseTopLevel
+}
+
+
+

recipe.go

@@ -0,0 +1,57 @@
+
+package main
+
+
+import (
+    "os/exec"
+    "os"
+    "io"
+)
+
+
+// A monolithic function for executing recipes.
+func executeRecipe(program string,
+                   args []string,
+                   input string,
+                   echo_out bool,
+                   echo_err bool,
+                   capture_out bool) string {
+    cmd := exec.Command(program, args...)
+
+    if echo_out {
+        cmdout, err := cmd.StdoutPipe()
+        if err != nil {
+            go io.Copy(os.Stdout, cmdout)
+        }
+    }
+
+    if echo_err {
+        cmderr, err := cmd.StdoutPipe()
+        if err != nil {
+            go io.Copy(os.Stderr, cmderr)
+        }
+    }
+
+    if len(input) > 0 {
+        cmdin, err := cmd.StdinPipe()
+        go func () { cmdin.Write([]byte(input)) }()
+    }
+
+
+    output := ""
+    var err error
+    if capture_out {
+        output, err = cmd.Output()
+    } else {
+        err = cmd.Run()
+    }
+
+    if err != nil {
+        // TODO: better error output
+        log.Fatal("Recipe failed")
+    }
+
+    return output
+}
+
+