Concurrent rule execution.
This commit is contained in:
Daniel Jones
parent
115d8425be
commit
ee70f46012
6 changed files with 518 additions and 31 deletions
38
expand.go
38
expand.go
|
|
@ -142,7 +142,7 @@ func expandSigil(input string, vars map[string][]string) ([]string, int) {
|
|||
i := 0
|
||||
j := i
|
||||
for j < len(input) {
|
||||
c, w = utf8.DecodeRuneInString(input)
|
||||
c, w = utf8.DecodeRuneInString(input[j:])
|
||||
if !(isalpha(c) || c == '_' || (j > i && isdigit(c))) {
|
||||
break
|
||||
}
|
||||
|
|
@ -196,6 +196,39 @@ func expandSigils(input string, vars map[string][]string) []string {
|
|||
}
|
||||
|
||||
|
||||
// Find and expand all sigils in a recipe, producing a flat string.
|
||||
func expandRecipeSigils(input string, vars map[string][]string) string {
|
||||
expanded := ""
|
||||
for i := 0; i < len(input); {
|
||||
j := strings.IndexAny(input[i:], "$\\")
|
||||
if j < 0 {
|
||||
expanded += input[i:]
|
||||
break
|
||||
}
|
||||
|
||||
expanded += input[i:j]
|
||||
i = j
|
||||
c, w := utf8.DecodeRuneInString(input[i:])
|
||||
if c == '$' {
|
||||
i += w
|
||||
ex, k := expandSigil(input[i:], vars)
|
||||
expanded += strings.Join(ex, " ")
|
||||
i += k
|
||||
} else if c == '\\' {
|
||||
i += w
|
||||
c, w := utf8.DecodeRuneInString(input[i:])
|
||||
if c == '$' {
|
||||
expanded += "$"
|
||||
} else {
|
||||
expanded += "\\" + string(c)
|
||||
}
|
||||
i += w
|
||||
}
|
||||
}
|
||||
|
||||
return expanded
|
||||
}
|
||||
|
||||
|
||||
// Expand all unescaped '%' characters.
|
||||
func expandSuffixes(input string, stem string) string {
|
||||
|
|
@ -236,7 +269,8 @@ func expandBackQuoted(input string, vars map[string][]string) (string, int) {
|
|||
return input, len(input)
|
||||
}
|
||||
|
||||
output := executeRecipe("sh", nil, input[:j], false, false, true)
|
||||
// TODO: handle errors
|
||||
output, _ := subprocess("sh", nil, input[:j], false, false, true)
|
||||
return output, (j + 1)
|
||||
}
|
||||
|
||||
|
|
|
|||
206
graph.go
Normal file
206
graph.go
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A dependency graph
|
||||
type graph struct {
|
||||
root *node // the intial target's node
|
||||
nodes map[string]*node // map targets to their nodes
|
||||
}
|
||||
|
||||
// An edge in the graph.
|
||||
type edge struct {
|
||||
v *node // node this edge directs to
|
||||
stem string // stem matched for meta-rule applications
|
||||
matches []string // regular expression matches
|
||||
r *rule
|
||||
}
|
||||
|
||||
|
||||
// Current status of a node in the build.
|
||||
type nodeStatus int
|
||||
|
||||
const (
|
||||
nodeStatusReady nodeStatus = iota
|
||||
nodeStatusStarted
|
||||
nodeStatusDone
|
||||
nodeStatusFailed
|
||||
)
|
||||
|
||||
|
||||
// A node in the dependency graph
|
||||
type node struct {
|
||||
r *rule // rule to be applied
|
||||
name string // target name
|
||||
prog string // custom program to compare times
|
||||
t time.Time // file modification time
|
||||
exists bool // does a non-virtual target exist
|
||||
prereqs []*edge // prerequisite rules
|
||||
status nodeStatus // current state of the node in the build
|
||||
mutex sync.Mutex // exclusivity for the status variable
|
||||
listeners []chan nodeStatus // channels to notify of completion
|
||||
}
|
||||
|
||||
// Create a new node
|
||||
func (g *graph) newnode(name string) *node {
|
||||
u := &node{name: name}
|
||||
info, err := os.Stat(name)
|
||||
if err == nil {
|
||||
u.t = info.ModTime()
|
||||
u.exists = true
|
||||
} else {
|
||||
_, ok := err.(*os.PathError)
|
||||
if ok {
|
||||
u.exists = false
|
||||
} else {
|
||||
mkError(err.Error())
|
||||
}
|
||||
}
|
||||
g.nodes[name] = u
|
||||
return u
|
||||
}
|
||||
|
||||
// Print a graph in graphviz format.
|
||||
func (g *graph) visualize(w io.Writer) {
|
||||
fmt.Fprintln(w, "digraph mk {")
|
||||
for t, u := range g.nodes {
|
||||
for i := range u.prereqs {
|
||||
if u.prereqs[i].v != nil {
|
||||
fmt.Fprintf(w, " \"%s\" -> \"%s\";\n", t, u.prereqs[i].v.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Fprintln(w, "}")
|
||||
}
|
||||
|
||||
|
||||
// Create a new arc.
|
||||
func (u *node) newedge(v *node, r *rule) *edge {
|
||||
e := &edge{v: v, r: r}
|
||||
u.prereqs = append(u.prereqs, e)
|
||||
return e
|
||||
}
|
||||
|
||||
|
||||
// Create a dependency graph for the given target.
|
||||
func buildgraph(rs *ruleSet, target string) *graph {
|
||||
g := &graph{nil, make(map[string]*node)}
|
||||
|
||||
// keep track of how many times each rule is visited, to avoid cycles.
|
||||
rulecnt := make([]int, len(rs.rules))
|
||||
g.root = applyrules(rs, g, target, rulecnt)
|
||||
|
||||
return g
|
||||
}
|
||||
|
||||
// Recursively match the given target to a rule in the rule set to construct the
|
||||
// full graph.
|
||||
func applyrules(rs *ruleSet, g *graph, target string, rulecnt []int) *node {
|
||||
u, ok := g.nodes[target]
|
||||
if ok {
|
||||
return u
|
||||
}
|
||||
u = g.newnode(target)
|
||||
|
||||
// does the target match a concrete rule?
|
||||
|
||||
ks, ok := rs.targetrules[target]
|
||||
if ok {
|
||||
for ki := range ks {
|
||||
k := ks[ki]
|
||||
if rulecnt[k] > max_rule_cnt {
|
||||
continue
|
||||
}
|
||||
|
||||
r := &rs.rules[k]
|
||||
|
||||
// skip meta-rules
|
||||
if r.ismeta {
|
||||
continue
|
||||
}
|
||||
|
||||
// skip rules that have no effect
|
||||
if r.recipe == "" && len(r.prereqs) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
rulecnt[k] += 1
|
||||
if len(r.prereqs) == 0 {
|
||||
u.newedge(nil, r)
|
||||
} else {
|
||||
for i := range r.prereqs {
|
||||
u.newedge(applyrules(rs, g, r.prereqs[i], rulecnt), r)
|
||||
}
|
||||
}
|
||||
rulecnt[k] -= 1
|
||||
}
|
||||
}
|
||||
|
||||
// find applicable metarules
|
||||
for k := range rs.rules {
|
||||
if rulecnt[k] > max_rule_cnt {
|
||||
continue
|
||||
}
|
||||
|
||||
r := &rs.rules[k]
|
||||
|
||||
if !r.ismeta {
|
||||
continue
|
||||
}
|
||||
|
||||
// skip rules that have no effect
|
||||
if r.recipe == "" && len(r.prereqs) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
for j := range r.targets {
|
||||
mat := r.targets[j].match(target)
|
||||
if mat == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var stem string
|
||||
var matches []string
|
||||
|
||||
if r.attributes.regex {
|
||||
matches = mat
|
||||
} else {
|
||||
stem = mat[1]
|
||||
}
|
||||
|
||||
rulecnt[k] += 1
|
||||
if len(r.prereqs) == 0 {
|
||||
e := u.newedge(nil, r)
|
||||
e.stem = stem
|
||||
e.matches = matches
|
||||
} else {
|
||||
for i := range r.prereqs {
|
||||
var prereq string
|
||||
if r.attributes.regex {
|
||||
// TODO: write substituteRegexpRefs and use that here
|
||||
prereq = r.prereqs[i]
|
||||
} else {
|
||||
prereq = expandSuffixes(r.prereqs[i], stem)
|
||||
}
|
||||
|
||||
e := u.newedge(applyrules(rs, g, prereq, rulecnt), r)
|
||||
e.stem = stem
|
||||
e.matches = matches
|
||||
}
|
||||
}
|
||||
rulecnt[k] -= 1
|
||||
}
|
||||
}
|
||||
|
||||
return u
|
||||
}
|
||||
|
||||
|
||||
|
||||
171
mk.go
171
mk.go
|
|
@ -8,25 +8,175 @@ import (
|
|||
)
|
||||
|
||||
|
||||
// True if messages should be printed without fancy colors.
|
||||
var nocolor bool = false
|
||||
|
||||
// True if we are no actualyl executing any recipes or updating any timestamps.
|
||||
var dryrun bool = false
|
||||
|
||||
// The maximum number of times an rule may be applied.
|
||||
const max_rule_cnt = 3
|
||||
|
||||
// Ansi color codes.
|
||||
const (
|
||||
colorDefault string = "\033[0m"
|
||||
colorBlack string = "\033[30m"
|
||||
colorRed string = "\033[31m"
|
||||
colorGreen string = "\033[32m"
|
||||
colorYellow string = "\033[33m"
|
||||
colorBlue string = "\033[34m"
|
||||
colorMagenta string = "\033[35m"
|
||||
)
|
||||
|
||||
|
||||
func mk(rs *ruleSet, target string, dryrun bool) {
|
||||
|
||||
// Build a graph
|
||||
|
||||
|
||||
|
||||
// 1. Introduce special variables into the ruleSet
|
||||
g := buildgraph(rs, target)
|
||||
//g.visualize(os.Stdout)
|
||||
mkNode(g, g.root)
|
||||
}
|
||||
|
||||
// Build a target in the graph.
|
||||
//
|
||||
// This selects an appropriate rule (edge) and builds all prerequisites
|
||||
// concurrently.
|
||||
//
|
||||
// TODO: control the number of concurrent rules being built. This would involve
|
||||
// some sort of global counter protected by a mutex, so we wait our turn to
|
||||
// execute our rule.
|
||||
//
|
||||
func mkNode(g *graph, u *node) {
|
||||
// try to claim on this node
|
||||
u.mutex.Lock()
|
||||
if u.status != nodeStatusReady {
|
||||
u.mutex.Unlock()
|
||||
return
|
||||
} else {
|
||||
u.status = nodeStatusStarted
|
||||
}
|
||||
u.mutex.Unlock()
|
||||
|
||||
// when finished, notify the listeners
|
||||
finalstatus := nodeStatusDone
|
||||
defer func () {
|
||||
u.mutex.Lock()
|
||||
u.status = finalstatus
|
||||
u.mutex.Unlock()
|
||||
for i := range u.listeners {
|
||||
u.listeners[i] <- u.status
|
||||
}
|
||||
}()
|
||||
|
||||
// there's no fucking rules, dude
|
||||
if len(u.prereqs) == 0 {
|
||||
if !u.r.attributes.virtual && !u.exists {
|
||||
wd, _ := os.Getwd()
|
||||
mkError(fmt.Sprintf("don't know how to make %s in %s", u.name, wd))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// there should otherwise be exactly one edge with an associated rule
|
||||
prereqs := make([]*node, 0)
|
||||
var e *edge = nil
|
||||
for i := range u.prereqs {
|
||||
if u.prereqs[i].r != nil {
|
||||
e = u.prereqs[i]
|
||||
}
|
||||
if u.prereqs[i].v != nil {
|
||||
prereqs = append(prereqs, u.prereqs[i].v)
|
||||
}
|
||||
}
|
||||
|
||||
// this should have been caught during graph building
|
||||
if e == nil {
|
||||
wd, _ := os.Getwd()
|
||||
mkError(fmt.Sprintf("don't know how to make %s in %s", u.name, wd))
|
||||
}
|
||||
|
||||
prereqstat := make(chan nodeStatus)
|
||||
pending := 0
|
||||
|
||||
// build prereqs that need building
|
||||
e.r.mutex.Lock()
|
||||
for i := range prereqs {
|
||||
prereqs[i].mutex.Lock()
|
||||
// needs to be built?
|
||||
if !prereqs[i].exists || e.r.attributes.virtual || (u.exists && u.t.Before(prereqs[i].t)) {
|
||||
switch prereqs[i].status {
|
||||
case nodeStatusReady:
|
||||
go mkNode(g, prereqs[i])
|
||||
fallthrough
|
||||
case nodeStatusStarted:
|
||||
prereqs[i].listeners = append(prereqs[i].listeners, prereqstat)
|
||||
pending++
|
||||
}
|
||||
}
|
||||
prereqs[i].mutex.Unlock()
|
||||
}
|
||||
e.r.mutex.Unlock()
|
||||
|
||||
// wait until all the prereqs are built
|
||||
//fmt.Printf("%s: %d\n", u.name, pending)
|
||||
for pending > 0 {
|
||||
//for i := range prereqs {
|
||||
//fmt.Println(prereqs[i].name)
|
||||
//}
|
||||
|
||||
s := <-prereqstat
|
||||
pending--
|
||||
if s == nodeStatusFailed {
|
||||
finalstatus = nodeStatusFailed
|
||||
}
|
||||
}
|
||||
|
||||
// execute the recipe, unless the prereqs failed
|
||||
if finalstatus != nodeStatusFailed {
|
||||
mkPrintMessage("mking " + u.name)
|
||||
if !dorecipe(u.name, u, e) {
|
||||
finalstatus = nodeStatusFailed
|
||||
}
|
||||
}
|
||||
|
||||
mkPrintSuccess("finished mking " + u.name)
|
||||
}
|
||||
|
||||
func mkError(msg string) {
|
||||
if !nocolor {
|
||||
os.Stderr.WriteString(colorRed)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mk: %s\n", msg)
|
||||
if !nocolor {
|
||||
os.Stderr.WriteString(colorDefault)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func mkPrintSuccess(msg string) {
|
||||
if nocolor {
|
||||
fmt.Println(msg)
|
||||
} else {
|
||||
fmt.Printf("%s%s%s\n", colorGreen, msg, colorDefault)
|
||||
}
|
||||
}
|
||||
|
||||
func mkPrintMessage(msg string) {
|
||||
if nocolor {
|
||||
fmt.Println(msg)
|
||||
} else {
|
||||
fmt.Printf("%s%s%s\n", colorBlue, msg, colorDefault)
|
||||
}
|
||||
}
|
||||
|
||||
func mkPrintRecipe(msg string) {
|
||||
if !nocolor {
|
||||
os.Stdout.WriteString(colorYellow)
|
||||
}
|
||||
printIndented(os.Stdout, msg)
|
||||
if !nocolor {
|
||||
os.Stdout.WriteString(colorDefault)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
func main() {
|
||||
var mkfilepath string
|
||||
|
|
@ -51,6 +201,7 @@ func main() {
|
|||
for j := range rs.rules[i].targets {
|
||||
targets = append(targets, rs.rules[i].targets[j].spat)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -59,11 +210,9 @@ func main() {
|
|||
return
|
||||
}
|
||||
|
||||
// TODO: For multiple targets, we should add a dummy rule that depends on
|
||||
// all let mk handle executing each.
|
||||
for _, target := range targets {
|
||||
//fmt.Printf("building: %q\n", target)
|
||||
g := buildgraph(rs, target)
|
||||
g.visualize(os.Stdout)
|
||||
//mk(rs, target, dryrun)
|
||||
mk(rs, target, dryrun)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
6
parse.go
6
parse.go
|
|
@ -114,7 +114,11 @@ func parsePipeInclude(p *parser, t token) parserStateFun {
|
|||
args[i-1] = p.tokenbuf[i].val
|
||||
}
|
||||
|
||||
output := executeRecipe("sh", args, "", false, false, true)
|
||||
output, success := subprocess("sh", args, "", false, false, true)
|
||||
if !success {
|
||||
p.basicErrorAtToken("subprocess include failed", t)
|
||||
}
|
||||
|
||||
parseInto(output, fmt.Sprintf("%s:sh", p.name), p.rules)
|
||||
|
||||
p.clear()
|
||||
|
|
|
|||
110
recipe.go
110
recipe.go
|
|
@ -1,3 +1,6 @@
|
|||
|
||||
// Various function for dealing with recipes.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
|
@ -5,27 +8,111 @@ import (
|
|||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"bufio"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A monolithic function for executing recipes.
|
||||
func executeRecipe(program string,
|
||||
|
||||
// Try to unindent a recipe, so that it begins an column 0. (This is mainly for
|
||||
// recipes in python, or other indentation-significant languages.)
|
||||
//func stripIndentation(s string) string {
|
||||
|
||||
//}
|
||||
|
||||
// Indent each line of a recipe.
|
||||
func printIndented(out io.Writer, s string) {
|
||||
reader := bufio.NewReader(strings.NewReader(s))
|
||||
for {
|
||||
line, err := reader.ReadString('\n')
|
||||
if len(line) > 0 {
|
||||
io.WriteString(out, " ")
|
||||
io.WriteString(out, line)
|
||||
}
|
||||
|
||||
if (err != nil) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Execute a recipe.
|
||||
func dorecipe(target string, u *node, e *edge) bool {
|
||||
vars := make(map[string][]string)
|
||||
vars["target"] = []string{target}
|
||||
if e.r.ismeta {
|
||||
if e.r.attributes.regex {
|
||||
for i := range e.matches {
|
||||
vars[fmt.Sprintf("stem%d", i)] = e.matches[i:i+1]
|
||||
}
|
||||
} else {
|
||||
vars["stem"] = []string{e.stem}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: other variables to set
|
||||
// alltargets
|
||||
// newprereq
|
||||
|
||||
prereqs := make([]string, 0)
|
||||
for i := range u.prereqs {
|
||||
if u.prereqs[i].r == e.r && u.prereqs[i].v != nil {
|
||||
prereqs = append(prereqs, u.prereqs[i].v.name)
|
||||
}
|
||||
}
|
||||
vars["prereqs"] = prereqs
|
||||
|
||||
input := expandRecipeSigils(e.r.recipe, vars)
|
||||
sh := "sh"
|
||||
args := []string{}
|
||||
|
||||
if len(e.r.shell) > 0 {
|
||||
sh = e.r.shell[0]
|
||||
args = e.r.shell[1:]
|
||||
}
|
||||
|
||||
if !e.r.attributes.quiet {
|
||||
mkPrintRecipe(input)
|
||||
}
|
||||
|
||||
if dryrun {
|
||||
return true
|
||||
}
|
||||
|
||||
_, success := subprocess(
|
||||
sh,
|
||||
args,
|
||||
input,
|
||||
true,
|
||||
true,
|
||||
false)
|
||||
|
||||
|
||||
// TODO: update the timestamps of each target
|
||||
|
||||
return success
|
||||
}
|
||||
|
||||
|
||||
// A monolithic function for executing subprocesses
|
||||
func subprocess(program string,
|
||||
args []string,
|
||||
input string,
|
||||
echo_out bool,
|
||||
echo_err bool,
|
||||
capture_out bool) string {
|
||||
capture_out bool) (string, bool) {
|
||||
cmd := exec.Command(program, args...)
|
||||
|
||||
if echo_out {
|
||||
cmdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
if err == nil {
|
||||
go io.Copy(os.Stdout, cmdout)
|
||||
}
|
||||
}
|
||||
|
||||
if echo_err {
|
||||
cmderr, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
cmderr, err := cmd.StderrPipe()
|
||||
if err == nil {
|
||||
go io.Copy(os.Stderr, cmderr)
|
||||
}
|
||||
}
|
||||
|
|
@ -49,11 +136,16 @@ func executeRecipe(program string,
|
|||
} else {
|
||||
err = cmd.Run()
|
||||
}
|
||||
success := true
|
||||
|
||||
if err != nil {
|
||||
// TODO: better error output
|
||||
log.Fatal("Recipe failed")
|
||||
exiterr, ok := err.(*exec.ExitError)
|
||||
if ok {
|
||||
success = exiterr.ProcessState.Success()
|
||||
} else {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
return output
|
||||
return output, success
|
||||
}
|
||||
|
|
|
|||
18
ruleset.go
18
ruleset.go
|
|
@ -8,6 +8,7 @@ import (
|
|||
"fmt"
|
||||
"unicode/utf8"
|
||||
"regexp"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type attribSet struct {
|
||||
|
|
@ -52,13 +53,14 @@ func (p *pattern) match(target string) []string {
|
|||
|
||||
// A single rule.
|
||||
type rule struct {
|
||||
targets []pattern // non-empty array of targets
|
||||
attributes attribSet // rule attributes
|
||||
prereqs []string // possibly empty prerequesites
|
||||
shell []string // command used to execute the recipe
|
||||
recipe string // recipe source
|
||||
command []string // command attribute
|
||||
ismeta bool // is this a meta rule
|
||||
targets []pattern // non-empty array of targets
|
||||
attributes attribSet // rule attributes
|
||||
prereqs []string // possibly empty prerequesites
|
||||
shell []string // command used to execute the recipe
|
||||
recipe string // recipe source
|
||||
command []string // command attribute
|
||||
ismeta bool // is this a meta rule
|
||||
mutex sync.Mutex // prevent the rule from being executed multiple times
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -122,7 +124,7 @@ func (r *rule) parseAttribs(inputs []string) *attribError {
|
|||
|
||||
// Add a rule to the rule set.
|
||||
func (rs *ruleSet) add(r rule) {
|
||||
rs.rules = append(rs.rules, r)
|
||||
rs.rules = append(rs.rules, r)
|
||||
k := len(rs.rules) - 1
|
||||
for i := range r.targets {
|
||||
if r.targets[i].rpat == nil {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue