package main // Parser and executor for command lines and scripts: lists (; & && || and // newlines), pipelines, ! and the compound commands if, for, while, until, // case and { ...; }, and function definitions. Simple commands (with their pipes and redirections) are kept as // text; they are expanded and run by execCommand only when they are // executed, so that e.g. the variable of a for loop is expanded anew in // every iteration. import ( "errors" "fmt" "os" "path" "slices" "strconv" "strings" "syscall" ) // incompleteError is returned by parse if the input ends inside a // construct (if without fi, an open quote, a trailing && ...): an // interactive shell then reads another line. type incompleteError struct{} func (incompleteError) Error() string { return "syntax error: unexpected end of input" } var errIncomplete error = incompleteError{} // loopControl is returned by break and continue and ends the given number of // enclosing loops (break) or iterations (continue). type loopControl struct { isBreak bool n int } func (c loopControl) Error() string { if c.isBreak { return "break" } return "continue" } // loopDepth is the number of loops that are currently running. var loopDepth int // aliasExpansion is false while the result of an alias expansion runs, so // that it is not expanded again. var aliasExpansion = true // reserved words that end a list; they are only recognized as the first word // of a command. var terminators = map[string]bool{ "then": true, "elif": true, "else": true, "fi": true, "do": true, "done": true, "esac": true, "}": true, } // compoundStarts are the reserved words that start a compound command. var compoundStarts = map[string]bool{"if": true, "for": true, "while": true, "until": true, "case": true, "{": true} // --------------------------------------------------------------------------- // Lexer type tokenKind int const ( tWord tokenKind = iota tOp // ; ;; & && || | or newline ) type rawToken struct { kind tokenKind text string start, end int } // joinContinuationLines removes backslash-newline (outside single quotes), // which continues a line. continued reports whether src ends with such a // continuation (so that the command goes on in the next line). func joinContinuationLines(src string) (joined string, continued bool) { if !strings.Contains(src, "\\\n") { return src, false } var b strings.Builder var quote byte for i := 0; i < len(src); i++ { ch := src[i] switch { case quote == '\'': if ch == '\'' { quote = 0 } case ch == '\\' && i+1 < len(src): if src[i+1] == '\n' { i++ continued = i == len(src)-1 continue } continued = false b.WriteByte(ch) i++ ch = src[i] case ch == '"': if quote == '"' { quote = 0 } else { quote = '"' } case ch == '\'' && quote == 0: quote = '\'' } if ch != '\n' && ch != ' ' && ch != '\t' { continued = false } b.WriteByte(ch) } return b.String(), continued } // lex splits src into words and list operators. Words keep their quotes, // escapes, $(...) and redirections; they are only split at blanks and list // operators. func lex(src string) ([]rawToken, error) { var toks []rawToken op := func(text string, i int) { toks = append(toks, rawToken{tOp, text, i, i + len(text)}) } for i := 0; i < len(src); { ch := src[i] next := byte(0) if i+1 < len(src) { next = src[i+1] } switch { case ch == ' ' || ch == '\t': i++ case ch == ';' && next == ';': op(";;", i) i += 2 case ch == '\n' || ch == ';': op(string(ch), i) i++ case ch == '#': // Comment until the end of the line for i < len(src) && src[i] != '\n' { i++ } case ch == '|' && next == '|', ch == '&' && next == '&': op(src[i:i+2], i) i += 2 case ch == '|', ch == '&' && next != '>': op(string(ch), i) i++ default: end, err := lexWord(src, i) if err != nil { return nil, err } toks = append(toks, rawToken{tWord, src[i:end], i, end}) i = end } } return toks, nil } // lexWord returns the end of the word that starts at src[i]. func lexWord(src string, i int) (int, error) { var quote byte for j := i; j < len(src); j++ { ch := src[j] switch { case quote == '\'': if ch == '\'' { quote = 0 } case ch == '\\': if j+1 == len(src) { return 0, errIncomplete } j++ case isSubstStart(src, j): end := substEnd(src, j) if end < 0 { return 0, errIncomplete } j = end case ch == '$' && j+1 < len(src) && src[j+1] == '{': if k := strings.IndexByte(src[j:], '}'); k >= 0 { j += k } case quote == '"': if ch == '"' { quote = 0 } case ch == '"' || ch == '\'': quote = ch case ch == ' ' || ch == '\t' || ch == '\n' || ch == ';' || ch == '|': return j, nil case ch == '&': // &>, >& and <& are redirections, not the & operator if !(j+1 < len(src) && src[j+1] == '>') && !(j > i && (src[j-1] == '>' || src[j-1] == '<')) { return j, nil } } } if quote != 0 { return 0, errIncomplete } return len(src), nil } // --------------------------------------------------------------------------- // Syntax tree // node is a command that can be executed: it returns the error of the last // command it ran (so that exitStatus gives its status). type node interface { exec() error } // listNode is a list of and-or lists separated by ;, & or newlines. type listNode struct { items []listEntry } type listEntry struct { andOr *andOrNode background bool } // andOrNode is a pipeline followed by && and || pipelines. type andOrNode struct { first *pipelineNode rest []andOrPart } type andOrPart struct { op string // && or || pipeline *pipelineNode } // pipelineNode is a simple command (whose text may contain a pipeline) or a // control structure, optionally negated with !. type pipelineNode struct { negate bool cmd node } // simpleNode is the text of a pipeline of simple commands. type simpleNode struct { text string } type ifNode struct { conds []*listNode bodies []*listNode elseBody *listNode } type forNode struct { name string words string // the words after "in", expanded when the loop starts hasIn bool body *listNode } type whileNode struct { until bool cond *listNode body *listNode } // caseNode is case word in pattern|pattern) list ;; ... esac. type caseNode struct { word string // expanded when the case command runs clauses []caseClause } type caseClause struct { patterns []string // expanded when they are tried body *listNode } // groupNode is { list; }. type groupNode struct { body *listNode } // funcDefNode is a function definition name() compound-command. type funcDefNode struct { def *funcDef } // singleSimple returns the simple command if the list is just that. func (l *listNode) singleSimple() *simpleNode { if len(l.items) != 1 || l.items[0].background { return nil } ao := l.items[0].andOr if len(ao.rest) > 0 || ao.first.negate { return nil } s, _ := ao.first.cmd.(*simpleNode) return s } // --------------------------------------------------------------------------- // Parser type parser struct { src string toks []rawToken pos int } // parse parses a command line or script. It returns errIncomplete if the // input ends inside a construct. func parse(src string) (*listNode, error) { src, continued := joinContinuationLines(src) if continued { return nil, errIncomplete } toks, err := lex(src) if err != nil { return nil, err } p := &parser{src: src, toks: toks} list, err := p.parseList(nil) if err != nil { return nil, err } if t := p.peek(); t != nil { return nil, p.unexpected(t) } return list, nil } func (p *parser) peek() *rawToken { if p.pos < len(p.toks) { return &p.toks[p.pos] } return nil } func (p *parser) peekWord(words ...string) bool { t := p.peek() if t == nil || t.kind != tWord { return false } for _, w := range words { if t.text == w { return true } } return false } func (p *parser) skipNewlines() { for t := p.peek(); t != nil && t.kind == tOp && t.text == "\n"; t = p.peek() { p.pos++ } } func (p *parser) unexpected(t *rawToken) error { if t.text == "\n" { return syntaxError("newline") } return syntaxError(t.text) } // expect consumes the reserved word w. func (p *parser) expect(w string) error { t := p.peek() if t == nil { return errIncomplete } if t.kind != tWord || t.text != w { return p.unexpected(t) } p.pos++ return nil } // parseList parses and-or lists until the end of the input or one of the // reserved words in terms at the start of a command. func (p *parser) parseList(terms map[string]bool) (*listNode, error) { list := &listNode{} for { p.skipNewlines() t := p.peek() if t == nil || terms[t.text] { return list, nil } ao, err := p.parseAndOr() if err != nil { return nil, err } entry := listEntry{andOr: ao} t = p.peek() switch { case t == nil: case t.kind == tOp && (t.text == ";" || t.text == "&" || t.text == "\n"): p.pos++ entry.background = t.text == "&" case terms[t.text]: // e.g. "fi" right after the end of a nested control structure, // or ;; after a command in a case clause default: return nil, p.unexpected(t) } if entry.background { if _, ok := ao.first.cmd.(*simpleNode); !ok || len(ao.rest) > 0 || ao.first.negate { return nil, errors.New("only simple commands and pipelines can run in the background") } } list.items = append(list.items, entry) } } // nonEmpty returns an error if a list that must contain commands is empty. func (p *parser) nonEmpty(list *listNode) error { if len(list.items) > 0 { return nil } if t := p.peek(); t != nil { return p.unexpected(t) } return errIncomplete } func (p *parser) parseAndOr() (*andOrNode, error) { first, err := p.parsePipeline() if err != nil { return nil, err } ao := &andOrNode{first: first} for { t := p.peek() if t == nil || t.kind != tOp || (t.text != "&&" && t.text != "||") { return ao, nil } p.pos++ p.skipNewlines() if p.peek() == nil { return nil, errIncomplete } next, err := p.parsePipeline() if err != nil { return nil, err } ao.rest = append(ao.rest, andOrPart{op: t.text, pipeline: next}) } } func (p *parser) parsePipeline() (*pipelineNode, error) { pl := &pipelineNode{} if p.peekWord("!") { pl.negate = true p.pos++ } t := p.peek() if t == nil { return nil, errIncomplete } if t.kind == tOp || terminators[t.text] { return nil, p.unexpected(t) } var err error compound := true start := t.start // funcDefName may split the token if name, ok := p.funcDefName(); ok { pl.cmd, err = p.parseFuncDef(name, start) } else if compoundStarts[t.text] { pl.cmd, err = p.parseCompound() } else { compound = false pl.cmd, err = p.parseSimple() } if err != nil { return nil, err } if compound { if t := p.peek(); t != nil { switch { case t.kind == tOp && t.text == "|": return nil, errors.New("control structures in pipelines are not supported") case t.kind == tWord && !terminators[t.text]: return nil, fmt.Errorf("redirections of control structures are not supported (near `%s')", t.text) } } } return pl, nil } // parseSimple parses a pipeline of simple commands and returns its text. func (p *parser) parseSimple() (*simpleNode, error) { start := p.toks[p.pos].start end := start for t := p.peek(); t != nil; t = p.peek() { if t.kind == tWord { end = t.end p.pos++ continue } if t.text != "|" { break } p.pos++ end = t.end p.skipNewlines() next := p.peek() switch { case next == nil: return nil, errIncomplete case next.kind == tOp || terminators[next.text]: return nil, p.unexpected(next) case compoundStarts[next.text]: return nil, errors.New("control structures in pipelines are not supported") } } return &simpleNode{text: p.src[start:end]}, nil } // parseCompound parses the compound command that starts at the current // token. func (p *parser) parseCompound() (node, error) { switch p.peek().text { case "if": return p.parseIf() case "for": return p.parseFor() case "while", "until": return p.parseWhile() case "case": return p.parseCase() default: // { return p.parseGroup() } } func (p *parser) parseIf() (*ifNode, error) { p.pos++ // if n := &ifNode{} for { cond, err := p.parseList(map[string]bool{"then": true}) if err == nil { err = p.nonEmpty(cond) } if err == nil { err = p.expect("then") } if err != nil { return nil, err } body, err := p.parseList(map[string]bool{"elif": true, "else": true, "fi": true}) if err == nil { err = p.nonEmpty(body) } if err != nil { return nil, err } n.conds = append(n.conds, cond) n.bodies = append(n.bodies, body) t := p.peek() if t == nil { return nil, errIncomplete } p.pos++ switch t.text { case "elif": continue case "else": body, err := p.parseList(map[string]bool{"fi": true}) if err == nil { err = p.nonEmpty(body) } if err == nil { err = p.expect("fi") } if err != nil { return nil, err } n.elseBody = body } return n, nil // fi } } func (p *parser) parseFor() (*forNode, error) { p.pos++ // for t := p.peek() if t == nil { return nil, errIncomplete } if t.kind != tWord || !isVarName(t.text) { return nil, fmt.Errorf("for: `%s': not a valid identifier", sanitize(t.text)) } n := &forNode{name: t.text} p.pos++ p.skipNewlines() if p.peekWord("in") { p.pos++ n.hasIn = true var words []string for t := p.peek(); t != nil && t.kind == tWord; t = p.peek() { words = append(words, t.text) p.pos++ } n.words = strings.Join(words, " ") } t = p.peek() switch { case t == nil: return nil, errIncomplete case t.kind == tOp && (t.text == ";" || t.text == "\n"): p.pos++ case !n.hasIn && t.kind == tWord && t.text == "do": default: return nil, p.unexpected(t) } p.skipNewlines() body, err := p.parseLoopBody() if err != nil { return nil, err } n.body = body return n, nil } func (p *parser) parseWhile() (*whileNode, error) { n := &whileNode{until: p.peek().text == "until"} p.pos++ cond, err := p.parseList(map[string]bool{"do": true}) if err == nil { err = p.nonEmpty(cond) } if err != nil { return nil, err } n.cond = cond body, err := p.parseLoopBody() if err != nil { return nil, err } n.body = body return n, nil } // parseGroup parses { list; }. func (p *parser) parseGroup() (*groupNode, error) { p.pos++ // { body, err := p.parseList(map[string]bool{"}": true}) if err == nil { err = p.nonEmpty(body) } if err == nil { err = p.expect("}") } if err != nil { return nil, err } return &groupNode{body: body}, nil } // parseCase parses case word in [(]pattern[|pattern]...) list ;; ... esac. // The last clause may omit the ;;. func (p *parser) parseCase() (*caseNode, error) { p.pos++ // case t := p.peek() if t == nil { return nil, errIncomplete } if t.kind != tWord { return nil, p.unexpected(t) } n := &caseNode{word: t.text} p.pos++ p.skipNewlines() if err := p.expect("in"); err != nil { return nil, err } for { p.skipNewlines() t := p.peek() if t == nil { return nil, errIncomplete } if t.kind == tWord && t.text == "esac" { p.pos++ return n, nil } patterns, err := p.parsePatterns() if err != nil { return nil, err } body, err := p.parseList(map[string]bool{";;": true, "esac": true}) if err != nil { return nil, err } n.clauses = append(n.clauses, caseClause{patterns: patterns, body: body}) if t := p.peek(); t == nil { return nil, errIncomplete } else if t.text == ";;" { p.pos++ } } } // parsePatterns parses the patterns of a case clause up to and including the // ). Since ( and ) are not operators for the lexer, they can be part of a // word token ("a)", "(a", "*)echo"); such tokens are split here. func (p *parser) parsePatterns() ([]string, error) { var patterns []string if t := p.peek(); t.kind == tWord && strings.HasPrefix(t.text, "(") { // The optional ( before the first pattern if t.text == "(" { p.pos++ } else { t.text, t.start = t.text[1:], t.start+1 } } for { t := p.peek() if t == nil { return nil, errIncomplete } if t.kind != tWord { return nil, p.unexpected(t) } i := unquotedIndex(t.text, ')') if i < 0 { patterns = append(patterns, t.text) p.pos++ next := p.peek() switch { case next == nil: return nil, errIncomplete case next.kind == tOp && next.text == "|": p.pos++ case next.kind == tWord && strings.HasPrefix(next.text, ")"): default: return nil, p.unexpected(next) } continue } if i > 0 { patterns = append(patterns, t.text[:i]) } if len(patterns) == 0 { return nil, syntaxError(")") } if rest := t.text[i+1:]; rest == "" { p.pos++ } else { // The first word of the body follows right after the ) t.text, t.start = rest, t.start+i+1 } return patterns, nil } } // unquotedIndex returns the index of the first c in word that is not quoted, // escaped or inside $(...) or ${...}, or -1. func unquotedIndex(word string, c byte) int { var quote byte for i := 0; i < len(word); i++ { ch := word[i] switch { case quote == '\'': if ch == '\'' { quote = 0 } case ch == '\\': i++ case isSubstStart(word, i): if end := substEnd(word, i); end >= 0 { i = end } case ch == '$' && i+1 < len(word) && word[i+1] == '{': if k := strings.IndexByte(word[i:], '}'); k >= 0 { i += k } case quote == '"': if ch == '"' { quote = 0 } case ch == '"' || ch == '\'': quote = ch case ch == c: return i } } return -1 } // funcDefName reports whether a function definition starts at the current // token: name(), name () or function name [()]. It consumes the tokens up to // the body; a body that starts in the same token (name(){) becomes a token of // its own. func (p *parser) funcDefName() (string, bool) { start := p.pos t := p.peek() keyword := t.kind == tWord && t.text == "function" if keyword { p.pos++ if t = p.peek(); t == nil || t.kind != tWord { p.pos = start return "", false } } var name, rest string if n, r, ok := strings.Cut(t.text, "()"); ok && isFuncName(n) { // name() or name(){ name, rest = n, r } else if next := p.pos + 1; t.kind == tWord && isFuncName(t.text) && next < len(p.toks) && p.toks[next].kind == tWord && strings.HasPrefix(p.toks[next].text, "()") { // name () name = t.text p.pos = next t = &p.toks[next] rest = t.text[2:] } else if keyword && isFuncName(t.text) { // function name name = t.text } else { p.pos = start return "", false } if rest == "" { p.pos++ } else { t.text, t.start = rest, t.end-len(rest) } return name, true } // parseFuncDef parses the body of a function definition that started at // src[start]: a compound command. func (p *parser) parseFuncDef(name string, start int) (*funcDefNode, error) { p.skipNewlines() t := p.peek() if t == nil { return nil, errIncomplete } if t.kind != tWord || !compoundStarts[t.text] { return nil, fmt.Errorf("%s: the body of a function must be a compound command like { ...; }", sanitize(name)) } body, err := p.parseCompound() if err != nil { return nil, err } return &funcDefNode{def: &funcDef{name: name, body: body, text: p.src[start:p.toks[p.pos-1].end]}}, nil } // parseLoopBody parses "do list done". func (p *parser) parseLoopBody() (*listNode, error) { if err := p.expect("do"); err != nil { return nil, err } body, err := p.parseList(map[string]bool{"done": true}) if err == nil { err = p.nonEmpty(body) } if err == nil { err = p.expect("done") } return body, err } // --------------------------------------------------------------------------- // Execution // isAbort reports whether err stops the execution of the enclosing lists: // break, continue, return, or a command that was interrupted with Ctrl+C // (like in bash, Ctrl+C aborts the whole command line, e.g. a loop). func isAbort(err error) bool { var ( lc loopControl ret returnSignal ) return errors.As(err, &lc) || errors.As(err, &ret) || exitStatus(err) == 128+int(syscall.SIGINT) } // silent returns an error with only the exit status of err, for errors that // were already reported. func silent(err error) error { if err == nil { return nil } return statusError(exitStatus(err)) } func (l *listNode) exec() error { var err error for _, entry := range l.items { printError(err) if entry.background { err = entry.andOr.first.cmd.(*simpleNode).run(true) } else { err = entry.andOr.exec() } lastStatus = exitStatus(err) if isAbort(err) { return err } } return err } func (a *andOrNode) exec() error { err := a.first.exec() for _, part := range a.rest { lastStatus = exitStatus(err) if isAbort(err) { return err } if (part.op == "&&") != (lastStatus == 0) { continue } printError(err) err = part.pipeline.exec() } return err } func (pl *pipelineNode) exec() error { err := pl.cmd.exec() if !pl.negate || isAbort(err) { return err } printError(err) if exitStatus(err) == 0 { return statusError(1) } return nil } func (s *simpleNode) exec() error { return s.run(false) } // run executes the simple command, in the background if background is set. // Aliases are expanded here (in interactive shells), so that they work at // the start of every command, also inside control structures. func (s *simpleNode) run(background bool) error { text := s.text if interactiveShell && aliasExpansion { if expanded := expandAliases(text); expanded != text { tree, err := parse(expanded) if err != nil { return err } if simple := tree.singleSimple(); simple != nil { text = simple.text } else { if background { return errors.New("an alias with a command list cannot run in the background") } aliasExpansion = false defer func() { aliasExpansion = true }() return tree.exec() } } } return execCommand(text, background) } func (n *ifNode) exec() error { for i, cond := range n.conds { err := cond.exec() if isAbort(err) { return err } printError(err) if exitStatus(err) == 0 { return n.bodies[i].exec() } } if n.elseBody != nil { return n.elseBody.exec() } lastStatus = 0 return nil } // runLoopBody runs one iteration of a loop. It returns whether the loop // must stop, and the error to return from the loop. func runLoopBody(body *listNode, result *error) (stop bool, err error) { err = body.exec() var lc loopControl if errors.As(err, &lc) { *result = nil if lc.n > 1 { lc.n-- return true, lc // ends an outer loop too } return lc.isBreak, nil } if isAbort(err) { return true, err } printError(err) *result = silent(err) return false, nil } func (n *whileNode) exec() error { loopDepth++ defer func() { loopDepth-- }() var result error for { err := n.cond.exec() if isAbort(err) { return err } printError(err) if (exitStatus(err) == 0) == n.until { break } if stop, err := runLoopBody(n.body, &result); stop { if err != nil { return err } break } } lastStatus = exitStatus(result) return result } func (n *forNode) exec() error { // Without "in", the loop runs over the positional parameters words := slices.Clone(positional) if n.hasIn { var err error if words, err = expandWordList(n.words); err != nil { return err } } loopDepth++ defer func() { loopDepth-- }() var result error for _, word := range words { os.Setenv(n.name, word) if stop, err := runLoopBody(n.body, &result); stop { if err != nil { return err } break } } lastStatus = exitStatus(result) return result } func (n *groupNode) exec() error { return n.body.exec() } func (n *funcDefNode) exec() error { defineFunction(n.def) return nil } // exec runs the body of the first clause with a pattern that matches the // word. The status is 0 if no pattern matches. func (n *caseNode) exec() error { word, err := expandCaseWord(n.word) if err != nil { return err } for _, clause := range n.clauses { for _, pattern := range clause.patterns { ok, err := caseMatch(pattern, word) if err != nil { return err } if ok { return clause.body.exec() } } } lastStatus = 0 return nil } // expandCaseWord expands the word of a case command: like an argument, but // without brace expansion, wildcards and splitting. func expandCaseWord(text string) (string, error) { tokens, err := tokenize(text) if err != nil { return "", fmt.Errorf("invalid input: %w", err) } var words []string for _, t := range tokens { if t.op != "" { return "", syntaxError(t.op) } words = append(words, t.word) } return strings.Join(words, " "), nil } // caseMatch reports whether word matches the pattern of a case clause. The // pattern is expanded first; quoted characters in it match literally. Unlike // for file names, * and ? also match a /. func caseMatch(pattern, word string) (bool, error) { tokens, err := tokenize(pattern) if err != nil { return false, fmt.Errorf("invalid input: %w", err) } var literal, glob []string isGlob := false for _, t := range tokens { if t.op != "" { return false, syntaxError(t.op) } literal = append(literal, t.word) glob = append(glob, t.pattern) isGlob = isGlob || t.glob } if !isGlob { return word == strings.Join(literal, " "), nil } // path.Match does not let * match a /; NUL can be in neither string toNUL := strings.NewReplacer("/", "\x00") ok, err := path.Match(toNUL.Replace(strings.Join(glob, " ")), toNUL.Replace(word)) if err != nil { // An invalid pattern only matches itself return word == strings.Join(literal, " "), nil } return ok, nil } // expandWordList expands the words of a for loop like the arguments of a // command (braces, ~, variables, $(...), wildcards). func expandWordList(text string) ([]string, error) { text, err := expandBraceWords(text) if err != nil { return nil, err } tokens, err := tokenize(text) if err != nil { return nil, fmt.Errorf("invalid input: %w", err) } var words []string for _, t := range tokens { if t.op != "" { return nil, syntaxError(t.op) } words = append(words, expandWord(t)...) } return words, nil } // builtinLoopControl implements break [n] and continue [n]. func builtinLoopControl(args []string) error { n := 1 if len(args) > 1 { var err error if n, err = strconv.Atoi(args[1]); err != nil || n < 1 { return fmt.Errorf("%s: %s: loop count out of range", args[0], sanitize(args[1])) } } if loopDepth == 0 { return fmt.Errorf("%s: only meaningful in a loop", args[0]) } return loopControl{isBreak: args[0] == "break", n: min(n, loopDepth)} }