gosh/parser.go

package main

// Parser and executor for command lines and scripts: lists (; & && || and
// newlines), pipelines, ! and the control structures if, for, while and
// until. Simple commands (with their pipes and redirections) are kept as
// text; they are expanded and run by execCommand only when they are
// executed, so that e.g. the variable of a for loop is expanded anew in
// every iteration.

import (
	"errors"
	"fmt"
	"os"
	"strconv"
	"strings"
	"syscall"
)

// incompleteError is returned by parse if the input ends inside a
// construct (if without fi, an open quote, a trailing && ...): an
// interactive shell then reads another line.
type incompleteError struct{}

func (incompleteError) Error() string { return "syntax error: unexpected end of input" }

var errIncomplete error = incompleteError{}

// loopControl is returned by break and continue and ends the given number of
// enclosing loops (break) or iterations (continue).
type loopControl struct {
	isBreak bool
	n       int
}

func (c loopControl) Error() string {
	if c.isBreak {
		return "break"
	}
	return "continue"
}

// loopDepth is the number of loops that are currently running.
var loopDepth int

// aliasExpansion is false while the result of an alias expansion runs, so
// that it is not expanded again.
var aliasExpansion = true

// reserved words that end a list; they are only recognized as the first word
// of a command.
var terminators = map[string]bool{"then": true, "elif": true, "else": true, "fi": true, "do": true, "done": true}

// ---------------------------------------------------------------------------
// Lexer

type tokenKind int

const (
	tWord tokenKind = iota
	tOp             // ; & && || | or newline
)

type rawToken struct {
	kind       tokenKind
	text       string
	start, end int
}

// joinContinuationLines removes backslash-newline (outside single quotes),
// which continues a line. continued reports whether src ends with such a
// continuation (so that the command goes on in the next line).
func joinContinuationLines(src string) (joined string, continued bool) {
	if !strings.Contains(src, "\\\n") {
		return src, false
	}
	var b strings.Builder
	var quote byte
	for i := 0; i < len(src); i++ {
		ch := src[i]
		switch {
		case quote == '\'':
			if ch == '\'' {
				quote = 0
			}
		case ch == '\\' && i+1 < len(src):
			if src[i+1] == '\n' {
				i++
				continued = i == len(src)-1
				continue
			}
			continued = false
			b.WriteByte(ch)
			i++
			ch = src[i]
		case ch == '"':
			if quote == '"' {
				quote = 0
			} else {
				quote = '"'
			}
		case ch == '\'' && quote == 0:
			quote = '\''
		}
		if ch != '\n' && ch != ' ' && ch != '\t' {
			continued = false
		}
		b.WriteByte(ch)
	}
	return b.String(), continued
}

// lex splits src into words and list operators. Words keep their quotes,
// escapes, $(...) and redirections; they are only split at blanks and list
// operators.
func lex(src string) ([]rawToken, error) {
	var toks []rawToken
	op := func(text string, i int) {
		toks = append(toks, rawToken{tOp, text, i, i + len(text)})
	}
	for i := 0; i < len(src); {
		ch := src[i]
		next := byte(0)
		if i+1 < len(src) {
			next = src[i+1]
		}
		switch {
		case ch == ' ' || ch == '\t':
			i++
		case ch == '\n' || ch == ';':
			op(string(ch), i)
			i++
		case ch == '#':
			// Comment until the end of the line
			for i < len(src) && src[i] != '\n' {
				i++
			}
		case ch == '|' && next == '|', ch == '&' && next == '&':
			op(src[i:i+2], i)
			i += 2
		case ch == '|', ch == '&' && next != '>':
			op(string(ch), i)
			i++
		default:
			end, err := lexWord(src, i)
			if err != nil {
				return nil, err
			}
			toks = append(toks, rawToken{tWord, src[i:end], i, end})
			i = end
		}
	}
	return toks, nil
}

// lexWord returns the end of the word that starts at src[i].
func lexWord(src string, i int) (int, error) {
	var quote byte
	for j := i; j < len(src); j++ {
		ch := src[j]
		switch {
		case quote == '\'':
			if ch == '\'' {
				quote = 0
			}
		case ch == '\\':
			if j+1 == len(src) {
				return 0, errIncomplete
			}
			j++
		case isSubstStart(src, j):
			end := substEnd(src, j)
			if end < 0 {
				return 0, errIncomplete
			}
			j = end
		case ch == '$' && j+1 < len(src) && src[j+1] == '{':
			if k := strings.IndexByte(src[j:], '}'); k >= 0 {
				j += k
			}
		case quote == '"':
			if ch == '"' {
				quote = 0
			}
		case ch == '"' || ch == '\'':
			quote = ch
		case ch == ' ' || ch == '\t' || ch == '\n' || ch == ';' || ch == '|':
			return j, nil
		case ch == '&':
			// &>, >& and <& are redirections, not the & operator
			if !(j+1 < len(src) && src[j+1] == '>') && !(j > i && (src[j-1] == '>' || src[j-1] == '<')) {
				return j, nil
			}
		}
	}
	if quote != 0 {
		return 0, errIncomplete
	}
	return len(src), nil
}

// ---------------------------------------------------------------------------
// Syntax tree

// node is a command that can be executed: it returns the error of the last
// command it ran (so that exitStatus gives its status).
type node interface {
	exec() error
}

// listNode is a list of and-or lists separated by ;, & or newlines.
type listNode struct {
	items []listEntry
}

type listEntry struct {
	andOr      *andOrNode
	background bool
}

// andOrNode is a pipeline followed by && and || pipelines.
type andOrNode struct {
	first *pipelineNode
	rest  []andOrPart
}

type andOrPart struct {
	op       string // && or ||
	pipeline *pipelineNode
}

// pipelineNode is a simple command (whose text may contain a pipeline) or a
// control structure, optionally negated with !.
type pipelineNode struct {
	negate bool
	cmd    node
}

// simpleNode is the text of a pipeline of simple commands.
type simpleNode struct {
	text string
}

type ifNode struct {
	conds    []*listNode
	bodies   []*listNode
	elseBody *listNode
}

type forNode struct {
	name  string
	words string // the words after "in", expanded when the loop starts
	hasIn bool
	body  *listNode
}

type whileNode struct {
	until bool
	cond  *listNode
	body  *listNode
}

// singleSimple returns the simple command if the list is just that.
func (l *listNode) singleSimple() *simpleNode {
	if len(l.items) != 1 || l.items[0].background {
		return nil
	}
	ao := l.items[0].andOr
	if len(ao.rest) > 0 || ao.first.negate {
		return nil
	}
	s, _ := ao.first.cmd.(*simpleNode)
	return s
}

// ---------------------------------------------------------------------------
// Parser

type parser struct {
	src  string
	toks []rawToken
	pos  int
}

// parse parses a command line or script. It returns errIncomplete if the
// input ends inside a construct.
func parse(src string) (*listNode, error) {
	src, continued := joinContinuationLines(src)
	if continued {
		return nil, errIncomplete
	}
	toks, err := lex(src)
	if err != nil {
		return nil, err
	}
	p := &parser{src: src, toks: toks}
	list, err := p.parseList(nil)
	if err != nil {
		return nil, err
	}
	if t := p.peek(); t != nil {
		return nil, p.unexpected(t)
	}
	return list, nil
}

func (p *parser) peek() *rawToken {
	if p.pos < len(p.toks) {
		return &p.toks[p.pos]
	}
	return nil
}

func (p *parser) peekWord(words ...string) bool {
	t := p.peek()
	if t == nil || t.kind != tWord {
		return false
	}
	for _, w := range words {
		if t.text == w {
			return true
		}
	}
	return false
}

func (p *parser) skipNewlines() {
	for t := p.peek(); t != nil && t.kind == tOp && t.text == "\n"; t = p.peek() {
		p.pos++
	}
}

func (p *parser) unexpected(t *rawToken) error {
	if t.text == "\n" {
		return syntaxError("newline")
	}
	return syntaxError(t.text)
}

// expect consumes the reserved word w.
func (p *parser) expect(w string) error {
	t := p.peek()
	if t == nil {
		return errIncomplete
	}
	if t.kind != tWord || t.text != w {
		return p.unexpected(t)
	}
	p.pos++
	return nil
}

// parseList parses and-or lists until the end of the input or one of the
// reserved words in terms at the start of a command.
func (p *parser) parseList(terms map[string]bool) (*listNode, error) {
	list := &listNode{}
	for {
		p.skipNewlines()
		t := p.peek()
		if t == nil || t.kind == tWord && terms[t.text] {
			return list, nil
		}
		ao, err := p.parseAndOr()
		if err != nil {
			return nil, err
		}
		entry := listEntry{andOr: ao}

		t = p.peek()
		switch {
		case t == nil:
		case t.kind == tOp && (t.text == ";" || t.text == "&" || t.text == "\n"):
			p.pos++
			entry.background = t.text == "&"
		case t.kind == tWord && terms[t.text]:
			// e.g. "fi" right after the end of a nested control structure
		default:
			return nil, p.unexpected(t)
		}
		if entry.background {
			if _, ok := ao.first.cmd.(*simpleNode); !ok || len(ao.rest) > 0 || ao.first.negate {
				return nil, errors.New("only simple commands and pipelines can run in the background")
			}
		}
		list.items = append(list.items, entry)
	}
}

// nonEmpty returns an error if a list that must contain commands is empty.
func (p *parser) nonEmpty(list *listNode) error {
	if len(list.items) > 0 {
		return nil
	}
	if t := p.peek(); t != nil {
		return p.unexpected(t)
	}
	return errIncomplete
}

func (p *parser) parseAndOr() (*andOrNode, error) {
	first, err := p.parsePipeline()
	if err != nil {
		return nil, err
	}
	ao := &andOrNode{first: first}
	for {
		t := p.peek()
		if t == nil || t.kind != tOp || (t.text != "&&" && t.text != "||") {
			return ao, nil
		}
		p.pos++
		p.skipNewlines()
		if p.peek() == nil {
			return nil, errIncomplete
		}
		next, err := p.parsePipeline()
		if err != nil {
			return nil, err
		}
		ao.rest = append(ao.rest, andOrPart{op: t.text, pipeline: next})
	}
}

func (p *parser) parsePipeline() (*pipelineNode, error) {
	pl := &pipelineNode{}
	if p.peekWord("!") {
		pl.negate = true
		p.pos++
	}
	t := p.peek()
	if t == nil {
		return nil, errIncomplete
	}
	if t.kind == tOp || terminators[t.text] {
		return nil, p.unexpected(t)
	}

	var err error
	compound := true
	switch t.text {
	case "if":
		pl.cmd, err = p.parseIf()
	case "for":
		pl.cmd, err = p.parseFor()
	case "while", "until":
		pl.cmd, err = p.parseWhile()
	default:
		compound = false
		pl.cmd, err = p.parseSimple()
	}
	if err != nil {
		return nil, err
	}

	if compound {
		if t := p.peek(); t != nil {
			switch {
			case t.kind == tOp && t.text == "|":
				return nil, errors.New("control structures in pipelines are not supported")
			case t.kind == tWord && !terminators[t.text]:
				return nil, fmt.Errorf("redirections of control structures are not supported (near `%s')", t.text)
			}
		}
	}
	return pl, nil
}

// parseSimple parses a pipeline of simple commands and returns its text.
func (p *parser) parseSimple() (*simpleNode, error) {
	start := p.toks[p.pos].start
	end := start
	for t := p.peek(); t != nil; t = p.peek() {
		if t.kind == tWord {
			end = t.end
			p.pos++
			continue
		}
		if t.text != "|" {
			break
		}
		p.pos++
		end = t.end
		p.skipNewlines()
		next := p.peek()
		switch {
		case next == nil:
			return nil, errIncomplete
		case next.kind == tOp || terminators[next.text]:
			return nil, p.unexpected(next)
		case next.text == "if" || next.text == "for" || next.text == "while" || next.text == "until":
			return nil, errors.New("control structures in pipelines are not supported")
		}
	}
	return &simpleNode{text: p.src[start:end]}, nil
}

func (p *parser) parseIf() (*ifNode, error) {
	p.pos++ // if
	n := &ifNode{}
	for {
		cond, err := p.parseList(map[string]bool{"then": true})
		if err == nil {
			err = p.nonEmpty(cond)
		}
		if err == nil {
			err = p.expect("then")
		}
		if err != nil {
			return nil, err
		}
		body, err := p.parseList(map[string]bool{"elif": true, "else": true, "fi": true})
		if err == nil {
			err = p.nonEmpty(body)
		}
		if err != nil {
			return nil, err
		}
		n.conds = append(n.conds, cond)
		n.bodies = append(n.bodies, body)

		t := p.peek()
		if t == nil {
			return nil, errIncomplete
		}
		p.pos++
		switch t.text {
		case "elif":
			continue
		case "else":
			body, err := p.parseList(map[string]bool{"fi": true})
			if err == nil {
				err = p.nonEmpty(body)
			}
			if err == nil {
				err = p.expect("fi")
			}
			if err != nil {
				return nil, err
			}
			n.elseBody = body
		}
		return n, nil // fi
	}
}

func (p *parser) parseFor() (*forNode, error) {
	p.pos++ // for
	t := p.peek()
	if t == nil {
		return nil, errIncomplete
	}
	if t.kind != tWord || !isVarName(t.text) {
		return nil, fmt.Errorf("for: `%s': not a valid identifier", sanitize(t.text))
	}
	n := &forNode{name: t.text}
	p.pos++

	p.skipNewlines()
	if p.peekWord("in") {
		p.pos++
		n.hasIn = true
		var words []string
		for t := p.peek(); t != nil && t.kind == tWord; t = p.peek() {
			words = append(words, t.text)
			p.pos++
		}
		n.words = strings.Join(words, " ")
	}
	t = p.peek()
	switch {
	case t == nil:
		return nil, errIncomplete
	case t.kind == tOp && (t.text == ";" || t.text == "\n"):
		p.pos++
	case !n.hasIn && t.kind == tWord && t.text == "do":
	default:
		return nil, p.unexpected(t)
	}

	p.skipNewlines()
	body, err := p.parseLoopBody()
	if err != nil {
		return nil, err
	}
	n.body = body
	return n, nil
}

func (p *parser) parseWhile() (*whileNode, error) {
	n := &whileNode{until: p.peek().text == "until"}
	p.pos++
	cond, err := p.parseList(map[string]bool{"do": true})
	if err == nil {
		err = p.nonEmpty(cond)
	}
	if err != nil {
		return nil, err
	}
	n.cond = cond
	body, err := p.parseLoopBody()
	if err != nil {
		return nil, err
	}
	n.body = body
	return n, nil
}

// parseLoopBody parses "do list done".
func (p *parser) parseLoopBody() (*listNode, error) {
	if err := p.expect("do"); err != nil {
		return nil, err
	}
	body, err := p.parseList(map[string]bool{"done": true})
	if err == nil {
		err = p.nonEmpty(body)
	}
	if err == nil {
		err = p.expect("done")
	}
	return body, err
}

// ---------------------------------------------------------------------------
// Execution

// isAbort reports whether err stops the execution of the enclosing lists:
// break, continue, or a command that was interrupted with Ctrl+C (like in
// bash, Ctrl+C aborts the whole command line, e.g. a loop).
func isAbort(err error) bool {
	var lc loopControl
	return errors.As(err, &lc) || exitStatus(err) == 128+int(syscall.SIGINT)
}

// silent returns an error with only the exit status of err, for errors that
// were already reported.
func silent(err error) error {
	if err == nil {
		return nil
	}
	return statusError(exitStatus(err))
}

func (l *listNode) exec() error {
	var err error
	for _, entry := range l.items {
		printError(err)
		if entry.background {
			err = entry.andOr.first.cmd.(*simpleNode).run(true)
		} else {
			err = entry.andOr.exec()
		}
		lastStatus = exitStatus(err)
		if isAbort(err) {
			return err
		}
	}
	return err
}

func (a *andOrNode) exec() error {
	err := a.first.exec()
	for _, part := range a.rest {
		lastStatus = exitStatus(err)
		if isAbort(err) {
			return err
		}
		if (part.op == "&&") != (lastStatus == 0) {
			continue
		}
		printError(err)
		err = part.pipeline.exec()
	}
	return err
}

func (pl *pipelineNode) exec() error {
	err := pl.cmd.exec()
	if !pl.negate || isAbort(err) {
		return err
	}
	printError(err)
	if exitStatus(err) == 0 {
		return statusError(1)
	}
	return nil
}

func (s *simpleNode) exec() error {
	return s.run(false)
}

// run executes the simple command, in the background if background is set.
// Aliases are expanded here (in interactive shells), so that they work at
// the start of every command, also inside control structures.
func (s *simpleNode) run(background bool) error {
	text := s.text
	if interactiveShell && aliasExpansion {
		if expanded := expandAliases(text); expanded != text {
			tree, err := parse(expanded)
			if err != nil {
				return err
			}
			if simple := tree.singleSimple(); simple != nil {
				text = simple.text
			} else {
				if background {
					return errors.New("an alias with a command list cannot run in the background")
				}
				aliasExpansion = false
				defer func() { aliasExpansion = true }()
				return tree.exec()
			}
		}
	}
	return execCommand(text, background)
}

func (n *ifNode) exec() error {
	for i, cond := range n.conds {
		err := cond.exec()
		if isAbort(err) {
			return err
		}
		printError(err)
		if exitStatus(err) == 0 {
			return n.bodies[i].exec()
		}
	}
	if n.elseBody != nil {
		return n.elseBody.exec()
	}
	lastStatus = 0
	return nil
}

// runLoopBody runs one iteration of a loop. It returns whether the loop
// must stop, and the error to return from the loop.
func runLoopBody(body *listNode, result *error) (stop bool, err error) {
	err = body.exec()
	var lc loopControl
	if errors.As(err, &lc) {
		*result = nil
		if lc.n > 1 {
			lc.n--
			return true, lc // ends an outer loop too
		}
		return lc.isBreak, nil
	}
	if isAbort(err) {
		return true, err
	}
	printError(err)
	*result = silent(err)
	return false, nil
}

func (n *whileNode) exec() error {
	loopDepth++
	defer func() { loopDepth-- }()

	var result error
	for {
		err := n.cond.exec()
		if isAbort(err) {
			return err
		}
		printError(err)
		if (exitStatus(err) == 0) == n.until {
			break
		}
		if stop, err := runLoopBody(n.body, &result); stop {
			if err != nil {
				return err
			}
			break
		}
	}
	lastStatus = exitStatus(result)
	return result
}

func (n *forNode) exec() error {
	var words []string
	if n.hasIn {
		var err error
		if words, err = expandWordList(n.words); err != nil {
			return err
		}
	}

	loopDepth++
	defer func() { loopDepth-- }()

	var result error
	for _, word := range words {
		os.Setenv(n.name, word)
		if stop, err := runLoopBody(n.body, &result); stop {
			if err != nil {
				return err
			}
			break
		}
	}
	lastStatus = exitStatus(result)
	return result
}

// expandWordList expands the words of a for loop like the arguments of a
// command (braces, ~, variables, $(...), wildcards).
func expandWordList(text string) ([]string, error) {
	text, err := expandBraceWords(text)
	if err != nil {
		return nil, err
	}
	tokens, err := tokenize(text)
	if err != nil {
		return nil, fmt.Errorf("invalid input: %w", err)
	}
	var words []string
	for _, t := range tokens {
		if t.op != "" {
			return nil, syntaxError(t.op)
		}
		words = append(words, expandWord(t)...)
	}
	return words, nil
}

// builtinLoopControl implements break [n] and continue [n].
func builtinLoopControl(args []string) error {
	n := 1
	if len(args) > 1 {
		var err error
		if n, err = strconv.Atoi(args[1]); err != nil || n < 1 {
			return fmt.Errorf("%s: %s: loop count out of range", args[0], sanitize(args[1]))
		}
	}
	if loopDepth == 0 {
		return fmt.Errorf("%s: only meaningful in a loop", args[0])
	}
	return loopControl{isBreak: args[0] == "break", n: min(n, loopDepth)}
}