Florian.Walther/gosh
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 24 Wiki Activity

add subshells, here-documents and redirections/pipes for control structures

Browse Source 
- control structures can have redirections (while read l; do ...; done
  < file), which apply to all commands in them; they run in the shell
- in pipes and in the background they run in a subshell (cmd | while
  read l; do ...; done, for ...; done &, a && b &)
- ( list ) subshells; ( and ) are operators now
- here-documents <<EOF, <<-EOF, <<'EOF' and here-strings <<<word
- redirections of source apply to the commands of the file
- fix non-ASCII characters in quotes being mangled ('für')

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Florian WaltherandClaude Opus 5.5 committed 2026-10-04 20:57:15 +02:00
1 parent 443367f66f
commit f66cf538c0
7 files changed
+1172 -221
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

parser.go
+323 -164
View File
@@ -2,7 +2,7 @@ package main
// Parser and executor for command lines and scripts: lists (; & && || and
// newlines), pipelines, ! and the compound commands if, for, while, until,
// case and { ...; }, and function definitions. Simple commands (with their pipes and redirections) are kept as
// case, { ...; } and ( ... ), and function definitions. Simple commands (with their pipes and redirections) are kept as
// text; they are expanded and run by execCommand only when they are
// executed, so that e.g. the variable of a for loop is expanded anew in
// every iteration.
@@ -65,7 +65,7 @@ type tokenKind int
const (
tWord tokenKind = iota
tOp // ; ;; & && || | or newline
tOp // ; ;; & && || | ( ) or newline
)
type rawToken struct {
@@ -148,7 +148,7 @@ func lex(src string) ([]rawToken, error) {
case ch == '|' && next == '|', ch == '&' && next == '&':
op(src[i:i+2], i)
i += 2
case ch == '|', ch == '&' && next != '>':
case ch == '|', ch == '&' && next != '>', ch == '(', ch == ')':
op(string(ch), i)
i++
default:
@@ -178,6 +178,12 @@ func lexWord(src string, i int) (int, error) {
return 0, errIncomplete
}
j++
if isSubstStart(src, j) {
// \$(...) is literal text, but its ( and ) belong to the word
if end := substEnd(src, j); end >= 0 {
j = end
}
}
case isSubstStart(src, j):
end := substEnd(src, j)
if end < 0 {
@@ -194,7 +200,7 @@ func lexWord(src string, i int) (int, error) {
}
case ch == '"' || ch == '\'':
quote = ch
case ch == ' ' || ch == '\t' || ch == '\n' || ch == ';' || ch == '|':
case ch == ' ' || ch == '\t' || ch == '\n' || ch == ';' || ch == '|' || ch == '(' || ch == ')':
return j, nil
case ch == '&':
// &>, >& and <& are redirections, not the & operator
@@ -232,6 +238,7 @@ type listEntry struct {
type andOrNode struct {
first *pipelineNode
rest []andOrPart
text string // the source, to run it in a subshell in the background
}
type andOrPart struct {
@@ -239,8 +246,8 @@ type andOrPart struct {
pipeline *pipelineNode
}
// pipelineNode is a simple command (whose text may contain a pipeline) or a
// control structure, optionally negated with !.
// pipelineNode is a pipeline (a simpleNode or pipeNode) or a single
// compound command, optionally negated with !.
type pipelineNode struct {
negate bool
cmd node
@@ -251,6 +258,33 @@ type simpleNode struct {
text string
}
// pipeNode is a pipeline that contains compound commands. Like all commands
// of a pipeline in sh, they run in subshells (new gosh processes), since
// the commands of a pipeline run concurrently.
type pipeNode struct {
elems []pipeElem
text string
}
type pipeElem struct {
text string // the source of the command, with its redirections
compound bool
subshell string // for ( list ): the list, which runs in the subshell directly
}
// redirNode is a compound command with redirections; they apply to all
// commands in it.
type redirNode struct {
cmd node
redirs string // the source text of the redirections
}
// subshellNode is ( list ): the list runs in a new gosh process, so that it
// cannot change the shell.
type subshellNode struct {
text string // the source of the list
}
type ifNode struct {
conds []*listNode
bodies []*listNode
@@ -316,6 +350,10 @@ type parser struct {
// parse parses a command line or script. It returns errIncomplete if the
// input ends inside a construct.
func parse(src string) (*listNode, error) {
src, err := extractHeredocs(src)
if err != nil {
return nil, err
}
src, continued := joinContinuationLines(src)
if continued {
return nil, errIncomplete
@@ -409,11 +447,6 @@ func (p *parser) parseList(terms map[string]bool) (*listNode, error) {
default:
return nil, p.unexpected(t)
}
if entry.background {
if _, ok := ao.first.cmd.(*simpleNode); !ok || len(ao.rest) > 0 || ao.first.negate {
return nil, errors.New("only simple commands and pipelines can run in the background")
}
}
list.items = append(list.items, entry)
}
}
@@ -430,6 +463,7 @@ func (p *parser) nonEmpty(list *listNode) error {
}
func (p *parser) parseAndOr() (*andOrNode, error) {
start := p.peek().start
first, err := p.parsePipeline()
if err != nil {
return nil, err
@@ -438,6 +472,7 @@ func (p *parser) parseAndOr() (*andOrNode, error) {
for {
t := p.peek()
if t == nil || t.kind != tOp || (t.text != "&&" && t.text != "||") {
ao.text = p.src[start:p.toks[p.pos-1].end]
return ao, nil
}
p.pos++
@@ -453,73 +488,165 @@ func (p *parser) parseAndOr() (*andOrNode, error) {
}
}
// parsePipeline parses [!] command [| command]... A pipeline of simple
// commands stays one simpleNode (its text is split into the commands when it
// runs); a pipeline with compound commands becomes a pipeNode.
func (p *parser) parsePipeline() (*pipelineNode, error) {
pl := &pipelineNode{}
if p.peekWord("!") {
pl.negate = true
p.pos++
}
t := p.peek()
if t == nil {
return nil, errIncomplete
}
if t.kind == tOp || terminators[t.text] {
return nil, p.unexpected(t)
}
var err error
compound := true
start := t.start // funcDefName may split the token
if name, ok := p.funcDefName(); ok {
pl.cmd, err = p.parseFuncDef(name, start)
} else if compoundStarts[t.text] {
pl.cmd, err = p.parseCompound()
} else {
compound = false
pl.cmd, err = p.parseSimple()
}
if err != nil {
return nil, err
}
if compound {
if t := p.peek(); t != nil {
switch {
case t.kind == tOp && t.text == "|":
return nil, errors.New("control structures in pipelines are not supported")
case t.kind == tWord && !terminators[t.text]:
return nil, fmt.Errorf("redirections of control structures are not supported (near `%s')", t.text)
}
var (
elems []pipeElem
cmds []node
start = -1
)
for {
t := p.peek()
if t == nil {
return nil, errIncomplete
}
if t.kind == tOp && t.text != "(" || t.kind == tWord && terminators[t.text] {
return nil, p.unexpected(t)
}
if start < 0 {
start = t.start
}
cmd, compound, err := p.parseCommand()
if err != nil {
return nil, err
}
elem := pipeElem{text: p.src[t.start:p.toks[p.pos-1].end], compound: compound}
if sub, ok := cmd.(*subshellNode); ok {
elem.subshell = sub.text
}
elems = append(elems, elem)
cmds = append(cmds, cmd)
if t := p.peek(); t == nil || t.kind != tOp || t.text != "|" {
break
}
p.pos++
p.skipNewlines()
if p.peek() == nil {
return nil, errIncomplete
}
}
text := p.src[start:p.toks[p.pos-1].end]
compound := false
for i, e := range elems {
if _, ok := cmds[i].(*funcDefNode); ok && len(elems) > 1 {
return nil, errors.New("function definitions in pipelines are not supported")
}
compound = compound || e.compound
}
switch {
case len(elems) == 1:
pl.cmd = cmds[0]
case !compound:
pl.cmd = &simpleNode{text: text}
default:
pl.cmd = &pipeNode{elems: elems, text: text}
}
return pl, nil
}
// parseSimple parses a pipeline of simple commands and returns its text.
// parseCommand parses a simple command, a compound command with its
// redirections or a function definition, and reports whether it is not a
// simple command.
func (p *parser) parseCommand() (node, bool, error) {
t := p.peek()
start := t.start
if name, ok := p.funcDefName(); ok {
def, err := p.parseFuncDef(name, start)
if err != nil {
return nil, true, err
}
if t := p.peek(); t != nil && t.kind == tWord && !terminators[t.text] {
return nil, true, errors.New("redirections of function definitions are not supported")
}
return def, true, nil
}
if !isCompoundStart(t) {
cmd, err := p.parseSimple()
return cmd, false, err
}
cmd, err := p.parseCompound()
if err != nil {
return nil, true, err
}
redirs, err := p.parseRedirections()
if err != nil {
return nil, true, err
}
if redirs != "" {
cmd = &redirNode{cmd: cmd, redirs: redirs}
}
return cmd, true, nil
}
// isCompoundStart reports whether a compound command starts at t.
func isCompoundStart(t *rawToken) bool {
return t.kind == tWord && compoundStarts[t.text] || t.kind == tOp && t.text == "("
}
// parseRedirections parses the redirections after a compound command and
// returns their source text (expanded when the command runs).
func (p *parser) parseRedirections() (string, error) {
start, end := -1, -1
for t := p.peek(); t != nil && t.kind == tWord && !terminators[t.text]; t = p.peek() {
needsTarget, ok := redirectionWord(t.text)
if !ok {
return "", syntaxError(t.text)
}
if start < 0 {
start = t.start
}
end = t.end
p.pos++
if needsTarget {
t = p.peek()
if t == nil || t.kind != tWord {
if t == nil {
return "", syntaxError("newline")
}
return "", p.unexpected(t)
}
end = t.end
p.pos++
}
}
if start < 0 {
return "", nil
}
return p.src[start:end], nil
}
// redirectionWord reports whether word starts with a redirection operator
// (optionally after a file descriptor), and whether the target is the next
// word (2> file instead of 2>file).
func redirectionWord(word string) (needsTarget, ok bool) {
w := strings.TrimLeft(word, "0123456789")
for _, op := range []string{"&>>", "&>", "<<<", ">>", ">&", "<&", ">", "<"} {
if strings.HasPrefix(w, op) {
if op[0] == '&' && len(w) != len(word) {
return false, false
}
return w == op, true
}
}
return false, false
}
// parseSimple parses a simple command (without pipes) and returns its text.
func (p *parser) parseSimple() (*simpleNode, error) {
start := p.toks[p.pos].start
end := start
for t := p.peek(); t != nil; t = p.peek() {
if t.kind == tWord {
end = t.end
p.pos++
continue
}
if t.text != "|" {
break
}
p.pos++
for t := p.peek(); t != nil && t.kind == tWord; t = p.peek() {
end = t.end
p.skipNewlines()
next := p.peek()
switch {
case next == nil:
return nil, errIncomplete
case next.kind == tOp || terminators[next.text]:
return nil, p.unexpected(next)
case compoundStarts[next.text]:
return nil, errors.New("control structures in pipelines are not supported")
}
p.pos++
}
return &simpleNode{text: p.src[start:end]}, nil
}
@@ -536,6 +663,8 @@ func (p *parser) parseCompound() (node, error) {
return p.parseWhile()
case "case":
return p.parseCase()
case "(":
return p.parseSubshell()
default: // {
return p.parseGroup()
}
@@ -668,6 +797,28 @@ func (p *parser) parseGroup() (*groupNode, error) {
return &groupNode{body: body}, nil
}
// parseSubshell parses ( list ).
func (p *parser) parseSubshell() (*subshellNode, error) {
p.pos++ // (
start := p.toks[p.pos-1].end
body, err := p.parseList(map[string]bool{")": true})
if err == nil {
err = p.nonEmpty(body)
}
if err != nil {
return nil, err
}
t := p.peek()
if t == nil {
return nil, errIncomplete
}
if t.kind != tOp || t.text != ")" {
return nil, p.unexpected(t)
}
p.pos++
return &subshellNode{text: p.src[start:t.start]}, nil
}
// parseCase parses case word in [(]pattern[|pattern]...) list ;; ... esac.
// The last clause may omit the ;;.
func (p *parser) parseCase() (*caseNode, error) {
@@ -713,18 +864,12 @@ func (p *parser) parseCase() (*caseNode, error) {
}
// parsePatterns parses the patterns of a case clause up to and including the
// ). Since ( and ) are not operators for the lexer, they can be part of a
// word token ("a)", "(a", "*)echo"); such tokens are split here.
// ).
func (p *parser) parsePatterns() ([]string, error) {
var patterns []string
if t := p.peek(); t.kind == tWord && strings.HasPrefix(t.text, "(") {
// The optional ( before the first pattern
if t.text == "(" {
p.pos++
} else {
t.text, t.start = t.text[1:], t.start+1
}
if t := p.peek(); t.kind == tOp && t.text == "(" {
p.pos++ // the optional ( before the first pattern
}
var patterns []string
for {
t := p.peek()
if t == nil {
@@ -733,109 +878,48 @@ func (p *parser) parsePatterns() ([]string, error) {
if t.kind != tWord {
return nil, p.unexpected(t)
}
i := unquotedIndex(t.text, ')')
if i < 0 {
patterns = append(patterns, t.text)
p.pos++
next := p.peek()
switch {
case next == nil:
return nil, errIncomplete
case next.kind == tOp && next.text == "|":
p.pos++
case next.kind == tWord && strings.HasPrefix(next.text, ")"):
default:
return nil, p.unexpected(next)
}
continue
}
if i > 0 {
patterns = append(patterns, t.text[:i])
}
if len(patterns) == 0 {
return nil, syntaxError(")")
}
if rest := t.text[i+1:]; rest == "" {
p.pos++
} else {
// The first word of the body follows right after the )
t.text, t.start = rest, t.start+i+1
}
return patterns, nil
}
}
// unquotedIndex returns the index of the first c in word that is not quoted,
// escaped or inside $(...) or ${...}, or -1.
func unquotedIndex(word string, c byte) int {
var quote byte
for i := 0; i < len(word); i++ {
ch := word[i]
patterns = append(patterns, t.text)
p.pos++
t = p.peek()
switch {
case quote == '\'':
if ch == '\'' {
quote = 0
}
case ch == '\\':
i++
case isSubstStart(word, i):
if end := substEnd(word, i); end >= 0 {
i = end
}
case ch == '$' && i+1 < len(word) && word[i+1] == '{':
if k := strings.IndexByte(word[i:], '}'); k >= 0 {
i += k
}
case quote == '"':
if ch == '"' {
quote = 0
}
case ch == '"' || ch == '\'':
quote = ch
case ch == c:
return i
case t == nil:
return nil, errIncomplete
case t.kind == tOp && t.text == "|":
p.pos++
case t.kind == tOp && t.text == ")":
p.pos++
return patterns, nil
default:
return nil, p.unexpected(t)
}
}
return -1
}
// funcDefName reports whether a function definition starts at the current
// token: name(), name () or function name [()]. It consumes the tokens up to
// the body; a body that starts in the same token (name(){) becomes a token of
// its own.
// token: name(), or function name [()]. It consumes the tokens up to the
// body.
func (p *parser) funcDefName() (string, bool) {
start := p.pos
t := p.peek()
keyword := t.kind == tWord && t.text == "function"
if keyword {
if t := p.peek(); t.kind != tWord {
return "", false
} else if t.text == "function" {
p.pos++
if t = p.peek(); t == nil || t.kind != tWord {
if t := p.peek(); t == nil || t.kind != tWord || !isFuncName(t.text) {
p.pos = start
return "", false
}
}
var name, rest string
if n, r, ok := strings.Cut(t.text, "()"); ok && isFuncName(n) {
// name() or name(){
name, rest = n, r
} else if next := p.pos + 1; t.kind == tWord && isFuncName(t.text) && next < len(p.toks) &&
p.toks[next].kind == tWord && strings.HasPrefix(p.toks[next].text, "()") {
// name ()
name = t.text
p.pos = next
t = &p.toks[next]
rest = t.text[2:]
} else if keyword && isFuncName(t.text) {
// function name
name = t.text
} else {
p.pos = start
return "", false
name := p.peek().text
isOp := func(i int, text string) bool {
return i < len(p.toks) && p.toks[i].kind == tOp && p.toks[i].text == text
}
if rest == "" {
p.pos++
} else {
t.text, t.start = rest, t.end-len(rest)
switch {
case isOp(p.pos+1, "(") && isOp(p.pos+2, ")") && isFuncName(name):
p.pos += 3
case p.pos > start:
p.pos++ // function name
default:
return "", false
}
return name, true
}
@@ -848,7 +932,7 @@ func (p *parser) parseFuncDef(name string, start int) (*funcDefNode, error) {
if t == nil {
return nil, errIncomplete
}
if t.kind != tWord || !compoundStarts[t.text] {
if !isCompoundStart(t) {
return nil, fmt.Errorf("%s: the body of a function must be a compound command like { ...; }", sanitize(name))
}
body, err := p.parseCompound()
@@ -901,7 +985,7 @@ func (l *listNode) exec() error {
for _, entry := range l.items {
printError(err)
if entry.background {
err = entry.andOr.first.cmd.(*simpleNode).run(true)
err = entry.andOr.runBackground()
} else {
err = entry.andOr.exec()
}
@@ -929,6 +1013,22 @@ func (a *andOrNode) exec() error {
return err
}
// runBackground starts the and-or list as a background job. Pipelines run
// like in the foreground, everything else in a subshell.
func (a *andOrNode) runBackground() error {
if len(a.rest) == 0 && !a.first.negate {
switch cmd := a.first.cmd.(type) {
case *simpleNode:
return cmd.run(true)
case *pipeNode:
return cmd.run(true)
case *subshellNode:
return runPipeline([]stage{{script: cmd.text}}, a.text, true)
}
}
return runPipeline([]stage{{script: a.text}}, a.text, true)
}
func (pl *pipelineNode) exec() error {
err := pl.cmd.exec()
if !pl.negate || isAbort(err) {
@@ -1066,6 +1166,65 @@ func (n *groupNode) exec() error {
return n.body.exec()
}
func (n *subshellNode) exec() error {
return runPipeline([]stage{{script: n.text}}, "("+strings.TrimSpace(n.text)+")", false)
}
// exec runs the compound command with stdin, stdout and stderr of the shell
// redirected.
func (n *redirNode) exec() error {
fds, opened, err := openRedirections(n.redirs)
if err != nil {
return err
}
saved := ioOverride
ioOverride = fds
defer func() {
ioOverride = saved
for _, f := range opened {
f.Close()
}
}()
return n.cmd.exec()
}
func (n *pipeNode) exec() error {
return n.run(false)
}
// run builds the stages of the pipeline: simple commands are expanded,
// compound commands run in subshells.
func (n *pipeNode) run(background bool) error {
substStatus = 0
var stages []stage
for _, e := range n.elems {
switch {
case e.subshell != "":
stages = append(stages, stage{script: e.subshell})
case e.compound:
stages = append(stages, stage{script: e.text})
default:
text := e.text
if interactiveShell && aliasExpansion {
if tree, err := parse(expandAliases(text)); err == nil {
if simple := tree.singleSimple(); simple != nil {
text = simple.text
}
}
}
st, err := expandStages(text)
if err != nil {
return err
}
if len(st) == 0 {
st = []stage{{}}
}
stages = append(stages, st...)
}
}
return runPipeline(stages, n.text, background)
}
func (n *funcDefNode) exec() error {
defineFunction(n.def)
return nil