From f66cf538c0565f4f051ea85c16ac9836b8d0be76 Mon Sep 17 00:00:00 2001 From: Florian Walther Date: Sun, 4 Oct 2026 20:57:15 +0200 Subject: [PATCH] add subshells, here-documents and redirections/pipes for control structures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - control structures can have redirections (while read l; do ...; done < file), which apply to all commands in them; they run in the shell - in pipes and in the background they run in a subshell (cmd | while read l; do ...; done, for ...; done &, a && b &) - ( list ) subshells; ( and ) are operators now - here-documents < --- README.md | 99 ++++++++- compound_test.go | 274 ++++++++++++++++++++++++ heredoc.go | 268 ++++++++++++++++++++++++ main.go | 169 +++++++++++---- main_test.go | 54 ++++- packaging/man/gosh.1 | 42 +++- parser.go | 487 ++++++++++++++++++++++++++++--------------- 7 files changed, 1172 insertions(+), 221 deletions(-) create mode 100644 compound_test.go create mode 100644 heredoc.go diff --git a/README.md b/README.md index ff662a8..a2bed87 100644 --- a/README.md +++ b/README.md @@ -67,12 +67,16 @@ Features: a || b (b only if a failed); $? is the exit status of the last command - Control structures: if cmd; then ...; elif cmd; then ...; else ...; fi, for x in words; do ...; done, while cmd; do ...; done, until ..., - case word in pattern|pattern) ...;; esac, { cmd; cmd; } groups; - ! cmd negates the status; they can span several lines (prompt "> ") + case word in pattern|pattern) ...;; esac, { cmd; cmd; } groups, + ( cmd; cmd ) subshells; ! cmd negates the status; they can span + several lines (prompt "> "), have redirections (done < file) and can + be used in pipes (| while read l; do ...; done) and with & - Functions: name() { ...; } (or function name { ...; }); the arguments are $1, $2, ...; in pipes, & and $(...) they run in a subshell - Pipes (a | b) and redirections: < file, > file, >> file, 2> file, - 2>> file, 2>&1, >&2, &> file (stdout and stderr), &>> file + 2>> file, 2>&1, >&2, &> file (stdout and stderr), &>> file, + < `; Ctrl+C verwirft die angefangene Eingabe. Skripte und Startdateien lesen entsprechend weiter. - Ctrl+C bricht die ganze Befehlszeile ab, also auch eine Schleife (`while true; do sleep 1; done`). - Der Exit-Status ist der des zuletzt ausgeführten Befehls; ein `if` ohne passenden Zweig und eine Schleife ohne Durchlauf haben den Status 0. -- Nicht unterstützt: Kontrollstrukturen in Pipes (`for …; done | sort`, `… | while read line`), mit Umleitung (`done > datei`, `done < datei`) oder im Hintergrund (`done &`), außerdem `( … )`-Subshells. Funktionen lassen sich dagegen in Pipes und mit Umleitungen verwenden (siehe unten), damit lässt sich das oft umgehen. Arithmetik geht z. B. mit `n=$(expr $n + 1)`. +- Arithmetik geht noch nicht mit `$((…))`, sondern z. B. mit `n=$(expr $n + 1)`. + +### Umleitungen, Pipes und Hintergrund bei Kontrollstrukturen + +Kontrollstrukturen lassen sich wie einfache Befehle umleiten, in Pipes verwenden und im Hintergrund starten: + +```bash +# Datei Zeile für Zeile lesen +while read -r name rest; do + echo "Name: $name" +done < /etc/passwd + +# Ausgabe einer Schleife sortieren bzw. umleiten +for f in *.log; do wc -l < "$f"; done | sort -n +if make; then echo ok; else echo kaputt; fi > build.status 2>&1 +{ date; uptime; } >> status.log + +# Zeilen aus einer Pipe verarbeiten +git ls-files | while read -r f; do echo "== $f"; done + +# Schleife im Hintergrund +for host in a b c; do ping -c1 $host; done > ping.log & +``` + +- Umleitungen nach `done`, `fi`, `esac`, `}` und `)` gelten für alle Befehle darin. Die Kontrollstruktur läuft dabei in der Shell selbst, Variablen aus `while read x; do …; done < datei` sind danach also gesetzt. +- In einer Pipe und im Hintergrund (`&`) läuft eine Kontrollstruktur (wie in `sh`) in einer Subshell, einem eigenen gosh-Prozess, weil alle Teile einer Pipe gleichzeitig laufen. Variablen, die dort gesetzt werden, gelten danach nicht mehr: Nach `ls | while read f; do n=$f; done` ist `n` nicht gesetzt. Die Subshell kennt die Funktionen und Positionsparameter der Shell. +- Auch Listen lassen sich in den Hintergrund schicken: `make && make install &` läuft als ein Job in einer Subshell. +- Mit Ctrl+C wird die ganze Pipe abgebrochen, auch eine Endlosschleife wie `while true; do date; sleep 1; done | tee log`. + +### Subshells + +`( befehle )` führt Befehle in einer Subshell aus, einem eigenen gosh-Prozess. Änderungen an Verzeichnis, Variablen und Funktionen gelten nur dort: + +```bash +(cd /tmp && tar xzf ~/paket.tar.gz) # danach ist man noch im alten Verzeichnis +(export LANG=C; sort datei) +(exit 3); echo $? # 3 +(cd build && make) > build.log 2>&1 +``` + +- `exit` beendet nur die Subshell, ihr Exit-Status ist der des letzten Befehls bzw. von `exit`. +- Die Subshell bekommt alle Variablen (in gosh sind alle exportiert), die Funktionen und die Positionsparameter. Aliase gelten dort nicht (wie in Skripten). +- `{ …; }` fasst dagegen Befehle zusammen, ohne eine Subshell zu starten. +- `(` und `)` sind Operatoren: Ein Wort wie `a(b)` ist ein Syntaxfehler, Klammern als Text brauchen Anführungszeichen oder `\` (`echo "(a)"`, `find . \( -name x \)`). ### Funktionen @@ -229,11 +277,12 @@ ist_root() { } ist_root || echo "kein root" -# Zeilen einer Datei verarbeiten: die Funktion liest von stdin +# Die Funktion liest von stdin zeilen() { while read -r zeile; do echo "> $zeile"; done } zeilen < /etc/hostname +ls | zeilen ``` - Eine Funktion wird mit `name() { …; }`, `name () { …; }` oder `function name { …; }` definiert. Der Rumpf ist ein zusammengesetzter Befehl, meist `{ …; }`, aber auch `if`, `for`, `while` oder `case` sind möglich. Die Definition darf über mehrere Zeilen gehen. @@ -295,7 +344,7 @@ while read -r zeile; do echo "$zeile"; done # in einem Skript: alle Zeilen von - Ohne `-r` maskiert `\` das nächste Zeichen (`a\ b` bleibt ein Feld), und ein `\` am Zeilenende setzt die Zeile fort. `-p prompt` gibt den Prompt auf stderr aus, wenn stdin ein Terminal ist. - Der Status ist 0, oder 1 am Ende der Eingabe (die Variablen werden trotzdem mit dem Rest gesetzt). So endet `while read …` nach der letzten Zeile. - `read` liest Byte für Byte und damit nie mehr als eine Zeile, den Rest bekommen die nächsten Befehle. Ctrl+C bricht `read` in der interaktiven Shell ab (Status 130). -- `read` verändert die Shell und lässt sich deshalb nicht in einer Pipeline verwenden (`echo a | read x`, in `sh` wirkungslos). Umleitungen funktionieren: `read x < datei`. +- `read` verändert die Shell und lässt sich deshalb nicht als einfacher Befehl in einer Pipeline verwenden (`echo a | read x`, in `sh` wirkungslos). Umleitungen funktionieren: `read x < datei`, `read x <<< "$text"`, `while read l; do …; done < datei`. In einer Schleife in einer Pipe (`… | while read l; do …; done`) geht es auch, da die ganze Schleife in einer Subshell läuft. ### Variablen @@ -435,13 +484,47 @@ $ grep -c error < build.log | `2>&1` | stderr dorthin, wohin stdout gerade geht | | `>&2` | stdout dorthin, wohin stderr gerade geht | | `&> datei` | stdout und stderr in `datei` (`&>>` hängt an) | +| `< datei` | ohne Befehl: legt `datei` an bzw. leert sie | - Umleitungen werden von links nach rechts ausgewertet: `befehl > log 2>&1` schreibt beides in `log`, `befehl 2>&1 > log` schickt stderr dorthin, wo stdout vorher hinging (z. B. in eine Pipe). - Unterstützt werden die Dateideskriptoren 0, 1 und 2. Vor `>` und `<` zählt nur eine Zahl ohne Anführungszeichen als Deskriptor (`2>f`, aber nicht `x2>f` oder `"2">f`). - Operatoren in Anführungszeichen oder mit `\` davor sind normale Zeichen: `echo "a|b" 'c>d' e\>f`. - Der Exit-Status einer Pipeline ist der des letzten Befehls. Beendet sich ein Befehl am Ende (z. B. `head`), bekommt der schreibende Befehl davor `SIGPIPE`, `yes | head -1` endet also sofort. -- Die Built-ins `pwd` und `help` lassen sich umleiten und in Pipes verwenden (`help | less`). `cd`, `set`, `export`, `exit` und `quit` verändern die Shell selbst und sind in einer Pipeline nicht erlaubt. +- Die Built-ins `pwd` und `help` lassen sich umleiten und in Pipes verwenden (`help | less`). `cd`, `set`, `export`, `exit` und `quit` verändern die Shell selbst und sind in einer Pipeline nicht erlaubt. Umleitungen von `source datei > log` gelten für alle Befehle der Datei. +- Zu Kontrollstrukturen mit Umleitungen und in Pipes siehe oben. + +### Here-Docs und Here-Strings + +```bash +cat < config.ini +[server] +host = $HOSTNAME +started = $(date +%F) +EOF + +cat <<'EOF' > script.sh # 'EOF': nichts wird ersetzt +echo "$HOME bleibt wörtlich stehen" +EOF + +if true; then + cat <<-EOF # <<-: Tabs am Zeilenanfang werden entfernt + eingerückt + EOF +fi + +grep -c x <<< "$text" # Here-String +read a b <<< "1 2" +``` + +- Nach `< `. +- Ohne Anführungszeichen am Wort werden `$VAR`, `${VAR}`, `$(…)` und `\` vor `$`, `` ` ``, `\` und dem Zeilenende ersetzt (wie in `"…"`, nur dass `"` ein normales Zeichen ist). Wildcards, `~` und `{a,b}` werden nicht ersetzt. Ist das Wort ganz oder teilweise in Anführungszeichen (`'EOF'`, `"EOF"`, `\EOF`), wird nichts ersetzt. +- `<<-WORT` entfernt Tabs (keine Leerzeichen) am Anfang jeder Zeile und der Endzeile, damit sich Here-Docs in Skripten einrücken lassen. +- Mehrere Here-Docs in einer Zeile (`cat < out", "yes\n"}, + {"for x in a b; do echo $x; done > out", "a\nb\n"}, + {"for x in a b; do echo $x; done >> out; echo c >> out", "a\nb\nc\n"}, + {"{ echo out; echo err >&2; } > out 2>&1", "out\nerr\n"}, + {"{ echo err >&2; } 2> out", "err\n"}, + {"case x in x) echo case;; esac > out", "case\n"}, + // The redirections apply to all commands, also nested ones + {"{ echo a; { echo b; }; } > out", "a\nb\n"}, + {"{ echo a; echo b > other; echo c; } > out", "a\nc\n"}, + {"f() { echo from-f; }; { f; } > out", "from-f\n"}, + // Reading a file line by line in the shell itself + {"printf 'l1\\nl2 x\\n' > in; while read -r l; do echo \"<$l>\" >> out; done < in", "\n\n"}, + {"printf 'a b\\nc d\\n' > in; while read x y; do echo $y$x >> out; done < in", "ba\ndc\n"}, + {"printf '1\\n2\\n3\\n' > in; GOSH_N=0; while read l; do GOSH_N=$(expr $GOSH_N + $l); done < in; echo $GOSH_N > out", "6\n"}, + // ... with the loop's variables changed in the shell + {"printf 'v\\n' > in; while read GOSH_A; do :; done < in; echo ${GOSH_A:-empty} > out", "empty\n"}, + {"printf 'v\\n' > in; { read GOSH_A; } < in; echo $GOSH_A > out", "v\n"}, + // Redirections are expanded + {"GOSH_A=out; { echo var; } > $GOSH_A", "var\n"}, + // Files are closed and stdout is restored afterwards + {"{ echo a; } > out; echo b >> out", "a\nb\n"}, + } + for _, tc := range tests { + if got := runIn(t, tc.input); got != tc.want { + t.Errorf("%q: out = %q, want %q", tc.input, got, tc.want) + } + if ioOverride != [3]*os.File{} { + t.Fatalf("%q: ioOverride not restored", tc.input) + } + } + + for input, want := range map[string]string{ + "{ :; } < gosh-no-such-file": "gosh-no-such-file: no such file or directory", + "{ :; } > {a,b}": "{a,b}: ambiguous redirect", + } { + if err := runInErr(t, input); err == nil || err.Error() != want { + t.Errorf("%q: err = %v, want %q", input, err, want) + } + } +} + +// runInErr runs input in a temporary directory and returns its error. +func runInErr(t *testing.T, input string) error { + t.Helper() + origDir, _ := os.Getwd() + defer os.Chdir(origDir) + os.Chdir(t.TempDir()) + return execInput(input) +} + +func TestExec_CompoundPipelines(t *testing.T) { + resetShell(t) + os.Unsetenv("GOSH_A") + defer os.Unsetenv("GOSH_A") + tests := []struct { + input string + want string + }{ + {"for x in 3 1 2; do echo $x; done | sort > out", "1\n2\n3\n"}, + {"printf 'a b\\nc d\\n' | while read x y; do echo $y$x; done > out", "ba\ndc\n"}, + {"echo hi | { read l; echo got $l; } > out", "got hi\n"}, + {"if true; then echo yes; fi | tr a-z A-Z > out", "YES\n"}, + {"echo x | case y in y) cat;; esac | tr x X > out", "X\n"}, + {"for x in a b; do echo $x; done > /dev/null | cat > out", ""}, + {"{ echo b; echo a; } | sort | head -1 > out", "a\n"}, + // The parts of a pipeline run in subshells: the shell is not changed + {"echo v | while read GOSH_A; do :; done; echo ${GOSH_A:-unset} > out", "unset\n"}, + // Functions and positional parameters are known in the subshell + {"f() { echo f-$1; }; set -- p; for x in 1; do f $1; done | cat > out", "f-p\n"}, + // The status is the one of the last command + {"echo | while read l; do false; done; echo $? > out", "1\n"}, + {"true | if false; then :; else sh -c 'exit 3'; fi; echo $? > out", "3\n"}, + {"for x in 1; do sh -c 'exit 4'; done | true; echo $? > out", "0\n"}, + {"! echo | while read l; do false; done; echo $? > out", "0\n"}, + } + for _, tc := range tests { + if got := runIn(t, tc.input); got != tc.want { + t.Errorf("%q: out = %q, want %q", tc.input, got, tc.want) + } + } +} + +func TestExec_Subshells(t *testing.T) { + resetShell(t) + os.Unsetenv("GOSH_A") + defer os.Unsetenv("GOSH_A") + tests := []struct { + input string + want string + }{ + {"(echo a; echo b) > out", "a\nb\n"}, + {"(echo sub) | tr a-z A-Z > out", "SUB\n"}, + // Changes stay in the subshell + {"mkdir d; (cd d; pwd > ../in); cd d 2>/dev/null && echo still-outside > out", "still-outside\n"}, + {"GOSH_A=1; (GOSH_A=2; echo $GOSH_A > out); echo $GOSH_A >> out", "2\n1\n"}, + {"(exit 3); echo $? > out", "3\n"}, + {"(exit 3) || echo failed > out", "failed\n"}, + {"(f() { echo in-sub; }; f) > out; f 2>> out || echo no-f >> out", "in-sub\nno-f\n"}, + {"(echo a; (echo b)) > out", "a\nb\n"}, + {"( echo spaced ) > out", "spaced\n"}, + {"(\n echo multi\n echo line\n) > out", "multi\nline\n"}, + // The subshell knows the functions and positional parameters + {"f() { echo f-$1; }; set -- p; (f $1) > out", "f-p\n"}, + } + for _, tc := range tests { + if got := runIn(t, tc.input); got != tc.want { + t.Errorf("%q: out = %q, want %q", tc.input, got, tc.want) + } + } +} + +func TestExec_CompoundBackground(t *testing.T) { + resetShell(t) + tests := []struct { + input string + want string + }{ + {"for x in a b; do echo $x; done > out & wait", "a\nb\n"}, + {"true && echo and > out & wait", "and\n"}, + {"false || echo or > out & wait", "or\n"}, + {"(echo sub > out) & wait", "sub\n"}, + {"{ echo group; } > out & wait", "group\n"}, + {"! false && echo negated > out & wait", "negated\n"}, + {"for x in 3 1; do echo $x; done | sort > out & wait", "1\n3\n"}, + } + for _, tc := range tests { + if got := runIn(t, tc.input); got != tc.want { + t.Errorf("%q: out = %q, want %q", tc.input, got, tc.want) + } + } + // The shell does not wait for a background loop + start := time.Now() + runIn(t, "for x in 1; do sleep 1; done & echo $? > out") + if lastStatus != 0 || time.Since(start) > 500*time.Millisecond { + t.Errorf("background loop: $? = %d after %v", lastStatus, time.Since(start)) + } + execInput("wait") +} + +func TestExtractHeredocs(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"cat < out\nhello\nworld\nEOF", "hello\nworld\n"}, + {"cat < out\n$GOSH_A ${GOSH_A} $(echo sub) \\$GOSH_A\nEOF", "value value sub $GOSH_A\n"}, + {"cat <<'EOF' > out\n$GOSH_A $(echo sub)\nEOF", "$GOSH_A $(echo sub)\n"}, + {"cat < out\n\"quotes\" 'single' back\\slash\nEOF", "\"quotes\" 'single' back\\slash\n"}, + {"cat < out\nline \\\ncontinued\nEOF", "line continued\n"}, + {"cat <<'EOF' > out\nline \\\nkept\nEOF", "line \\\nkept\n"}, + {"cat <<-EOF > out\n\tindented\n\t\tmore\n\tEOF", "indented\nmore\n"}, + {"cat < out\n *.go {a,b} ~\nEOF", " *.go {a,b} ~\n"}, + {"cat < out\nEOF", ""}, + {"cat < out\npiped\nEOF", "PIPED\n"}, + {"cat > out < out; cat <> out\na\nA\nb\nB", "a\nb\n"}, + {"while read l; do echo \"<$l>\" >> out; done <\n\n"}, + {"f() { cat; }; f < out\nto f\nEOF", "to f\n"}, + {"set -- a 'b c'; cat < out\n$# $1 $@\nEOF", "2 a a b c\n"}, + {"for x in 1 2; do\n cat <> out\nx=$x\nEOF\ndone", "x=1\nx=2\n"}, + {"echo \"$(cat < out", "in subst\n"}, + // Here-strings + {"cat << out", "word\n"}, + {"cat <<< \"two words $GOSH_A\" > out", "two words value\n"}, + {"cat <<<'*.go' > out", "*.go\n"}, + {"read x y <<<'a b c'; echo $y > out", "b c\n"}, + } + for _, tc := range tests { + if got := runIn(t, tc.input); got != tc.want { + t.Errorf("%q: out = %q, want %q", tc.input, got, tc.want) + } + } +} + +// TestRunScript_Heredoc checks that a script reads a here-document and +// continues after it, with correct line numbers in error messages. +func TestRunScript_Heredoc(t *testing.T) { + resetShell(t) + got := runIn(t, `printf 'cat < out\nbody\nEOF\necho after >> out\n' > s; . ./s`) + if got != "body\nafter\n" { + t.Errorf("script with here-document: out = %q", got) + } + got = runIn(t, `printf 'cat < s; . ./s 2> out`) + if !strings.Contains(got, "s:1: syntax error: unexpected end of file") { + t.Errorf("unterminated here-document: %q", got) + } +} + +// TestTokenize_UTF8 checks that non-ASCII characters survive quoting and +// escaping (each byte used to be converted to a rune of its own). +func TestTokenize_UTF8(t *testing.T) { + got, err := splitWords(`echo 'wörtlich' "für" über \ä 'a'"ö"`) + want := []string{"echo", "wörtlich", "für", "über", "ä", "aö"} + if err != nil || !reflect.DeepEqual(got, want) { + t.Errorf("got %q, %v, want %q", got, err, want) + } + if got := runIn(t, "cat <<'EOF' > out\nGrüße\nEOF"); got != "Grüße\n" { + t.Errorf("here-document: %q", got) + } +} diff --git a/heredoc.go b/heredoc.go new file mode 100644 index 0000000..0c5b123 --- /dev/null +++ b/heredoc.go @@ -0,0 +1,268 @@ +package main + +// Here-documents: cmd < len(src) { + end = len(src) + } + cur.WriteString(src[i:end]) + return end + } + + for i := 0; i < len(src); { + ch := src[i] + switch { + case ch == '\\': + i = copyTo(i, i+2) + wordStart = false + case ch == '\'': + end := strings.IndexByte(src[i+1:], '\'') + if end >= 0 { + end += i + 2 + } + i = copyTo(i, end) + wordStart = false + case ch == '"': + end := dquoteEnd(src, i) + if end >= 0 { + end++ + } + i = copyTo(i, end) + wordStart = false + case isSubstStart(src, i): + end := substEnd(src, i) + if end >= 0 { + end++ + } + i = copyTo(i, end) + wordStart = false + case ch == '#' && wordStart: + // A comment, until the end of the line + end := strings.IndexByte(src[i:], '\n') + if end >= 0 { + end += i + } + i = copyTo(i, end) + case strings.HasPrefix(src[i:], "<<<"): + i = copyTo(i, i+3) + wordStart = false + case strings.HasPrefix(src[i:], "<<"): + h, n, err := parseHereOp(src[i:]) + if err != nil { + return "", err + } + segs = append(segs, cur.String()) + cur.Reset() + h.seg = len(segs) + segs = append(segs, "") // filled in when the body has been read + pending = append(pending, h) + i += n + wordStart = false + case ch == '\n': + cur.WriteByte(ch) + i++ + wordStart = true + // The bodies of the here-documents of this line follow + for _, h := range pending { + body, n, ok := readHereBody(src[i:], h) + if !ok { + return "", errIncomplete + } + segs[h.seg] = hereString(body, h.quoted) + i += n + } + pending = nil + default: + cur.WriteByte(ch) + i++ + wordStart = strings.IndexByte(" \t;&|()<>", ch) >= 0 + } + } + if len(pending) > 0 { + return "", errIncomplete + } + segs = append(segs, cur.String()) + return strings.Join(segs, ""), nil +} + +// dquoteEnd returns the index of the " that ends the double-quoted string +// starting at s[i], or -1. +func dquoteEnd(s string, i int) int { + for j := i + 1; j < len(s); j++ { + switch { + case s[j] == '\\': + j++ + case isSubstStart(s, j): + end := substEnd(s, j) + if end < 0 { + return -1 + } + j = end + case s[j] == '"': + return j + } + } + return -1 +} + +// parseHereOp parses a here-document operator <", s[i]) < 0 { + switch c := s[i]; c { + case '\'', '"': + end := strings.IndexByte(s[i+1:], c) + if end < 0 { + return h, 0, errIncomplete + } + delim.WriteString(s[i+1 : i+1+end]) + i += end + 2 + h.quoted = true + case '\\': + if i+1 < len(s) { + delim.WriteByte(s[i+1]) + } + i += 2 + h.quoted = true + default: + delim.WriteByte(c) + i++ + } + } + if i == start { + if i == len(s) || s[i] == '\n' { + return h, 0, syntaxError("newline") + } + return h, 0, syntaxError(s[i : i+1]) + } + h.delim = delim.String() + return h, min(i, len(s)), nil +} + +// readHereBody reads the lines of a here-document from s up to the +// delimiter line. It returns the body (each line with its newline) and the +// number of bytes consumed, or ok == false if there is no delimiter line yet. +func readHereBody(s string, h hereDoc) (body string, n int, ok bool) { + var b strings.Builder + for pos := 0; ; { + end := strings.IndexByte(s[pos:], '\n') + last := end < 0 + if last { + end = len(s) - pos + } + line := s[pos : pos+end] + if h.stripTabs { + line = strings.TrimLeft(line, "\t") + } + next := min(pos+end+1, len(s)) + if line == h.delim { + return b.String(), next, true + } + if last { + return "", 0, false + } + b.WriteString(line) + b.WriteByte('\n') + pos = next + } +} + +// hereString returns the here-string (or redirection) that has the effect of +// a here-document with the given body. +func hereString(body string, quoted bool) string { + if body == "" { + return "= 0: + b.WriteString(body[i : i+2]) + i++ + case c == '\\': + b.WriteString(`\\`) + case c == '"': + b.WriteString(`\"`) + default: + b.WriteByte(c) + } + } + return b.String() +} diff --git a/main.go b/main.go index 0d7d074..fc4b4f8 100644 --- a/main.go +++ b/main.go @@ -191,14 +191,14 @@ func tokenize(input string) ([]token, error) { if quote == '"' && !strings.ContainsRune("$\"\\`", rune(ch)) { literal("\\") } - literal(string(ch)) + literal(input[i : i+1]) inWord = true escaped = false case quote == '\'': if ch == '\'' { quote = 0 } else { - literal(string(ch)) + literal(input[i : i+1]) } case ch == '\\': startRaw(i) @@ -263,7 +263,7 @@ func tokenize(input string) ([]token, error) { if ch == '"' { quote = 0 } else { - literal(string(ch)) + literal(input[i : i+1]) } case ch == '"' || ch == '\'': startRaw(i) @@ -306,6 +306,10 @@ func tokenize(input string) ([]token, error) { } t := token{op: string(ch), fd: fd} switch { + case strings.HasPrefix(input[i:], "<<<"): + // <<' && i+1 < len(input) && input[i+1] == '>': t.op = ">>" i++ @@ -1418,27 +1422,8 @@ func execInput(input string) error { // execCommand executes a pipeline of simple commands with redirections, in // the background if background is set. func execCommand(input string, background bool) error { - // Brace expansion comes first, so that its results are expanded further substStatus = 0 - input, err := expandBraceWords(input) - if err != nil { - return err - } - - // Split the input into words and operators, expanding variables and ~ on - // the way. - tokens, err := tokenize(input) - var ( - synErr syntaxError - statusErr statusError - ) - if errors.As(err, &synErr) || errors.As(err, &statusErr) { - return err - } - if err != nil { - return fmt.Errorf("invalid input: %w", err) - } - stages, err := parsePipeline(tokens) + stages, err := expandStages(input) if err != nil { return err } @@ -1456,19 +1441,62 @@ func execCommand(input string, background bool) error { return runPipeline(stages, strings.TrimSpace(input), background) } -// redirect is a redirection of a simple command, e.g. 2>>file or 2>&1. -type redirect struct { - op string // <, >, >>, &>, &>>, >& or <& - fd int // the redirected file descriptor - dup int // the target file descriptor of >& and <& - file string // the file name for all other operators +// expandStages expands the text of a pipeline of simple commands and splits +// it into its stages. +func expandStages(input string) ([]stage, error) { + // Brace expansion comes first, so that its results are expanded further + input, err := expandBraceWords(input) + if err != nil { + return nil, err + } + + // Split the input into words and operators, expanding variables and ~ on + // the way. + tokens, err := tokenize(input) + var ( + synErr syntaxError + statusErr statusError + ) + if errors.As(err, &synErr) || errors.As(err, &statusErr) { + return nil, err + } + if err != nil { + return nil, fmt.Errorf("invalid input: %w", err) + } + return parsePipeline(tokens) } -// stage is a simple command of a pipeline. +// openRedirections expands and opens the redirections of a compound command +// and returns the resulting stdin, stdout and stderr, and the files to close +// afterwards. +func openRedirections(text string) ([3]*os.File, []*os.File, error) { + fds := [3]*os.File{stdio(0), stdio(1), stdio(2)} + stages, err := expandStages(text) + if err != nil { + return fds, nil, err + } + if len(stages) != 1 || len(stages[0].args) > 0 || len(stages[0].assigns) > 0 { + return fds, nil, syntaxError(text) + } + opened, err := applyRedirects(&fds, stages[0].redirects) + return fds, opened, err +} + +// redirect is a redirection of a simple command, e.g. 2>>file or 2>&1. +type redirect struct { + op string // <, >, >>, &>, &>>, >&, <& or <<< + fd int // the redirected file descriptor + dup int // the target file descriptor of >& and <& + file string // the file name for all other operators (the text for <<<) +} + +// stage is a simple command of a pipeline, or a script that runs in a +// subshell (a compound command in a pipeline or in the background). type stage struct { assigns []string // name=value before the command name args []string redirects []redirect + script string } // parsePipeline groups tokens into the stages of a pipeline. @@ -1505,6 +1533,11 @@ func parsePipeline(tokens []token) ([]stage, error) { return nil, syntaxError(next.op) } i++ + if t.op == "<<<" { + // A here-string is not expanded as a pattern + cur.redirects = append(cur.redirects, redirect{op: t.op, fd: t.fd, file: tokens[i].word}) + continue + } files := expandWord(tokens[i]) if len(files) > 1 { return nil, fmt.Errorf("%s: ambiguous redirect", sanitize(tokens[i].word)) @@ -1621,18 +1654,22 @@ func startStage(st stage, stdin, stdout *os.File, j *job, inPipeline, background err = statusError(substStatus) } } - if err != nil || len(st.args) == 0 { + if err != nil || len(st.args) == 0 && st.script == "" { // A command without arguments only creates or truncates the files closeOwned() return finishedProc(err) } - fn, isFunc := functions[st.args[0]] + var fn *funcDef + isFunc := false + if st.script == "" { + fn, isFunc = functions[st.args[0]] + } if isFunc && !inPipeline && !background { defer closeOwned() return finishedProc(callFunction(fn, st.args, fds, st.assigns)) } - if builtin, ok := builtins[st.args[0]]; !isFunc && ok { + if builtin, ok := builtins[firstArg(st.args)]; !isFunc && st.script == "" && ok { switch { case inPipeline && builtin.changesState: closeOwned() @@ -1641,10 +1678,11 @@ func startStage(st stage, stdin, stdout *os.File, j *job, inPipeline, background closeOwned() return finishedProc(fmt.Errorf("%s: cannot be run in the background", st.args[0])) } + shellStderr := stdio(2) run := func() error { defer closeOwned() err := runBuiltin(st.args, fds[0], fds[1]) - if err != nil && fds[2] != stdio(2) { + if err != nil && fds[2] != shellStderr { // Report the error where stderr is redirected to fmt.Fprintln(fds[2], "gosh:", err) return statusError(1) @@ -1652,8 +1690,12 @@ func startStage(st stage, stdin, stdout *os.File, j *job, inPipeline, background return err } if !inPipeline && !background { - // name=value before a builtin applies only while it runs + // name=value before a builtin applies only while it runs, its + // redirections also to the commands it runs (source file > out) defer setTemporarily(st.assigns)() + saved := ioOverride + ioOverride = fds + defer func() { ioOverride = saved }() return finishedProc(run()) } p := &proc{done: make(chan error, 1)} @@ -1662,7 +1704,13 @@ func startStage(st stage, stdin, stdout *os.File, j *job, inPipeline, background } var cmd *exec.Cmd - if isFunc { + if st.script != "" { + cmd, err = subshellCommand(st.script, positional) + if err != nil { + closeOwned() + return finishedProc(err) + } + } else if isFunc { // A function in a pipeline or in the background runs in a subshell cmd, err = subshellCommand(`"$@"`, st.args) if err != nil { @@ -1695,7 +1743,7 @@ func startStage(st stage, stdin, stdout *os.File, j *job, inPipeline, background // The child has its own copies of the files now closeOwned() if errors.Is(err, exec.ErrNotFound) { - err = fmt.Errorf("%s: %w", sanitize(st.args[0]), errCommandNotFound) + err = fmt.Errorf("%s: %w", sanitize(firstArg(st.args)), errCommandNotFound) } if err != nil { return finishedProc(err) @@ -1710,6 +1758,13 @@ func startStage(st stage, stdin, stdout *os.File, j *job, inPipeline, background return &proc{cmd: cmd} } +func firstArg(args []string) string { + if len(args) == 0 { + return "" + } + return args[0] +} + // setTemporarily sets the variables of the assignments and returns a // function that restores the previous values. func setTemporarily(assigns []string) func() { @@ -1758,6 +1813,16 @@ func applyRedirects(fds *[3]*os.File, redirects []redirect) ([]*os.File, error) continue } + if r.op == "<<<" { + f, err := hereFile(r.file + "\n") + if err != nil { + return fail(err) + } + opened = append(opened, f) + fds[r.fd] = f + continue + } + flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC switch r.op { case "<": @@ -1783,6 +1848,26 @@ func applyRedirects(fds *[3]*os.File, redirects []redirect) ([]*os.File, error) return opened, nil } +// hereFile returns a file to read content from: a temporary file that is +// already removed (like bash does for here-documents), so that large inputs +// work and nothing is left behind. +func hereFile(content string) (*os.File, error) { + f, err := os.CreateTemp("", "gosh-here-*") + if err != nil { + return nil, err + } + os.Remove(f.Name()) + if _, err := f.WriteString(content); err != nil { + f.Close() + return nil, err + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + f.Close() + return nil, err + } + return f, nil +} + // builtins lists the built-in commands. Commands that change the state of // the shell cannot be used in a pipeline, since there they would run // concurrently (sh runs them in a subshell where they have no effect). @@ -2021,12 +2106,16 @@ Features: a || b (b only if a failed); $? is the exit status of the last command - Control structures: if cmd; then ...; elif cmd; then ...; else ...; fi, for x in words; do ...; done, while cmd; do ...; done, until ..., - case word in pattern|pattern) ...;; esac, { cmd; cmd; } groups; - ! cmd negates the status; they can span several lines (prompt "> ") + case word in pattern|pattern) ...;; esac, { cmd; cmd; } groups, + ( cmd; cmd ) subshells; ! cmd negates the status; they can span + several lines (prompt "> "), have redirections (done < file) and can + be used in pipes (| while read l; do ...; done) and with & - Functions: name() { ...; } (or function name { ...; }); the arguments are $1, $2, ...; in pipes, & and $(...) they run in a subshell - Pipes (a | b) and redirections: < file, > file, >> file, 2> file, - 2>> file, 2>&1, >&2, &> file (stdout and stderr), &>> file + 2>> file, 2>&1, >&2, &> file (stdout and stderr), &>> file, + <" + case *pipeNode: + var elems []string + for _, e := range n.elems { + elems = append(elems, "<"+e.text+">") + } + s += strings.Join(elems, " | ") + case *funcDefNode: + s += n.def.name + "() " + pipelineString(&pipelineNode{cmd: n.def.body}) } return s } @@ -1504,6 +1524,27 @@ func TestParse_ControlStructures(t *testing.T) { {"while a; do if b; then break; fi; done", "while [a] do if [b] then [break] fi done"}, {"for i in 1 2; do for j in 3 4; do echo $i$j; done; done", "for i in (1 2) do for j in (3 4) do [echo $i$j] done done"}, {"if a; then if b; then c; fi fi", "if [a] then if [b] then [c] fi fi"}, + // Compound commands in pipelines, with redirections, in the background + {"for x in a; do b; done | cat", " | "}, + {"a | while b; do c; done | d", " | | "}, + {"if a; then b; fi > out 2>&1", "if [a] then [b] fi <> out 2>&1>"}, + {"while read l; do b; done < in", "while [read l] do [b] done << in>"}, + {"{ a; b; } 2> err", "{ [a]; [b] } <2> err>"}, + {"for x in a; do b; done &", "for x in (a) do [b] done &"}, + {"a && b &", "[a] && [b] &"}, + // Subshells + {"(a; b)", "(a; b)"}, + {"( a ) | b", "<( a )> | "}, + {"(a) > f", "(a) <> f>"}, + {"(a; (b)) && c", "(a; (b)) && [c]"}, + // case, groups and functions + {"case $x in a|b) c;; (d) e; f;; esac", "case $x in (a|b) [c] ;; (d) [e]; [f] ;; esac"}, + {"case x in\n a)\n b\n ;;\nesac", "case x in (a) [b] ;; esac"}, + {"{ a; }", "{ [a] }"}, + {"f() { a; }", "f() { [a] }"}, + {"f ( ) { a; }", "f() { [a] }"}, + {"function f { a; }", "f() { [a] }"}, + {"f() (a)", "f() (a)"}, } for _, tc := range tests { tree, err := parse(tc.input) @@ -1522,6 +1563,8 @@ func TestParse_Incomplete(t *testing.T) { "if a", "if a; then", "if a; then b", "if a; then b; else", "if a; then b; elif c", "for x", "for x in a b", "for x in a; do", "for x in a; do b", "while a", "while a; do b", "until a; do", + "(a", "(a; b", "{ a;", "case x in", "case x in a) b;;", "f() {", "f()", + "cat < out": "redirections of control structures are not supported (near `>')", - "for x in a; do b; done &": "only simple commands and pipelines can run in the background", - "a && b &": "only simple commands and pipelines can run in the background", + "f() { a; } | b": "function definitions in pipelines are not supported", + "f() { a; } > out": "redirections of function definitions are not supported", + "if a; then b; fi c": "syntax error near unexpected token `c'", } { if _, err := parse(input); err == nil || err.Error() != want { t.Errorf("parse(%q) = %v, want %q", input, err, want) diff --git a/packaging/man/gosh.1 b/packaging/man/gosh.1 index 53ca9f6..f9c85d4 100644 --- a/packaging/man/gosh.1 +++ b/packaging/man/gosh.1 @@ -320,6 +320,24 @@ Redirect standard output to where standard error goes. .BR &> " \fIfile\fP, " &>> " \fIfile\fP" Write (or append) standard output and standard error to .IR file . +.TP +.BI << word +Here-document: the lines after the command, up to a line that is exactly +.IR word , +are standard input. If no part of +.I word +is quoted, variables, command substitutions and backslashes before +.BR $ , +.BR \(ga , +.B \e +and newline are expanded in the lines. +.BI <<\- word +also removes leading tabs from the lines and the delimiter line. +.TP +.BI <<< " word" +Here-string: +.I word +and a newline are standard input. .PP Redirections are evaluated from left to right. Only file descriptors 0, 1 and 2 are supported. @@ -331,6 +349,7 @@ and 2 are supported. \fBuntil\fP \fIlist\fP; \fBdo\fP \fIlist\fP; \fBdone\fP \fBcase\fP \fIword\fP \fBin\fP [\fB(\fP]\fIpattern\fP[\fB|\fP\fIpattern\fP]...\fB)\fP \fIlist\fP \fB;;\fP ... \fBesac\fP \fB{\fP \fIlist\fP; \fB}\fP +\fB(\fP \fIlist\fP \fB)\fP \fB!\fP \fIcommand\fP .fi .PP @@ -362,11 +381,23 @@ of the last clause may be omitted. .IR list ; .B } groups commands; they run in the current shell. +.B ( +.I list +.B ) +runs the commands in a subshell, a new +.B gosh +process, so that changes of the directory, variables and functions only +apply there. .PP -Control structures cannot be used in pipes, with redirections or in the -background, and there are no -.B ( ... ) -subshells. +Redirections after a control structure +.RB ( "done < file" , +.BR "fi > log 2>&1" ) +apply to all commands in it, which still run in the current shell. In a +pipe and in the background +.RB ( & ), +a control structure runs in a subshell, like in +.BR sh (1). +The subshell knows the functions and positional parameters of the shell. .SS Functions .nf \fIname\fP\fB() {\fP \fIlist\fP; \fB}\fP @@ -631,6 +662,9 @@ make > build.log 2>&1 || echo "failed: $?" LANG=C sort file alias ll=\(aqls \-l\(aq greet() { echo "Hello ${1:\-World}"; } +git ls\-files | while read \-r f; do wc \-l "$f"; done +while read \-r user rest; do echo "$user"; done < /etc/passwd +(cd build && make) > build.log 2>&1 case "$1" in start|stop) echo "$1ing" ;; *) echo "usage: $0 start|stop" ;; esac read \-r \-p \(aqName? \(aq name chsh \-s /usr/bin/gosh diff --git a/parser.go b/parser.go index 7005010..23a810f 100644 --- a/parser.go +++ b/parser.go @@ -2,7 +2,7 @@ package main // Parser and executor for command lines and scripts: lists (; & && || and // newlines), pipelines, ! and the compound commands if, for, while, until, -// case and { ...; }, and function definitions. Simple commands (with their pipes and redirections) are kept as +// case, { ...; } and ( ... ), and function definitions. Simple commands (with their pipes and redirections) are kept as // text; they are expanded and run by execCommand only when they are // executed, so that e.g. the variable of a for loop is expanded anew in // every iteration. @@ -65,7 +65,7 @@ type tokenKind int const ( tWord tokenKind = iota - tOp // ; ;; & && || | or newline + tOp // ; ;; & && || | ( ) or newline ) type rawToken struct { @@ -148,7 +148,7 @@ func lex(src string) ([]rawToken, error) { case ch == '|' && next == '|', ch == '&' && next == '&': op(src[i:i+2], i) i += 2 - case ch == '|', ch == '&' && next != '>': + case ch == '|', ch == '&' && next != '>', ch == '(', ch == ')': op(string(ch), i) i++ default: @@ -178,6 +178,12 @@ func lexWord(src string, i int) (int, error) { return 0, errIncomplete } j++ + if isSubstStart(src, j) { + // \$(...) is literal text, but its ( and ) belong to the word + if end := substEnd(src, j); end >= 0 { + j = end + } + } case isSubstStart(src, j): end := substEnd(src, j) if end < 0 { @@ -194,7 +200,7 @@ func lexWord(src string, i int) (int, error) { } case ch == '"' || ch == '\'': quote = ch - case ch == ' ' || ch == '\t' || ch == '\n' || ch == ';' || ch == '|': + case ch == ' ' || ch == '\t' || ch == '\n' || ch == ';' || ch == '|' || ch == '(' || ch == ')': return j, nil case ch == '&': // &>, >& and <& are redirections, not the & operator @@ -232,6 +238,7 @@ type listEntry struct { type andOrNode struct { first *pipelineNode rest []andOrPart + text string // the source, to run it in a subshell in the background } type andOrPart struct { @@ -239,8 +246,8 @@ type andOrPart struct { pipeline *pipelineNode } -// pipelineNode is a simple command (whose text may contain a pipeline) or a -// control structure, optionally negated with !. +// pipelineNode is a pipeline (a simpleNode or pipeNode) or a single +// compound command, optionally negated with !. type pipelineNode struct { negate bool cmd node @@ -251,6 +258,33 @@ type simpleNode struct { text string } +// pipeNode is a pipeline that contains compound commands. Like all commands +// of a pipeline in sh, they run in subshells (new gosh processes), since +// the commands of a pipeline run concurrently. +type pipeNode struct { + elems []pipeElem + text string +} + +type pipeElem struct { + text string // the source of the command, with its redirections + compound bool + subshell string // for ( list ): the list, which runs in the subshell directly +} + +// redirNode is a compound command with redirections; they apply to all +// commands in it. +type redirNode struct { + cmd node + redirs string // the source text of the redirections +} + +// subshellNode is ( list ): the list runs in a new gosh process, so that it +// cannot change the shell. +type subshellNode struct { + text string // the source of the list +} + type ifNode struct { conds []*listNode bodies []*listNode @@ -316,6 +350,10 @@ type parser struct { // parse parses a command line or script. It returns errIncomplete if the // input ends inside a construct. func parse(src string) (*listNode, error) { + src, err := extractHeredocs(src) + if err != nil { + return nil, err + } src, continued := joinContinuationLines(src) if continued { return nil, errIncomplete @@ -409,11 +447,6 @@ func (p *parser) parseList(terms map[string]bool) (*listNode, error) { default: return nil, p.unexpected(t) } - if entry.background { - if _, ok := ao.first.cmd.(*simpleNode); !ok || len(ao.rest) > 0 || ao.first.negate { - return nil, errors.New("only simple commands and pipelines can run in the background") - } - } list.items = append(list.items, entry) } } @@ -430,6 +463,7 @@ func (p *parser) nonEmpty(list *listNode) error { } func (p *parser) parseAndOr() (*andOrNode, error) { + start := p.peek().start first, err := p.parsePipeline() if err != nil { return nil, err @@ -438,6 +472,7 @@ func (p *parser) parseAndOr() (*andOrNode, error) { for { t := p.peek() if t == nil || t.kind != tOp || (t.text != "&&" && t.text != "||") { + ao.text = p.src[start:p.toks[p.pos-1].end] return ao, nil } p.pos++ @@ -453,73 +488,165 @@ func (p *parser) parseAndOr() (*andOrNode, error) { } } +// parsePipeline parses [!] command [| command]... A pipeline of simple +// commands stays one simpleNode (its text is split into the commands when it +// runs); a pipeline with compound commands becomes a pipeNode. func (p *parser) parsePipeline() (*pipelineNode, error) { pl := &pipelineNode{} if p.peekWord("!") { pl.negate = true p.pos++ } - t := p.peek() - if t == nil { - return nil, errIncomplete - } - if t.kind == tOp || terminators[t.text] { - return nil, p.unexpected(t) - } - - var err error - compound := true - start := t.start // funcDefName may split the token - if name, ok := p.funcDefName(); ok { - pl.cmd, err = p.parseFuncDef(name, start) - } else if compoundStarts[t.text] { - pl.cmd, err = p.parseCompound() - } else { - compound = false - pl.cmd, err = p.parseSimple() - } - if err != nil { - return nil, err - } - - if compound { - if t := p.peek(); t != nil { - switch { - case t.kind == tOp && t.text == "|": - return nil, errors.New("control structures in pipelines are not supported") - case t.kind == tWord && !terminators[t.text]: - return nil, fmt.Errorf("redirections of control structures are not supported (near `%s')", t.text) - } + var ( + elems []pipeElem + cmds []node + start = -1 + ) + for { + t := p.peek() + if t == nil { + return nil, errIncomplete } + if t.kind == tOp && t.text != "(" || t.kind == tWord && terminators[t.text] { + return nil, p.unexpected(t) + } + if start < 0 { + start = t.start + } + cmd, compound, err := p.parseCommand() + if err != nil { + return nil, err + } + elem := pipeElem{text: p.src[t.start:p.toks[p.pos-1].end], compound: compound} + if sub, ok := cmd.(*subshellNode); ok { + elem.subshell = sub.text + } + elems = append(elems, elem) + cmds = append(cmds, cmd) + + if t := p.peek(); t == nil || t.kind != tOp || t.text != "|" { + break + } + p.pos++ + p.skipNewlines() + if p.peek() == nil { + return nil, errIncomplete + } + } + + text := p.src[start:p.toks[p.pos-1].end] + compound := false + for i, e := range elems { + if _, ok := cmds[i].(*funcDefNode); ok && len(elems) > 1 { + return nil, errors.New("function definitions in pipelines are not supported") + } + compound = compound || e.compound + } + switch { + case len(elems) == 1: + pl.cmd = cmds[0] + case !compound: + pl.cmd = &simpleNode{text: text} + default: + pl.cmd = &pipeNode{elems: elems, text: text} } return pl, nil } -// parseSimple parses a pipeline of simple commands and returns its text. +// parseCommand parses a simple command, a compound command with its +// redirections or a function definition, and reports whether it is not a +// simple command. +func (p *parser) parseCommand() (node, bool, error) { + t := p.peek() + start := t.start + if name, ok := p.funcDefName(); ok { + def, err := p.parseFuncDef(name, start) + if err != nil { + return nil, true, err + } + if t := p.peek(); t != nil && t.kind == tWord && !terminators[t.text] { + return nil, true, errors.New("redirections of function definitions are not supported") + } + return def, true, nil + } + if !isCompoundStart(t) { + cmd, err := p.parseSimple() + return cmd, false, err + } + cmd, err := p.parseCompound() + if err != nil { + return nil, true, err + } + redirs, err := p.parseRedirections() + if err != nil { + return nil, true, err + } + if redirs != "" { + cmd = &redirNode{cmd: cmd, redirs: redirs} + } + return cmd, true, nil +} + +// isCompoundStart reports whether a compound command starts at t. +func isCompoundStart(t *rawToken) bool { + return t.kind == tWord && compoundStarts[t.text] || t.kind == tOp && t.text == "(" +} + +// parseRedirections parses the redirections after a compound command and +// returns their source text (expanded when the command runs). +func (p *parser) parseRedirections() (string, error) { + start, end := -1, -1 + for t := p.peek(); t != nil && t.kind == tWord && !terminators[t.text]; t = p.peek() { + needsTarget, ok := redirectionWord(t.text) + if !ok { + return "", syntaxError(t.text) + } + if start < 0 { + start = t.start + } + end = t.end + p.pos++ + if needsTarget { + t = p.peek() + if t == nil || t.kind != tWord { + if t == nil { + return "", syntaxError("newline") + } + return "", p.unexpected(t) + } + end = t.end + p.pos++ + } + } + if start < 0 { + return "", nil + } + return p.src[start:end], nil +} + +// redirectionWord reports whether word starts with a redirection operator +// (optionally after a file descriptor), and whether the target is the next +// word (2> file instead of 2>file). +func redirectionWord(word string) (needsTarget, ok bool) { + w := strings.TrimLeft(word, "0123456789") + for _, op := range []string{"&>>", "&>", "<<<", ">>", ">&", "<&", ">", "<"} { + if strings.HasPrefix(w, op) { + if op[0] == '&' && len(w) != len(word) { + return false, false + } + return w == op, true + } + } + return false, false +} + +// parseSimple parses a simple command (without pipes) and returns its text. func (p *parser) parseSimple() (*simpleNode, error) { start := p.toks[p.pos].start end := start - for t := p.peek(); t != nil; t = p.peek() { - if t.kind == tWord { - end = t.end - p.pos++ - continue - } - if t.text != "|" { - break - } - p.pos++ + for t := p.peek(); t != nil && t.kind == tWord; t = p.peek() { end = t.end - p.skipNewlines() - next := p.peek() - switch { - case next == nil: - return nil, errIncomplete - case next.kind == tOp || terminators[next.text]: - return nil, p.unexpected(next) - case compoundStarts[next.text]: - return nil, errors.New("control structures in pipelines are not supported") - } + p.pos++ } return &simpleNode{text: p.src[start:end]}, nil } @@ -536,6 +663,8 @@ func (p *parser) parseCompound() (node, error) { return p.parseWhile() case "case": return p.parseCase() + case "(": + return p.parseSubshell() default: // { return p.parseGroup() } @@ -668,6 +797,28 @@ func (p *parser) parseGroup() (*groupNode, error) { return &groupNode{body: body}, nil } +// parseSubshell parses ( list ). +func (p *parser) parseSubshell() (*subshellNode, error) { + p.pos++ // ( + start := p.toks[p.pos-1].end + body, err := p.parseList(map[string]bool{")": true}) + if err == nil { + err = p.nonEmpty(body) + } + if err != nil { + return nil, err + } + t := p.peek() + if t == nil { + return nil, errIncomplete + } + if t.kind != tOp || t.text != ")" { + return nil, p.unexpected(t) + } + p.pos++ + return &subshellNode{text: p.src[start:t.start]}, nil +} + // parseCase parses case word in [(]pattern[|pattern]...) list ;; ... esac. // The last clause may omit the ;;. func (p *parser) parseCase() (*caseNode, error) { @@ -713,18 +864,12 @@ func (p *parser) parseCase() (*caseNode, error) { } // parsePatterns parses the patterns of a case clause up to and including the -// ). Since ( and ) are not operators for the lexer, they can be part of a -// word token ("a)", "(a", "*)echo"); such tokens are split here. +// ). func (p *parser) parsePatterns() ([]string, error) { - var patterns []string - if t := p.peek(); t.kind == tWord && strings.HasPrefix(t.text, "(") { - // The optional ( before the first pattern - if t.text == "(" { - p.pos++ - } else { - t.text, t.start = t.text[1:], t.start+1 - } + if t := p.peek(); t.kind == tOp && t.text == "(" { + p.pos++ // the optional ( before the first pattern } + var patterns []string for { t := p.peek() if t == nil { @@ -733,109 +878,48 @@ func (p *parser) parsePatterns() ([]string, error) { if t.kind != tWord { return nil, p.unexpected(t) } - i := unquotedIndex(t.text, ')') - if i < 0 { - patterns = append(patterns, t.text) - p.pos++ - next := p.peek() - switch { - case next == nil: - return nil, errIncomplete - case next.kind == tOp && next.text == "|": - p.pos++ - case next.kind == tWord && strings.HasPrefix(next.text, ")"): - default: - return nil, p.unexpected(next) - } - continue - } - if i > 0 { - patterns = append(patterns, t.text[:i]) - } - if len(patterns) == 0 { - return nil, syntaxError(")") - } - if rest := t.text[i+1:]; rest == "" { - p.pos++ - } else { - // The first word of the body follows right after the ) - t.text, t.start = rest, t.start+i+1 - } - return patterns, nil - } -} - -// unquotedIndex returns the index of the first c in word that is not quoted, -// escaped or inside $(...) or ${...}, or -1. -func unquotedIndex(word string, c byte) int { - var quote byte - for i := 0; i < len(word); i++ { - ch := word[i] + patterns = append(patterns, t.text) + p.pos++ + t = p.peek() switch { - case quote == '\'': - if ch == '\'' { - quote = 0 - } - case ch == '\\': - i++ - case isSubstStart(word, i): - if end := substEnd(word, i); end >= 0 { - i = end - } - case ch == '$' && i+1 < len(word) && word[i+1] == '{': - if k := strings.IndexByte(word[i:], '}'); k >= 0 { - i += k - } - case quote == '"': - if ch == '"' { - quote = 0 - } - case ch == '"' || ch == '\'': - quote = ch - case ch == c: - return i + case t == nil: + return nil, errIncomplete + case t.kind == tOp && t.text == "|": + p.pos++ + case t.kind == tOp && t.text == ")": + p.pos++ + return patterns, nil + default: + return nil, p.unexpected(t) } } - return -1 } // funcDefName reports whether a function definition starts at the current -// token: name(), name () or function name [()]. It consumes the tokens up to -// the body; a body that starts in the same token (name(){) becomes a token of -// its own. +// token: name(), or function name [()]. It consumes the tokens up to the +// body. func (p *parser) funcDefName() (string, bool) { start := p.pos - t := p.peek() - keyword := t.kind == tWord && t.text == "function" - if keyword { + if t := p.peek(); t.kind != tWord { + return "", false + } else if t.text == "function" { p.pos++ - if t = p.peek(); t == nil || t.kind != tWord { + if t := p.peek(); t == nil || t.kind != tWord || !isFuncName(t.text) { p.pos = start return "", false } } - var name, rest string - if n, r, ok := strings.Cut(t.text, "()"); ok && isFuncName(n) { - // name() or name(){ - name, rest = n, r - } else if next := p.pos + 1; t.kind == tWord && isFuncName(t.text) && next < len(p.toks) && - p.toks[next].kind == tWord && strings.HasPrefix(p.toks[next].text, "()") { - // name () - name = t.text - p.pos = next - t = &p.toks[next] - rest = t.text[2:] - } else if keyword && isFuncName(t.text) { - // function name - name = t.text - } else { - p.pos = start - return "", false + name := p.peek().text + isOp := func(i int, text string) bool { + return i < len(p.toks) && p.toks[i].kind == tOp && p.toks[i].text == text } - if rest == "" { - p.pos++ - } else { - t.text, t.start = rest, t.end-len(rest) + switch { + case isOp(p.pos+1, "(") && isOp(p.pos+2, ")") && isFuncName(name): + p.pos += 3 + case p.pos > start: + p.pos++ // function name + default: + return "", false } return name, true } @@ -848,7 +932,7 @@ func (p *parser) parseFuncDef(name string, start int) (*funcDefNode, error) { if t == nil { return nil, errIncomplete } - if t.kind != tWord || !compoundStarts[t.text] { + if !isCompoundStart(t) { return nil, fmt.Errorf("%s: the body of a function must be a compound command like { ...; }", sanitize(name)) } body, err := p.parseCompound() @@ -901,7 +985,7 @@ func (l *listNode) exec() error { for _, entry := range l.items { printError(err) if entry.background { - err = entry.andOr.first.cmd.(*simpleNode).run(true) + err = entry.andOr.runBackground() } else { err = entry.andOr.exec() } @@ -929,6 +1013,22 @@ func (a *andOrNode) exec() error { return err } +// runBackground starts the and-or list as a background job. Pipelines run +// like in the foreground, everything else in a subshell. +func (a *andOrNode) runBackground() error { + if len(a.rest) == 0 && !a.first.negate { + switch cmd := a.first.cmd.(type) { + case *simpleNode: + return cmd.run(true) + case *pipeNode: + return cmd.run(true) + case *subshellNode: + return runPipeline([]stage{{script: cmd.text}}, a.text, true) + } + } + return runPipeline([]stage{{script: a.text}}, a.text, true) +} + func (pl *pipelineNode) exec() error { err := pl.cmd.exec() if !pl.negate || isAbort(err) { @@ -1066,6 +1166,65 @@ func (n *groupNode) exec() error { return n.body.exec() } +func (n *subshellNode) exec() error { + return runPipeline([]stage{{script: n.text}}, "("+strings.TrimSpace(n.text)+")", false) +} + +// exec runs the compound command with stdin, stdout and stderr of the shell +// redirected. +func (n *redirNode) exec() error { + fds, opened, err := openRedirections(n.redirs) + if err != nil { + return err + } + saved := ioOverride + ioOverride = fds + defer func() { + ioOverride = saved + for _, f := range opened { + f.Close() + } + }() + return n.cmd.exec() +} + +func (n *pipeNode) exec() error { + return n.run(false) +} + +// run builds the stages of the pipeline: simple commands are expanded, +// compound commands run in subshells. +func (n *pipeNode) run(background bool) error { + substStatus = 0 + var stages []stage + for _, e := range n.elems { + switch { + case e.subshell != "": + stages = append(stages, stage{script: e.subshell}) + case e.compound: + stages = append(stages, stage{script: e.text}) + default: + text := e.text + if interactiveShell && aliasExpansion { + if tree, err := parse(expandAliases(text)); err == nil { + if simple := tree.singleSimple(); simple != nil { + text = simple.text + } + } + } + st, err := expandStages(text) + if err != nil { + return err + } + if len(st) == 0 { + st = []stage{{}} + } + stages = append(stages, st...) + } + } + return runPipeline(stages, n.text, background) +} + func (n *funcDefNode) exec() error { defineFunction(n.def) return nil