add subshells, here-documents and redirections/pipes for control structures

- control structures can have redirections (while read l; do ...; done
  < file), which apply to all commands in them; they run in the shell
- in pipes and in the background they run in a subshell (cmd | while
  read l; do ...; done, for ...; done &, a && b &)
- ( list ) subshells; ( and ) are operators now
- here-documents <<EOF, <<-EOF, <<'EOF' and here-strings <<<word
- redirections of source apply to the commands of the file
- fix non-ASCII characters in quotes being mangled ('für')

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

heredoc.go

@@ -0,0 +1,268 @@
+package main
+
+// Here-documents: cmd <<WORD, followed by lines up to a line WORD, is the
+// same as a here-string cmd <<<"lines" (without the last newline, which the
+// here-string adds). extractHeredocs rewrites the source that way before it
+// is parsed, so that the rest of gosh (and subshells, which get the source
+// text) only has to know here-strings.
+
+import (
+	"strings"
+)
+
+// hereDoc is a here-document whose body has not been read yet.
+type hereDoc struct {
+	seg       int // the index of its placeholder in the output segments
+	delim     string
+	quoted    bool // the delimiter was quoted: the body is not expanded
+	stripTabs bool // <<-: leading tabs are removed from the lines
+}
+
+// extractHeredocs replaces every here-document operator in src by a
+// here-string with its body and removes the body and delimiter lines. The
+// body starts after the end of the line with the operator. If the input
+// ends before the delimiter line, it returns errIncomplete (an interactive
+// shell then reads more lines).
+//
+// The body is expanded like "..." if the delimiter is unquoted ($VAR, $(...),
+// \ before $ ` \ and newline), and taken literally if any part of the
+// delimiter is quoted ('EOF', "EOF", \EOF).
+func extractHeredocs(src string) (string, error) {
+	if !strings.Contains(src, "<<") {
+		return src, nil
+	}
+	var (
+		segs      []string
+		cur       strings.Builder
+		pending   []hereDoc
+		wordStart = true
+	)
+	copyTo := func(i, end int) int {
+		if end < 0 || end > len(src) {
+			end = len(src)
+		}
+		cur.WriteString(src[i:end])
+		return end
+	}
+
+	for i := 0; i < len(src); {
+		ch := src[i]
+		switch {
+		case ch == '\\':
+			i = copyTo(i, i+2)
+			wordStart = false
+		case ch == '\'':
+			end := strings.IndexByte(src[i+1:], '\'')
+			if end >= 0 {
+				end += i + 2
+			}
+			i = copyTo(i, end)
+			wordStart = false
+		case ch == '"':
+			end := dquoteEnd(src, i)
+			if end >= 0 {
+				end++
+			}
+			i = copyTo(i, end)
+			wordStart = false
+		case isSubstStart(src, i):
+			end := substEnd(src, i)
+			if end >= 0 {
+				end++
+			}
+			i = copyTo(i, end)
+			wordStart = false
+		case ch == '#' && wordStart:
+			// A comment, until the end of the line
+			end := strings.IndexByte(src[i:], '\n')
+			if end >= 0 {
+				end += i
+			}
+			i = copyTo(i, end)
+		case strings.HasPrefix(src[i:], "<<<"):
+			i = copyTo(i, i+3)
+			wordStart = false
+		case strings.HasPrefix(src[i:], "<<"):
+			h, n, err := parseHereOp(src[i:])
+			if err != nil {
+				return "", err
+			}
+			segs = append(segs, cur.String())
+			cur.Reset()
+			h.seg = len(segs)
+			segs = append(segs, "") // filled in when the body has been read
+			pending = append(pending, h)
+			i += n
+			wordStart = false
+		case ch == '\n':
+			cur.WriteByte(ch)
+			i++
+			wordStart = true
+			// The bodies of the here-documents of this line follow
+			for _, h := range pending {
+				body, n, ok := readHereBody(src[i:], h)
+				if !ok {
+					return "", errIncomplete
+				}
+				segs[h.seg] = hereString(body, h.quoted)
+				i += n
+			}
+			pending = nil
+		default:
+			cur.WriteByte(ch)
+			i++
+			wordStart = strings.IndexByte(" \t;&|()<>", ch) >= 0
+		}
+	}
+	if len(pending) > 0 {
+		return "", errIncomplete
+	}
+	segs = append(segs, cur.String())
+	return strings.Join(segs, ""), nil
+}
+
+// dquoteEnd returns the index of the " that ends the double-quoted string
+// starting at s[i], or -1.
+func dquoteEnd(s string, i int) int {
+	for j := i + 1; j < len(s); j++ {
+		switch {
+		case s[j] == '\\':
+			j++
+		case isSubstStart(s, j):
+			end := substEnd(s, j)
+			if end < 0 {
+				return -1
+			}
+			j = end
+		case s[j] == '"':
+			return j
+		}
+	}
+	return -1
+}
+
+// parseHereOp parses a here-document operator <<WORD or <<-WORD at the
+// start of s and returns it and its length.
+func parseHereOp(s string) (hereDoc, int, error) {
+	h := hereDoc{}
+	i := 2
+	if i < len(s) && s[i] == '-' {
+		h.stripTabs = true
+		i++
+	}
+	for i < len(s) && (s[i] == ' ' || s[i] == '\t') {
+		i++
+	}
+	var delim strings.Builder
+	start := i
+	for i < len(s) && strings.IndexByte(" \t\n;&|()<>", s[i]) < 0 {
+		switch c := s[i]; c {
+		case '\'', '"':
+			end := strings.IndexByte(s[i+1:], c)
+			if end < 0 {
+				return h, 0, errIncomplete
+			}
+			delim.WriteString(s[i+1 : i+1+end])
+			i += end + 2
+			h.quoted = true
+		case '\\':
+			if i+1 < len(s) {
+				delim.WriteByte(s[i+1])
+			}
+			i += 2
+			h.quoted = true
+		default:
+			delim.WriteByte(c)
+			i++
+		}
+	}
+	if i == start {
+		if i == len(s) || s[i] == '\n' {
+			return h, 0, syntaxError("newline")
+		}
+		return h, 0, syntaxError(s[i : i+1])
+	}
+	h.delim = delim.String()
+	return h, min(i, len(s)), nil
+}
+
+// readHereBody reads the lines of a here-document from s up to the
+// delimiter line. It returns the body (each line with its newline) and the
+// number of bytes consumed, or ok == false if there is no delimiter line yet.
+func readHereBody(s string, h hereDoc) (body string, n int, ok bool) {
+	var b strings.Builder
+	for pos := 0; ; {
+		end := strings.IndexByte(s[pos:], '\n')
+		last := end < 0
+		if last {
+			end = len(s) - pos
+		}
+		line := s[pos : pos+end]
+		if h.stripTabs {
+			line = strings.TrimLeft(line, "\t")
+		}
+		next := min(pos+end+1, len(s))
+		if line == h.delim {
+			return b.String(), next, true
+		}
+		if last {
+			return "", 0, false
+		}
+		b.WriteString(line)
+		b.WriteByte('\n')
+		pos = next
+	}
+}
+
+// hereString returns the here-string (or redirection) that has the effect of
+// a here-document with the given body.
+func hereString(body string, quoted bool) string {
+	if body == "" {
+		return "</dev/null"
+	}
+	body = strings.TrimSuffix(body, "\n")
+	if quoted {
+		return "<<<'" + strings.ReplaceAll(body, "'", `'\''`) + "'"
+	}
+	return `<<<"` + escapeHereBody(body) + `"`
+}
+
+// escapeHereBody quotes the body of an unquoted here-document for "...":
+// " becomes \", a backslash that only escapes $ ` \ and newline in a
+// here-document stays one, others become \\. $(...) and ${...} are kept as
+// they are, "$@" becomes "$*" (one word, as in a here-document).
+func escapeHereBody(body string) string {
+	var b strings.Builder
+	for i := 0; i < len(body); i++ {
+		c := body[i]
+		switch {
+		case isSubstStart(body, i):
+			end := substEnd(body, i)
+			if end < 0 {
+				end = len(body) - 1
+			}
+			b.WriteString(body[i : end+1])
+			i = end
+		case strings.HasPrefix(body[i:], "$@"):
+			b.WriteString("$*")
+			i++
+		case strings.HasPrefix(body[i:], "${"):
+			end := strings.IndexByte(body[i:], '}')
+			if end < 0 {
+				end = len(body) - i - 1
+			}
+			b.WriteString(strings.Replace(body[i:i+end+1], "${@", "${*", 1))
+			i += end
+		case c == '\\' && i+1 < len(body) && strings.IndexByte("$`\\\n", body[i+1]) >= 0:
+			b.WriteString(body[i : i+2])
+			i++
+		case c == '\\':
+			b.WriteString(`\\`)
+		case c == '"':
+			b.WriteString(`\"`)
+		default:
+			b.WriteByte(c)
+		}
+	}
+	return b.String()
+}