diff --git a/Makefile b/Makefile index 9d8b7c6..31d9a52 100644 --- a/Makefile +++ b/Makefile @@ -899,8 +899,9 @@ $(MKFS): tools/mkfs.c include/fs.h include/stat.h include/param.h # /etc/passwd and /etc/motd and should not become true of the boot # configuration as well. SCHINIT_JOBS = $(wildcard etc/schinit/*.job) etc/schinit/README +ETC_ZSH = $(wildcard etc/zsh/*) ETC_FILES = $(wildcard etc/*) -ETC_PLAIN = $(filter-out etc/schinit,$(ETC_FILES)) +ETC_PLAIN = $(filter-out etc/schinit etc/zsh,$(ETC_FILES)) # The mark as characters, for anything that draws a logo into a terminal -- # fastfetch takes it with --file-raw. Installed from docs/branding rather than @@ -912,12 +913,13 @@ ETC_PLAIN = $(filter-out etc/schinit,$(ETC_FILES)) # anything with more room. LOGO_ASCII = docs/branding/Ascii/ascii_50.txt -etc-install: $(SCHINIT_JOBS) $(ETC_PLAIN) $(LOGO_ASCII) - @mkdir -p $(SYSROOT)/etc/schinit $(SYSROOT)/usr/share/schos +etc-install: $(SCHINIT_JOBS) $(ETC_PLAIN) $(ETC_ZSH) $(LOGO_ASCII) + @mkdir -p $(SYSROOT)/etc/schinit $(SYSROOT)/etc/zsh $(SYSROOT)/usr/share/schos @cp $(ETC_PLAIN) $(SYSROOT)/etc/ @cp $(SCHINIT_JOBS) $(SYSROOT)/etc/schinit/ + @cp $(ETC_ZSH) $(SYSROOT)/etc/zsh/ @cp $(LOGO_ASCII) $(SYSROOT)/usr/share/schos/logo.txt - @echo "installed $(words $(ETC_PLAIN)) /etc file(s) and $(words $(SCHINIT_JOBS)) schinit manifest(s)" + @echo "installed $(words $(ETC_PLAIN)) /etc file(s), $(words $(ETC_ZSH)) zsh file(s) and $(words $(SCHINIT_JOBS)) schinit manifest(s)" # The mount points. Created here rather than by init, because a mkdir on a # fresh SchFS is a whole transaction -- a copy-on-write rewrite of the path to @@ -927,7 +929,7 @@ etc-install: $(SCHINIT_JOBS) $(ETC_PLAIN) $(LOGO_ASCII) # # Not committed as empty directories with a .gitkeep, because /sys and /run are # not mounted over and the marker file would be visible in them. -SYSROOT_DIRS = dev proc sys run tmp +SYSROOT_DIRS = dev proc sys run tmp root sysroot-install: $(addprefix $(SYSROOT)/bin/, $(CMDS)) etc-install @mkdir -p $(addprefix $(SYSROOT)/, $(SYSROOT_DIRS)) diff --git a/NOTICE b/NOTICE index 113b5b9..e14e588 100644 --- a/NOTICE +++ b/NOTICE @@ -50,6 +50,32 @@ Portions derived from the Public Domain Korn Shell (pdksh) Built from ports/mksh. Supplies /bin/sh. The MirOS Licence is permissive and requires the copyright notice be retained. +-------------------------------------------------------------------------------- +zsh 5.9 MIT-like +-------------------------------------------------------------------------------- +Copyright (c) 1992-2017 Paul Falstad, Richard Coleman, Zoltan Hidvegi, +Andrew Main, Peter Stephenson, Sven Wischnowsky, and others + +Built from ports/zsh. Installed as /bin/zsh and used as the interactive shell; +/bin/sh remains mksh. + +zsh's LICENCE notes that a few shell functions are licensed under the GNU +General Public Licence and states that, since no core function is affected, +those files may simply be omitted. They are. ports/zsh/Makefile removes four +files after extraction -- completions for darcs, qdbus, osc and zypper -- and +refuses to install if any of them is still present. No GPL-licensed file is +shipped in a SchOS image. + +-------------------------------------------------------------------------------- +ncurses 6.4 MIT License +-------------------------------------------------------------------------------- +Copyright 2018-2022 Thomas E. Dickey +Copyright 1998-2018 Free Software Foundation, Inc. + +Built from ports/ncurses. Linked into zsh so that its line editor can drive a +terminal, and into sysroot/bin/tinfotest. Built with --disable-database and +compiled-in fallbacks, so no terminfo files are installed. + -------------------------------------------------------------------------------- lwIP 2.2.0 BSD 3-Clause -------------------------------------------------------------------------------- diff --git a/cmd/schinit/main.c b/cmd/schinit/main.c index c67ec9e..6114f7b 100644 --- a/cmd/schinit/main.c +++ b/cmd/schinit/main.c @@ -104,12 +104,22 @@ outn(int v) // Jobs // --------------------------------------------------------------------------- +#define MAXENVV 16 // Environment lines honoured per job + struct job { char label[MAXLABEL]; char program[MAXPATH]; char *argv[MAXARGS + 1]; char argstore[MAXLINE]; + // Environment, one "KEY=VALUE" per Environment line, repeatable. Stored + // as pointers into envstore so the job owns its own strings -- the parse + // buffer is gone by the time anything execs. + char *envp[MAXENVV + 1]; + char envstore[MAXLINE * 2]; + int envlen; + int envn; + char stdin_path[MAXPATH]; char stdout_path[MAXPATH]; char stderr_path[MAXPATH]; @@ -192,6 +202,27 @@ split_args(struct job *j, const char *value) // into sock_path rather than treated as a job key. // --------------------------------------------------------------------------- +// One "KEY=VALUE" onto the job's environment. Repeatable, so a manifest reads +// +// Environment = TERM=linux +// Environment = PATH=/bin:/usr/bin +// +// rather than inventing a separator that a value might contain. +static void +add_env(struct job *j, const char *kv) +{ + if (j->envn >= MAXENVV) + return; + int n = slen(kv); + if (j->envlen + n + 1 > (int)sizeof(j->envstore)) + return; + char *dst = j->envstore + j->envlen; + scopy(dst, kv, n + 1); + j->envp[j->envn++] = dst; + j->envp[j->envn] = 0; + j->envlen += n + 1; +} + static char * trim(char *s) { @@ -267,6 +298,7 @@ load_job(const char *path) else if (sieq(key, "Disabled")) j->disabled = truthy(val); else if (sieq(key, "ThrottleInterval")) j->throttle = parse_int(val); else if (sieq(key, "ControllingTerminal")) j->own_tty = truthy(val); + else if (sieq(key, "Environment")) add_env(j, val); else if (sieq(key, "StandardIn")) scopy(j->stdin_path, val, MAXPATH); else if (sieq(key, "StandardOut")) scopy(j->stdout_path, val, MAXPATH); else if (sieq(key, "StandardError")) scopy(j->stderr_path, val, MAXPATH); @@ -379,7 +411,7 @@ start_job(struct job *j) // exec.c enables the Linux ABI automatically for musl binaries. Do NOT // call enable_linux_abi() here: it would renumber the syscalls this // child still has to make. - exec(j->program, j->argv); + execve(j->program, j->argv, j->envn ? j->envp : 0); out("schinit: cannot exec "); out(j->program); out(" for "); out(j->label); out("\n"); diff --git a/docs/history/kernel.md b/docs/history/kernel.md index ac0e112..2f8c5b9 100644 --- a/docs/history/kernel.md +++ b/docs/history/kernel.md @@ -904,3 +904,130 @@ separate and deliberately does not follow it: musl answers are scaled at the syscall boundary the way Linux does it. Before, `sys_times` returned raw ticks -- correct only while `HZ` happened to be 100, and silently wrong the day anyone changed it. + +## 14. A shell that would not start, and four bugs underneath it + +Porting zsh was supposed to be a port. It found four defects in the kernel and +one in the filesystem, none of which had anything to do with zsh, and every one +of which had been there the whole time. + +### The environment did not exist + +`kernel/fs/exec.c` built the initial stack for every new program and wrote this: + +```c +stack_slots[slot++] = 0; // end of envp (no environment) +``` + +Not "no environment yet" — no environment, ever, for any process. musl calls +`execve(path, argv, envp)`, which is Linux syscall 59, which SchOS answers; the +third argument was simply never read. So `$TERM` was unset, `$PATH` was whatever +each program invented for itself, and a variable exported in a shell did not +survive the exec that was supposed to carry it. + +That is fatal for zsh specifically. A line editor with no `TERM` cannot ask +terminfo how to move a cursor, so ZLE is exactly as useless as it would be with +no terminal library linked at all. + +`exec()` takes an `envp` now, `sys_exec` reads the third argument, and schinit +manifests grew an `Environment` key: + +``` +Environment = TERM=linux +Environment = PATH=/bin:/usr/bin +``` + +The userland `exec(path, argv)` spelling stays, as a wrapper in `lib/glue.c` +that passes a null environment explicitly. The raw stub is `execve` now — +otherwise a two-argument caller would leave `%rdx` holding whatever happened to +be in it, and the kernel would read that as a pointer. + +### Which immediately overflowed the kernel stack + +The three pointer arrays exec needs came to about 1.8 KB, and `KSTACKSIZE` is +4096, under a call chain that already runs exec → loaduvm → readi → SchFS → +btree. The first exec after the change took pid 1 down. + +The canary caught it cleanly, named the process, and printed a stop code, which +is the whole reason it exists. The arrays live in one `kalloc`'d page now. + +### `wait` said "operation not permitted" + +``` +compdef:80: wait failed: operation not permitted +``` + +`waitpid` returned a bare `-1` for two different failures: no children to wait +for, and interrupted by a signal. musl reads any return in [-4095, -1] as +`-errno`, so `-1` arrived in userspace as `EPERM`. + +zsh saw it, printed that message, and abandoned the rest of `/etc/zsh/zshrc` — +so the shell came up with its default prompt and no completion, which reads as +a broken config file rather than as a `wait()` that could not say what happened. +It returns `-ECHILD` and `-EINTR` now. + +This is the third time a bare `-1` has cost real time (see §12, `sys_chdir`). +**EPERM on SchOS should be read as "a syscall returned -1 and meant something +else."** There is no permission model here for it to legitimately come from. + +### mmap ran out of slots and blamed memory + +``` +compdef:80: fatal error: out of memory +``` + +On a machine with 1000 MB free, of which zsh had used 1.5. `NVMA` was 64 — the +per-process mmap table — and musl's allocator takes a fresh mapping per size +class and does not release them eagerly. compinit reads twelve hundred +completion functions, ran out of slots, and got `-1` from `sys_mmap`, which is +`EPERM` again, which `malloc` turns into NULL, which zsh reports as out of +memory. Three translations from the truth. + +`NVMA` is 256 and the failure returns `-ENOMEM`. + +### And underneath all of it: a small write costs 58 ms + +With everything above fixed, `compinit` still appeared to hang. It was not +hanging. Measured on this machine, 200 writes: + +| | time | per write | +|---|---|---| +| to one open file on SchFS | 11.75 s | **58 ms** | +| open + write + close each time | 21.39 s | **107 ms** | +| to `/dev/null` | 0.14 s | 0.7 ms | + +`compdump` writes its digest with a few thousand separate `print` calls. At +58 ms each that is minutes, and it looks exactly like a hang. `compinit -D`, +which skips the dump, finishes in 3.6 seconds — the scan was never the problem. + +The cause is not a bug; it is the design, stated plainly at the top of +`kernel/fs/schfs/kmount.c`: + +> Every entry point below that changes anything is one transaction group: it +> takes the volume lock, does its work, and commits before returning. That makes +> an operation durable the moment it returns. + +A commit is two cache flushes and an uberblock write, and a flush is a barrier. +`schfs_ktx_open` already lets one `write(2)` of 1 MB be a single group — 267 +block writes instead of 1410 — but nothing brackets *across* syscalls, so two +thousand small writes are two thousand barriers. + +`cp` does not notice, because it uses large buffers. Anything that writes a line +at a time notices enormously. + +**This is the open question, and it is a design decision rather than a fix.** +Deferring commits would mean giving up durable-on-return, which is a property +SchFS chose deliberately. The options are a writeback window like every other +filesystem has, committing on `fsync`/`close`/timer, or leaving it and accepting +that line-at-a-time writers are slow. Until it is decided, `/etc/zsh/zshrc` +leaves `compinit` switched off and says why. + +### What zsh actually got + +It runs, as `/bin/zsh`, and it is the console shell. ZLE works: arrow keys +recall history through terminfo, `$COLUMNS` reads 160 because `TIOCGWINSZ` now +answers with the real console geometry (§13), prompt expansion and colour work, +and startup is 0.14 s. mksh keeps `/bin/sh`, so schinit's no-jobs fallback is a +different program from the one that might have failed. + +Completion is the piece that waits on the filesystem. diff --git a/etc/schinit/console-shell.job b/etc/schinit/console-shell.job index 172689c..8b8a4e6 100644 --- a/etc/schinit/console-shell.job +++ b/etc/schinit/console-shell.job @@ -1,11 +1,23 @@ # The interactive shell on the system console. Label = os.schos.console-shell -Program = /bin/sh -Args = sh +# zsh, not /bin/sh. mksh stays as /bin/sh and is what schinit falls back to +# when no manifest loads -- so if zsh ever fails to start, the recovery shell +# is a different program rather than the same broken one. See ports/zsh. +Program = /bin/zsh +Args = -zsh RunAtLoad = yes KeepAlive = yes +# The environment this job's shell starts with. Until schinit grew an +# Environment key, every process on SchOS started with an empty environment -- +# the kernel built one and exec threw it away. $TERM is the one that matters +# most: a line editor with no TERM cannot ask terminfo how to move a cursor. +Environment = TERM=linux +Environment = PATH=/bin:/usr/bin +Environment = HOME=/root +Environment = SHELL=/bin/zsh + # This job reads the console, so it has to own it: schinit puts every job in # its own session, and a session with no controlling terminal is not delivered # terminal input at all. Without this the shell echoes and executes nothing. diff --git a/etc/zsh/zshenv b/etc/zsh/zshenv new file mode 100644 index 0000000..1d30f5e --- /dev/null +++ b/etc/zsh/zshenv @@ -0,0 +1,14 @@ +# /etc/zsh/zshenv — read by every zsh, interactive or not, before anything else. +# +# Only things that must be true for a non-interactive shell as well. schinit +# sets the console shell's environment from its job manifest, but a zsh started +# any other way (a script, an ssh session one day) gets no such help. + +: ${TERM:=linux} +: ${HOME:=/root} +: ${PATH:=/bin:/usr/bin} +export TERM HOME PATH + +# Where zsh's own functions live. Set here rather than compiled in so a second +# version can be installed beside this one without either moving. +fpath=(/usr/share/zsh/5.9/functions $fpath) diff --git a/etc/zsh/zshrc b/etc/zsh/zshrc new file mode 100644 index 0000000..1ec8e7e --- /dev/null +++ b/etc/zsh/zshrc @@ -0,0 +1,113 @@ +# /etc/zsh/zshrc — the system-wide interactive zsh configuration for SchOS. +# +# Deliberately short. This sets up the things an interactive shell is useless +# without -- history, completion, and the keys a console sends -- and stops. +# Anything beyond that belongs in a user's own ~/.zshrc, which is the file zsh +# reads after this one. + +# --------------------------------------------------------------------------- +# History +# --------------------------------------------------------------------------- +HISTFILE=${HOME:-/root}/.zsh_history +HISTSIZE=2000 +SAVEHIST=2000 +setopt APPEND_HISTORY INC_APPEND_HISTORY SHARE_HISTORY +setopt HIST_IGNORE_DUPS HIST_IGNORE_SPACE HIST_REDUCE_BLANKS + +# --------------------------------------------------------------------------- +# Behaviour +# --------------------------------------------------------------------------- +setopt AUTO_CD # `cd /tmp` is implied by typing `/tmp` +setopt EXTENDED_GLOB +setopt INTERACTIVE_COMMENTS # a # in an interactive line is a comment +setopt NO_BEEP # SchOS has no audio, but it would also scroll +unsetopt FLOW_CONTROL # ^S/^Q are worth more as keys than as flow control + +# --------------------------------------------------------------------------- +# Keys +# --------------------------------------------------------------------------- +# Emacs bindings, because that is what every other line editor on the system +# does -- mksh's default and the kernel's own canonical-mode editing both. +bindkey -e + +# Home, End, Delete and the arrows, taken from terminfo rather than written +# out as escape sequences. The sequences differ between the SchOS console and +# a serial terminal; the terminal's own description does not. +typeset -A _sch_keys=( + "${terminfo[khome]}" beginning-of-line + "${terminfo[kend]}" end-of-line + "${terminfo[kdch1]}" delete-char + "${terminfo[kich1]}" overwrite-mode + "${terminfo[kcuu1]}" up-line-or-search + "${terminfo[kcud1]}" down-line-or-search + "${terminfo[kcub1]}" backward-char + "${terminfo[kcuf1]}" forward-char +) +for _k _w in "${(@kv)_sch_keys}"; do + [[ -n $_k ]] && bindkey -- "$_k" "$_w" +done +unset _sch_keys _k _w + +# A terminal that has just been resized, or that zsh has just taken over, may +# not have told terminfo yet. This puts the keypad in the mode the escape +# sequences above were described for, and takes it back out on exit. +if (( ${+terminfo[smkx]} && ${+terminfo[rmkx]} )); then + autoload -Uz add-zle-hook-widget 2>/dev/null + function _sch_zle_init { echoti smkx } + function _sch_zle_line_finish { echoti rmkx } + zle -N zle-line-init _sch_zle_init + zle -N zle-line-finish _sch_zle_line_finish +fi + +# --------------------------------------------------------------------------- +# Completion +# --------------------------------------------------------------------------- +# Completion is NOT initialised here yet, and the reason is a filesystem +# measurement rather than anything to do with zsh. +# +# A small write to SchFS costs about 58 ms, because every operation is its own +# transaction group and a commit is two cache flushes and an uberblock write -- +# see the comment at the top of kernel/fs/schfs/kmount.c, which describes the +# trade-off exactly. Measured on this machine, 200 writes: +# +# to one open file on SchFS 11.75 s (58 ms each) +# open + write + close each 21.39 s (107 ms each) +# to /dev/null 0.14 s (0.7 ms each) +# +# compdump writes the digest with a few thousand separate `print` calls, so +# compinit -d takes minutes and looks exactly like a hang. compinit itself is +# fine: with -D, which skips the dump, it finishes in 3.6 seconds. +# +# So this waits for deferred commits. When small writes stop costing a disk +# barrier each, delete this comment and uncomment the three lines below. +# +# autoload -Uz compinit +# compinit -u -d ${HOME:-/root}/.zcompdump +# +# Until then, completion is available on demand in a shell you do not mind +# waiting three seconds for: +# +# autoload -Uz compinit && compinit -D -u +# +# -u is not optional whenever this is turned back on. Without it compinit runs +# compaudit, decides the function directories are "insecure" because SchOS has +# no ownership model to check, and *prompts* for an answer on a shell that is +# still starting up -- which is a real hang, unlike the one above. + +# These take effect as soon as compinit is turned on above; a zstyle with no +# completion system to read it is inert, not an error. +zstyle ':completion:*' menu select +zstyle ':completion:*' matcher-list 'm:{a-z}={A-Za-z}' # case-insensitive +zstyle ':completion:*' list-colors '' +zstyle ':completion:*:descriptions' format '%F{244}%d%f' + +# --------------------------------------------------------------------------- +# Prompt +# --------------------------------------------------------------------------- +# The accent violet from include/theme.h, which is 99 in the 256-colour cube +# and falls back to magenta on the 8-colour console. The last-status marker is +# the only thing here that ever turns red -- same rule as the rest of the +# system: normal states are calm, only failure is loud. +setopt PROMPT_SUBST +PROMPT='%F{magenta}%B%n@schos%b%f %F{cyan}%~%f %(?.%F{green}❯.%F{red}❯)%f ' +RPROMPT='' diff --git a/include/defs.h b/include/defs.h index 3a1e482..1ffc33c 100644 --- a/include/defs.h +++ b/include/defs.h @@ -79,7 +79,7 @@ void consoleintr(int(*)(void)); void panic(char*) __attribute__((noreturn)); // exec.c -int exec(char*, char**); +int exec(char*, char**, char**); // file.c struct file* filealloc(void); diff --git a/include/param.h b/include/param.h index a147212..81489d1 100644 --- a/include/param.h +++ b/include/param.h @@ -43,6 +43,7 @@ #define ROOTDEV 1 // device number of file system root disk #endif #define MAXARG 32 // max exec arguments +#define MAXENV 64 // max environment variables passed through exec #define MAXPATH 128 // max path #define MAXOPBLOCKS 30 // max # of blocks any FS op writes #define LOGSIZE (MAXOPBLOCKS*3) // max data blocks in on-disk log diff --git a/include/proc.h b/include/proc.h index 6876155..de4ef40 100644 --- a/include/proc.h +++ b/include/proc.h @@ -3,7 +3,18 @@ #include "types.h" #include "traps.h" -#define NVMA 64 +// Simultaneous mmap() regions per process. +// +// 64 was enough for every first-party program and for toybox, and not enough +// for zsh: musl's allocator takes a fresh mapping per size class and does not +// release them eagerly, so compinit -- which reads twelve hundred completion +// functions -- ran out of slots and reported "fatal error: out of memory" on +// a machine with a gigabyte free. The slots were the limit, not the memory. +// +// The table is inline in struct proc, so this is NPROC * NVMA * sizeof(struct +// vma) of kernel memory, always resident. At 256 that is about 1 MB total, +// which is worth it to stop a shell dying on a machine that has 1000 MB spare. +#define NVMA 256 struct vma { uint64 addr; diff --git a/include/user.h b/include/user.h index 77f058e..e23c707 100644 --- a/include/user.h +++ b/include/user.h @@ -18,7 +18,8 @@ int write(int, void*, int); int read(int, void*, int); int close(int); int kill(int); -int exec(char*, char**); +int exec(char*, char**); /* execve(path, argv, 0) */ +int execve(char*, char**, char**); int open(char*, int); int mknod(char*, short, short); int unlink(char*); diff --git a/kernel/fs/exec.c b/kernel/fs/exec.c index 185e41b..3d12453 100644 --- a/kernel/fs/exec.c +++ b/kernel/fs/exec.c @@ -18,20 +18,39 @@ extern int linux_syscall_map[]; extern int linux_syscall_map_size; +// envp may be null, which means "no environment" -- the same thing this +// function used to do unconditionally. int -exec(char *path, char **argv) +exec(char *path, char **argv, char **envp) { uint64 _t0 = rdtsc(); fsstat.exec_calls++; char *s, *last; int i, off; - uint64 argc, sz, sp, ustack[MAXARG+1]; + uint64 argc, envc, sz, sp; struct elfhdr elf; struct inode *ip; struct proghdr ph; pml4e_t *pgdir, *oldpgdir; - uint64 stack_slots[MAXARG + 32]; int slot; + + // The three pointer arrays live on the heap, not on the kernel stack. + // + // Together they are about 1.8 KB, and KSTACKSIZE is 4096 -- with the call + // chain under this function running exec -> loaduvm -> readi -> SchFS -> + // btree, there is not 1.8 KB to spare. Putting them here overflowed the + // stack on the first exec after the environment was added, and the canary + // in param.h caught it on pid 1 before it could corrupt anything. + // + // One page holds all three with room over. + uint64 *scratch = (uint64*)kalloc(); + if(!scratch) + return -1; + uint64 *ustack = scratch; /* MAXARG+1 */ + uint64 *estack = ustack + MAXARG + 1; /* MAXENV+1 */ + uint64 *stack_slots = estack + MAXENV + 1; /* the rest */ + _Static_assert((MAXARG + 1 + MAXENV + 1 + MAXARG + MAXENV + 32) * 8 <= 4096, + "exec scratch does not fit in one page"); uint64 max_sz = 0; uint64 rand_va = 0; @@ -97,6 +116,19 @@ exec(char *path, char **argv) ustack[argc] = sp; } + // The environment, copied the same way and for the same reason: these + // strings live in the *old* address space, which is still mapped, and will + // not be once the new pgdir is installed. + for(envc = 0; envp && envp[envc]; envc++) { + if(envc >= MAXENV) + goto bad; + sp -= strlen(envp[envc]) + 1; + sp &= ~7; + if(copyout(pgdir, sp, envp[envc], strlen(envp[envc]) + 1) < 0) + goto bad; + estack[envc] = sp; + } + // Push 16 pseudo-random bytes for AT_RANDOM. { uint8 randbuf[16]; @@ -117,13 +149,20 @@ exec(char *path, char **argv) } stack_slots[slot++] = 0; // end of argv - stack_slots[slot++] = 0; // end of envp (no environment) + // The environment used to be a single zero here, with the comment "(no + // environment)". Nothing on SchOS had one: not a shell, not a musl program + // that had just called execve with a perfectly good envp, not anything. So + // $TERM was unset, $PATH was whatever each program invented, and a variable + // exported in a shell did not survive the exec that was supposed to carry it. + for(i = 0; i < envc; i++) + stack_slots[slot++] = estack[i]; + stack_slots[slot++] = 0; // end of envp // Auxiliary vector — musl reads these for page size, clock, etc. stack_slots[slot++] = 6; // AT_PAGESZ stack_slots[slot++] = 4096; stack_slots[slot++] = 17; // AT_CLKTCK - stack_slots[slot++] = 100; + stack_slots[slot++] = USER_HZ; // what times() is scaled to, not a literal stack_slots[slot++] = 23; // AT_SECURE = 0 (not setuid) stack_slots[slot++] = 0; stack_slots[slot++] = 11; // AT_UID @@ -232,10 +271,12 @@ exec(char *path, char **argv) switchuvm(myproc()); freevm(oldpgdir); + kfree((char*)scratch); fsstat.exec_cycles += rdtsc() - _t0; return 0; bad: + kfree((char*)scratch); if(pgdir) freevm(pgdir); if(ip){ diff --git a/kernel/proc/proc.c b/kernel/proc/proc.c index 96678c0..ba37749 100644 --- a/kernel/proc/proc.c +++ b/kernel/proc/proc.c @@ -383,9 +383,20 @@ waitpid(int pid, int *status, int options) } } - if(!havekids || myproc()->killed){ + // Two different failures, and they used to share one bare -1. + // + // musl reads a return in [-4095, -1] as -errno, so -1 arrived in + // userspace as EPERM: "operation not permitted". zsh's compinit calls + // wait, got that, printed "wait failed: operation not permitted", and + // abandoned the rest of /etc/zsh/zshrc -- which looked like a broken + // config file rather than a wait() that could not say what went wrong. + if(!havekids){ release(&ptable.lock); - return -1; + return -ECHILD; // nothing to wait for, which is not an error + } + if(myproc()->killed){ + release(&ptable.lock); + return -EINTR; // a signal arrived while waiting } if (options & 1) { // WNOHANG diff --git a/kernel/sys/sys_fs.c b/kernel/sys/sys_fs.c index 61885e1..4adaf07 100644 --- a/kernel/sys/sys_fs.c +++ b/kernel/sys/sys_fs.c @@ -726,8 +726,8 @@ bad_unlink: iunlockput(dp); end_op(); return -1; addr_t sys_exec(void) { - char *path, *argv[MAXARG]; - addr_t uargv, uarg; + char *path, *argv[MAXARG], *envp[MAXENV]; + addr_t uargv, uenvp, uarg; if (argstr(0, &path) < 0 || argaddr(1, &uargv) < 0) return -EFAULT; memset(argv, 0, sizeof(argv)); for (int i = 0; ; i++) { @@ -736,7 +736,24 @@ addr_t sys_exec(void) if (!uarg) { argv[i] = 0; break; } if (fetchstr(uarg, &argv[i]) < 0) return -EFAULT; } - return exec(path, argv); + + // The third argument. This is Linux's execve(2), so musl has been passing + // an environment here all along -- it was simply never read, and every + // program on the system started with an empty one. + // + // A null envp is legal and means exactly that. It is also what the + // first-party exec() wrapper passes, so old callers keep their old + // behaviour without knowing about this. + memset(envp, 0, sizeof(envp)); + if (argaddr(2, &uenvp) == 0 && uenvp) { + for (int i = 0; ; i++) { + if (i >= NELEM(envp)) return -E2BIG; + if (fetchaddr(uenvp + sizeof(addr_t) * i, &uarg) < 0) return -EFAULT; + if (!uarg) { envp[i] = 0; break; } + if (fetchstr(uarg, &envp[i]) < 0) return -EFAULT; + } + } + return exec(path, argv, envp); } // ftruncate/truncate both grow and shrink. Growing used to be quietly dropped diff --git a/kernel/sys/sys_mem.c b/kernel/sys/sys_mem.c index 06e13a9..31222c1 100644 --- a/kernel/sys/sys_mem.c +++ b/kernel/sys/sys_mem.c @@ -95,7 +95,11 @@ sys_mmap(void) for (int i = 0; i < NVMA; i++) { if (!myproc()->vma[i].used) { v = &myproc()->vma[i]; break; } } - if (!v) return -1; + // -ENOMEM, not -1. A bare -1 reaches musl as -EPERM, so a process that + // had simply used all NVMA slots was told "operation not permitted" -- + // which sends you looking for a permission model this system does not + // have. malloc() turns either into NULL, but only one of them is a clue. + if (!v) return -ENOMEM; // Ignore non-zero hint addresses (not MAP_FIXED); always pick a fresh VA. // Bump-allocated: each mapping gets its own disjoint, exactly-length-sized diff --git a/lib/glue.c b/lib/glue.c index 3906114..e0cf063 100644 --- a/lib/glue.c +++ b/lib/glue.c @@ -22,3 +22,15 @@ __stack_chk_fail(void) _xv6_exit(); for (;;); } + +// exec() without an environment. +// +// The syscall is execve, three arguments, because that is what Linux numbers +// 59 and what musl calls. The two-argument spelling stays because most of +// cmd/ has no environment to pass and should not have to say so -- but it +// says so *here*, explicitly, rather than leaving %rdx to chance. +int +exec(char *path, char **argv) +{ + return execve(path, argv, 0); +} diff --git a/lib/usys.S b/lib/usys.S index 08b3ce8..990a20b 100644 --- a/lib/usys.S +++ b/lib/usys.S @@ -22,7 +22,15 @@ SYSCALL(read) SYSCALL(write) SYSCALL(close) SYSCALL(kill) -SYSCALL(exec) +/* The raw stub is execve: three arguments, the Linux shape. exec() is a C + wrapper in glue.c that passes a null environment, so the 2-argument callers + in cmd/ cannot leave %rdx holding whatever happened to be there. */ +.global execve +execve: + mov $SYS_exec, %rax + mov %rcx, %r10 + syscall + ret SYSCALL(open) SYSCALL(mknod) SYSCALL(unlink) diff --git a/ports/Makefile b/ports/Makefile index ef1781b..42128f1 100644 --- a/ports/Makefile +++ b/ports/Makefile @@ -6,6 +6,7 @@ # ports/mksh/ MirBSD Korn Shell — installed as sysroot/bin/sh # ports/toybox/ core utilities — installed as sysroot/bin/ # ports/ncurses/ terminfo + screen library — link-time only, ships nothing +# ports/zsh/ the Z shell — installed as sysroot/bin/zsh, beside /bin/sh # # A port directory holds only a Makefile and optional patches/. Everything # generated lives in build/ports// — see mk/port.mk for the contract. @@ -29,6 +30,7 @@ mksh: musl toybox: musl fastfetch: musl ncurses: musl +zsh: musl ncurses $(PORTS): $(MAKE) -C $@ diff --git a/ports/fastfetch/Makefile b/ports/fastfetch/Makefile index 4e890d6..121bb20 100644 --- a/ports/fastfetch/Makefile +++ b/ports/fastfetch/Makefile @@ -53,8 +53,10 @@ download: $(TARBALL) $(SRCDIR): $(TARBALL) mkdir -p $(PORTDIR) + rm -rf $@ $(PORTDIR)/fastfetch-$(FF_VERSION) tar -xzf $< -C $(PORTDIR) mv $(PORTDIR)/fastfetch-$(FF_VERSION) $@ + touch $@ extract: $(SRCDIR) diff --git a/ports/mksh/Makefile b/ports/mksh/Makefile index 3b8e17f..c758ea3 100644 --- a/ports/mksh/Makefile +++ b/ports/mksh/Makefile @@ -20,8 +20,10 @@ download: $(TARBALL) $(SRCDIR): $(TARBALL) mkdir -p $(PORTDIR) + rm -rf $@ $(PORTDIR)/mksh tar -xzf $< -C $(PORTDIR) mv $(PORTDIR)/mksh $@ + touch $@ # Order-only (|) on $(SRCDIR): the extracted tarball never changes, so a # directory mtime should not trigger a rebuild of an already-built shell. diff --git a/ports/ncurses/Makefile b/ports/ncurses/Makefile index 85f67fc..9caf01f 100644 --- a/ports/ncurses/Makefile +++ b/ports/ncurses/Makefile @@ -82,8 +82,10 @@ download: $(TARBALL) $(SRCDIR): $(TARBALL) mkdir -p $(PORTDIR) + rm -rf $@ $(PORTDIR)/ncurses-$(NC_VERSION) tar -xzf $< -C $(PORTDIR) mv $(PORTDIR)/ncurses-$(NC_VERSION) $@ + touch $@ extract: $(SRCDIR) @@ -106,7 +108,18 @@ $(PREFIX)/lib/libncursesw.a: $(BUILDDIR)/config.status $(MAKE) -C $(BUILDDIR) $(MAKE) -C $(BUILDDIR) install -build install: $(PREFIX)/lib/libncursesw.a +build: $(PREFIX)/lib/libncursesw.a + +# The acceptance test. It is the one thing this port puts in sysroot/, and it +# earns the space: the build proving the library compiles says nothing about +# whether terminfo works on a SchOS console, which is a different question +# with a different answer. sysroot/bin already carries socktest, nettest and +# shmtest for the same reason. +$(SYSROOT)/bin/tinfotest: tinfotest.c $(PREFIX)/lib/libncursesw.a | $(SYSROOT)/bin + $(HOSTCC_NC) $(NC_CPPFLAGS) -isystem $(PREFIX)/include $(NC_CFLAGS) $(NC_LDFLAGS) -L$(PREFIX)/lib -o $@ $< -lncursesw $(NC_LIBS) + chmod 755 $@ + +install: $(PREFIX)/lib/libncursesw.a $(SYSROOT)/bin/tinfotest clean: rm -rf $(PORTDIR) diff --git a/ports/zsh/Makefile b/ports/zsh/Makefile new file mode 100644 index 0000000..bb66da6 --- /dev/null +++ b/ports/zsh/Makefile @@ -0,0 +1,209 @@ +# SPDX-License-Identifier: Apache-2.0 +# zsh -- the Z shell, MIT-like: https://www.zsh.org/ +# +# Installed as /bin/zsh, *beside* mksh, which keeps /bin/sh. +# +# That is deliberate and it is not about taste. schinit's no-jobs fallback +# execs /bin/sh. If zsh were /bin/sh and zsh failed to start, the fallback +# would be the same broken binary and the machine would have no shell at all. +# Two different programs means the recovery path is a different program. +# It is also what real systems do: a small POSIX /bin/sh for scripts, a large +# interactive shell for people. docs/history/kernel.md section 2 says "exactly +# one name per role" -- these are two roles. +# +# --------------------------------------------------------------------------- +# Licensing +# +# zsh's LICENCE is MIT-like, with one condition that has to be honoured rather +# than noted: a handful of shell functions are GPL, and the licence says so +# explicitly, adding that "none of the core functions are affected by this, so +# those files may simply be omitted." +# +# So they are omitted. GPL_FILES below is that list -- four files, deleted +# after extraction and verified absent before install. Every one completes a +# command SchOS does not have (darcs, qdbus, osc, zypper), which is why this +# costs nothing at all. + +include ../mk/port.mk + +ZSH_VERSION := 5.9 +ZSH_URL := https://sourceforge.net/projects/zsh/files/zsh/$(ZSH_VERSION)/zsh-$(ZSH_VERSION).tar.xz/download +ZSH_SHA256 := 9b8d1ecedd5b5e81fbf1918e876752a7dd948e05c1a0dba10ab863842d45acd5 + +TARBALL := $(DISTDIR)/zsh-$(ZSH_VERSION).tar.xz + +# The GPL-licensed functions. Removed from the source tree, not merely skipped +# at install: a file that is not there cannot be shipped by a later edit to +# this Makefile. +# +# This list is four files, and it was audited rather than grepped. The obvious +# search -- `grep -rl GPL` -- returns three more, and all three are wrong: +# +# _composer completes SPDX licence identifiers, so the word +# "licence" is data, not a licence +# _luarocks has '--license=... (e.g. "MIT/X11" or "GNU GPL +# v3")' as a completion string, same thing +# VCS_INFO_get_data_hg says "Distributed under the same BSD-ish license +# as zsh itself" -- it matched on HGPLAIN, a +# Mercurial environment variable with GPL in the +# middle of it +# +# Deleting those three would cost nothing and would still be a false claim in +# a licensing file, which is the kind of thing that is believed later. +GPL_FILES := \ + Completion/Unix/Command/_darcs \ + Completion/Linux/Command/_qdbus \ + Completion/openSUSE/Command/_osc \ + Completion/openSUSE/Command/_zypper + +# Completion sets for systems that are not this one. Dropping them is size, +# not licensing -- together they are most of Completion/ and none of them can +# ever match a command that exists here. +FOREIGN_COMP := Completion/openSUSE Completion/Mandriva Completion/Redhat \ + Completion/Debian Completion/Solaris Completion/Cygwin \ + Completion/AIX Completion/BSD Completion/Darwin + +HOSTCC_ZSH ?= gcc +GCC_INC := $(shell $(HOSTCC_ZSH) -print-file-name=include) +LIBGCC := $(shell $(HOSTCC_ZSH) -print-libgcc-file-name) + +# Split the way autoconf consumes them -- see ports/ncurses/Makefile, which +# had to learn the same lesson: crt objects ride in LDFLAGS (before the object +# being linked), libraries in LIBS (after it). +ZSH_CPPFLAGS := -nostdinc -isystem $(MUSL)/include -isystem $(GCC_INC) \ + -isystem $(NCURSES)/include -D__SCHOS__ +# zsh 5.9 is from 2022; GCC 14 and 15 promoted three long-standing warnings to +# errors by default, and old autoconf projects fail their own feature probes on +# them. The probe that matters here is +# +# char **test = boolcodes; puts(*test); +# +# which ncurses answers with `const char * const boolcodes[]`. GCC 15 rejects +# the conversion outright, so the test "fails", HAVE_BOOLCODES goes undefined, +# and zsh helpfully declares its own `static char *boolcodes[]` -- which then +# collides with the declaration in term.h that was there the whole time. The +# error surfaces in termcap.c, three steps from the cause. +# +# These restore the diagnostic level the code was written against. They are +# scoped to this port and nothing else in the tree relaxes them. +ZSH_COMPAT := -Wno-incompatible-pointer-types -Wno-implicit-function-declaration \ + -Wno-int-conversion + +ZSH_CFLAGS := -O2 $(CFLAGS_BT) $(ZSH_COMPAT) +ZSH_LDFLAGS := -static -nostdlib -L$(MUSL)/lib -L$(NCURSES)/lib \ + $(MUSL)/lib/crt1.o $(MUSL)/lib/crti.o +ZSH_LIBS := -lncursesw $(MUSL)/lib/libc.a $(LIBGCC) $(MUSL)/lib/libc.a \ + $(MUSL)/lib/crtn.o + +# Answers configure cannot work out without running a program on the target. +# Each one is a fact about SchOS, so each one is written down here rather than +# guessed by autoconf's cross-compilation defaults. +ZSH_CACHE := \ + zsh_cv_shared_environ=yes \ + zsh_cv_shared_tgetent=no \ + zsh_cv_shared_tigetstr=no \ + zsh_cv_sys_nis=no \ + zsh_cv_sys_nss=no \ + zsh_cv_func_realpath_accepts_null=yes \ + zsh_cv_sys_path_dev_fd=/dev/fd \ + zsh_cv_sys_fifo=yes \ + zsh_cv_sys_killesrch=no \ + zsh_cv_sys_signal_handler=void \ + zsh_cv_prog_tcsetpgrp=yes \ + ac_cv_func_mmap_fixed_mapped=yes \ + ac_cv_func_setpgrp_void=yes + +# --disable-dynamic: SchOS links everything static, and zsh's module system +# would otherwise want dlopen. Every module is compiled in instead. +# --with-term-lib: name it, so configure cannot quietly decide it needs none +# and leave the line editor unable to move a cursor. +# --enable-etcdir: where zsh looks for zshenv/zshrc. NOT --sysconfdir, which +# autoconf accepts, zsh ignores, and configure reports no complaint about -- +# the shell simply starts with its default prompt and no completion, looking +# like a broken config file rather than a config file that was never read. +ZSH_CONFIGURE := \ + --host=x86_64-linux-musl \ + --build=x86_64-pc-linux-gnu \ + --prefix=/usr \ + --bindir=/bin \ + --enable-etcdir=/etc/zsh \ + --disable-dynamic \ + --enable-multibyte \ + --with-term-lib="ncursesw" \ + --disable-gdbm \ + --disable-pcre \ + --disable-cap \ + --disable-largefile \ + --enable-zsh-secure-free \ + --enable-fndir=/usr/share/zsh/$(ZSH_VERSION)/functions \ + --enable-scriptdir=/usr/share/zsh/$(ZSH_VERSION)/scripts + +$(TARBALL): | $(DISTDIR) + curl -fsSL -o $@ $(ZSH_URL) + sh $(SHA256CHECK) "$(ZSH_SHA256)" "$@" + +download: $(TARBALL) + +# rm -rf before the move, and touch after it. Both matter: +# +# Without the rm, `mv zsh-5.9 src` when src already exists moves the tree +# *inside* it, producing src/zsh-5.9 -- and zsh's configure globs */*/*.mdd, +# so it finds the nested copy and writes modfile=zsh-5.9/Src/zsh.mdd into +# config.modules. The build then fails a long way from the cause. +# +# Without the touch, this rule re-runs whenever the directory's mtime is +# older than the tarball's, which is how src came to exist in the first +# place. Editing files *inside* a directory does not update the directory. +$(SRCDIR): $(TARBALL) + mkdir -p $(PORTDIR) + rm -rf $@ $(PORTDIR)/zsh-$(ZSH_VERSION) + tar -xJf $< -C $(PORTDIR) + mv $(PORTDIR)/zsh-$(ZSH_VERSION) $@ + @echo "zsh: removing GPL-licensed functions named by zsh's own LICENCE" + @cd $@ && rm -f $(GPL_FILES) + @cd $@ && rm -rf $(FOREIGN_COMP) + touch $@ + +extract: $(SRCDIR) + +$(BUILDDIR)/config.status: | $(SRCDIR) + @test -f $(NCURSES)/lib/libncursesw.a || { \ + echo "zsh: build ports/ncurses first -- no libncursesw.a" >&2; exit 1; } + mkdir -p $(BUILDDIR) + cd $(BUILDDIR) && \ + CC="$(HOSTCC_ZSH)" \ + CPPFLAGS="$(ZSH_CPPFLAGS)" \ + CFLAGS="$(ZSH_CFLAGS)" \ + LDFLAGS="$(ZSH_LDFLAGS)" \ + LIBS="$(ZSH_LIBS)" \ + $(ZSH_CACHE) \ + $(SRCDIR)/configure $(ZSH_CONFIGURE) + +$(BUILDDIR)/Src/zsh: $(BUILDDIR)/config.status + $(MAKE) -C $(BUILDDIR) + +build: $(BUILDDIR)/Src/zsh + +# Refuse to install if a GPL file is somehow present. The rm above should have +# made this impossible; the check is here because "should have" is not a +# licence audit, and this runs before every image is built. +.PHONY: licence-check +licence-check: | $(SRCDIR) + @bad=0; for f in $(GPL_FILES); do \ + if [ -e "$(SRCDIR)/$$f" ]; then echo "zsh: GPL file present: $$f" >&2; bad=1; fi; \ + done; \ + if [ $$bad -ne 0 ]; then echo "zsh: refusing to install" >&2; exit 1; fi; \ + echo "zsh: no GPL-licensed files in the tree" + +$(SYSROOT)/bin/zsh: $(BUILDDIR)/Src/zsh licence-check | $(SYSROOT)/bin + $(MAKE) -C $(BUILDDIR) install DESTDIR=$(SYSROOT) + rm -rf $(SYSROOT)/usr/share/man $(SYSROOT)/usr/share/zsh/$(ZSH_VERSION)/help + chmod 755 $@ + +install: $(SYSROOT)/bin/zsh + +clean: + rm -rf $(PORTDIR) + +distclean: clean + rm -f $(TARBALL)