diff --git a/include/file.h b/include/file.h index 601ab85..97ba65c 100644 --- a/include/file.h +++ b/include/file.h @@ -21,6 +21,7 @@ struct file { // invite somebody to keep one. int memfd_seg; uint64 off; + struct file *fnext; // file.c's free list, while ref is 0 }; // --------------------------------------------------------------------------- @@ -105,6 +106,8 @@ struct inode { struct sleeplock lock; int valid; const struct fs_ops *ops; // chosen by iget from dev; never null once held + struct inode *hnext; // fs.c: hash chain while held, free list after + struct inode *anext; // fs.c: every inode there is, never reordered short type; short major; @@ -191,6 +194,10 @@ int sysfs_mount(const char *target); int iheld(uint dev, uint64 inum); // 1 if any inode on dev but `except` is referenced (0: no exception). In fs.c. int ibusy(uint dev, uint64 except); +// How many in-memory inodes exist; the table grows past NINODE. In fs.c. +uint inode_slots(void); +// How many open file descriptions exist; the table grows past NFILE. file.c. +uint file_slots(void); // Path lookup that follows a final symlink too; 0 with *err set on failure. // `start` 0 means the cwd. In fs.c. struct inode *nameifollow(struct inode *start, char *path, int *err); diff --git a/kernel/dev/procfs.c b/kernel/dev/procfs.c index c47f28d..7f3b678 100644 --- a/kernel/dev/procfs.c +++ b/kernel/dev/procfs.c @@ -295,10 +295,10 @@ gen_fsstat(struct sbuf *sb) // leak, and which line it is on says where to look. sb_puts(sb, "\ninode_used "); sb_putu(sb, fsstat.inode_used); sb_puts(sb, "\ninode_peak "); sb_putu(sb, fsstat.inode_peak); - sb_puts(sb, " / "); sb_putu(sb, NINODE); + sb_puts(sb, " / "); sb_putu(sb, inode_slots()); sb_puts(sb, "\nfile_used "); sb_putu(sb, fsstat.file_used); sb_puts(sb, "\nfile_peak "); sb_putu(sb, fsstat.file_peak); - sb_puts(sb, " / "); sb_putu(sb, NFILE); + sb_puts(sb, " / "); sb_putu(sb, file_slots()); sb_puts(sb, "\nbuf_pinned "); sb_putu(sb, fsstat.buf_pinned); { uint nb, maxb; diff --git a/kernel/fs/file.c b/kernel/fs/file.c index 12f4937..1aaa219 100644 --- a/kernel/fs/file.c +++ b/kernel/fs/file.c @@ -9,18 +9,38 @@ #include "fsstat.h" #include "posix.h" #include "errno.h" +#include "kmem.h" extern struct devsw devsw[]; +// Open file descriptions, shared by every process. NFILE of them in the +// kernel image and more from a slab cache once those are all open; a closed +// one goes on the free list. This was NFILE and then ENFILE, which a few +// processes each holding a hundred descriptors -- an editor, a build, a +// shell -- were already a third of the way to. struct { struct spinlock lock; struct file file[NFILE]; + struct file *free; + struct kmem_cache *more; + uint n; } ftable; void fileinit(void) { initlock(&ftable.lock, "ftable"); + for(int i = NFILE - 1; i >= 0; i--){ + ftable.file[i].fnext = ftable.free; + ftable.free = &ftable.file[i]; + } + ftable.n = NFILE; +} + +uint +file_slots(void) +{ + return ftable.n; } struct file* @@ -29,21 +49,28 @@ filealloc(void) struct file *f; acquire(&ftable.lock); - for(f = ftable.file; f < ftable.file + NFILE; f++){ - if(f->ref == 0){ - f->ref = 1; - FSSTAT_TAKE(fsstat.file_used, fsstat.file_peak); // 0 -> 1 - // Cleared here rather than left to the caller: every creator sets - // readable/writable/off, but only open() sets nonblock, so a struct file - // recycled from a non-blocking device would hand its flag to the next - // pipe or socket that happened to land in this slot. - f->nonblock = 0; + if((f = ftable.free) != 0){ + ftable.free = f->fnext; + } else { + // The slab allocator takes only its own lock and kalloc's. + if(ftable.more == 0) + ftable.more = kmem_cache_create("file", sizeof(struct file)); + if(ftable.more == 0 || (f = kmem_cache_alloc(ftable.more)) == 0){ release(&ftable.lock); - return f; + return 0; } + memset(f, 0, sizeof *f); + ftable.n++; } + f->ref = 1; + FSSTAT_TAKE(fsstat.file_used, fsstat.file_peak); // 0 -> 1 + // Cleared here rather than left to the caller: every creator sets + // readable/writable/off, but only open() sets nonblock, so a struct file + // recycled from a non-blocking device would hand its flag to the next + // pipe or socket that happened to land in this slot. + f->nonblock = 0; release(&ftable.lock); - return 0; + return f; } struct file* @@ -72,6 +99,8 @@ fileclose(struct file *f) ff = *f; f->ref = 0; f->type = FD_NONE; + f->fnext = ftable.free; + ftable.free = f; FSSTAT_DROP(fsstat.file_used); // 1 -> 0 release(&ftable.lock); diff --git a/kernel/fs/fs.c b/kernel/fs/fs.c index bd20f4c..e014336 100644 --- a/kernel/fs/fs.c +++ b/kernel/fs/fs.c @@ -45,6 +45,7 @@ #include "file.h" #include "fat.h" #include "errno.h" +#include "kmem.h" extern uint uptime(void); @@ -215,17 +216,80 @@ bfree(int dev, uint b) // Inode table // =========================================================================== +// Every inode somebody holds, found by a hash on (dev, inum). One with no +// reference goes on the free list and out of the hash, so a lookup never has +// to skip it. +// +// This was NINODE entries searched end to end on every iget(), and "iget: no +// inodes" -- a panic -- when all 200 were held. Held means open, a cwd, an +// exec'd binary, or a directory some path walk is in the middle of, so 128 +// processes with a few files each reached it. The table now starts as those +// NINODE and takes more from a slab cache when they are all held; it never +// gives them back, which costs 280 bytes for each inode that was ever held at +// once, and makes the panic mean "out of kernel memory". +#define INODE_NHASH 4096 // a power of two + struct { struct spinlock lock; struct inode inode[NINODE]; + struct inode *hash[INODE_NHASH]; + struct inode *free; // through hnext, ref 0 + struct inode *all; // through anext + struct kmem_cache *more; + uint n; } itable; +static inline uint +ihash(uint dev, uint64 inum) +{ + uint64 h = (inum ^ ((uint64)dev << 40)) * 0x9E3779B97F4A7C15ULL; + return (uint)(h >> 52) & (INODE_NHASH - 1); +} + +static void +inode_new(struct inode *ip) +{ + initsleeplock(&ip->lock, "inode"); + ip->anext = itable.all; + itable.all = ip; + ip->hnext = itable.free; + itable.free = ip; + itable.n++; +} + void iinit(void) { initlock(&itable.lock, "itable"); - for (int i = 0; i < NINODE; i++) - initsleeplock(&itable.inode[i].lock, "inode"); + for (int i = NINODE - 1; i >= 0; i--) + inode_new(&itable.inode[i]); +} + +// How many inode slots exist, for /proc/fsstat. +uint +inode_slots(void) +{ + return itable.n; +} + +// The held inode for (dev, inum), or 0. Under itable.lock. +static struct inode * +ifind(uint dev, uint64 inum) +{ + for (struct inode *ip = itable.hash[ihash(dev, inum)]; ip; ip = ip->hnext) + if (ip->dev == dev && ip->inum == inum) + return ip; + return 0; +} + +static void +iunhash(struct inode *ip) +{ + struct inode **pp = &itable.hash[ihash(ip->dev, ip->inum)]; + while (*pp && *pp != ip) + pp = &(*pp)->hnext; + if (*pp) + *pp = ip->hnext; } struct inode *iget(uint dev, uint64 inum); @@ -450,20 +514,28 @@ iget(uint dev, uint64 inum) struct inode *ip, *empty; acquire(&itable.lock); - empty = 0; - for (ip = &itable.inode[0]; ip < &itable.inode[NINODE]; ip++) { - if (ip->ref > 0 && ip->dev == dev && ip->inum == inum) { - ip->ref++; - release(&itable.lock); - return ip; - } - if (empty == 0 && ip->ref == 0) - empty = ip; + if ((ip = ifind(dev, inum)) != 0) { + ip->ref++; + release(&itable.lock); + return ip; + } + + if ((empty = itable.free) != 0) { + itable.free = empty->hnext; + } else { + // The slab allocator takes only its own lock and kalloc's. + if (itable.more == 0) + itable.more = kmem_cache_create("inode", sizeof(struct inode)); + if (itable.more == 0 || (empty = kmem_cache_alloc(itable.more)) == 0) + panic("iget: no inodes"); + memset(empty, 0, sizeof *empty); + inode_new(empty); + itable.free = empty->hnext; } - if (empty == 0) - panic("iget: no inodes"); ip = empty; + ip->hnext = itable.hash[ihash(dev, inum)]; + itable.hash[ihash(dev, inum)] = ip; ip->dev = dev; ip->inum = inum; ip->ref = 1; @@ -481,11 +553,7 @@ iheld(uint dev, uint64 inum) { int held = 0; acquire(&itable.lock); - for (struct inode *ip = &itable.inode[0]; ip < &itable.inode[NINODE]; ip++) - if (ip->ref > 0 && ip->dev == dev && ip->inum == inum) { - held = 1; - break; - } + held = ifind(dev, inum) != 0; release(&itable.lock); return held; } @@ -497,7 +565,7 @@ ibusy(uint dev, uint64 except) { int busy = 0; acquire(&itable.lock); - for (struct inode *ip = &itable.inode[0]; ip < &itable.inode[NINODE]; ip++) + for (struct inode *ip = itable.all; ip; ip = ip->anext) if (ip->ref > 0 && ip->dev == dev && ip->inum != except) { busy = 1; break; @@ -584,8 +652,12 @@ iput(struct inode *ip) acquire(&itable.lock); } - if (--ip->ref == 0) + if (--ip->ref == 0) { FSSTAT_DROP(fsstat.inode_used); + iunhash(ip); + ip->hnext = itable.free; + itable.free = ip; + } release(&itable.lock); } diff --git a/kernel/sys/sys_compat.c b/kernel/sys/sys_compat.c index 7342c66..3108531 100644 --- a/kernel/sys/sys_compat.c +++ b/kernel/sys/sys_compat.c @@ -74,6 +74,20 @@ sys_arch_prctl(void) return -EINVAL; } +// Limits are reported, not enforced: everything is unlimited except what the +// kernel actually has a ceiling for. RLIMIT_NOFILE was "unlimited" too, and +// musl turns that into sysconf(_SC_OPEN_MAX) == -1 -- after which zsh decided +// it had no descriptors at all, and `exec {fd}rlim_cur = r->rlim_max = (uint64)-1; + if (resource == RLIMIT_NOFILE_) + r->rlim_cur = r->rlim_max = NOFILE; +} + addr_t sys_getrlimit(void) { @@ -82,19 +96,18 @@ sys_getrlimit(void) argint(0, &resource); if (argptr(1, (char**)&rlp, sizeof(*rlp)) < 0) return -1; - rlp->rlim_cur = (uint64)-1; - rlp->rlim_max = (uint64)-1; + rlimit_of(resource, rlp); return 0; } addr_t sys_prlimit(void) { + int resource; struct rlimit *oldrlp; - if (argptr(3, (char**)&oldrlp, sizeof(*oldrlp)) == 0 && oldrlp) { - oldrlp->rlim_cur = (uint64)-1; - oldrlp->rlim_max = (uint64)-1; - } + argint(1, &resource); + if (argptr(3, (char**)&oldrlp, sizeof(*oldrlp)) == 0 && oldrlp) + rlimit_of(resource, oldrlp); return 0; } diff --git a/kernel/sys/sys_fs.c b/kernel/sys/sys_fs.c index 269f0f9..92d1075 100644 --- a/kernel/sys/sys_fs.c +++ b/kernel/sys/sys_fs.c @@ -16,7 +16,7 @@ #include "errno.h" #include "schfs.h" /* SCHFS_BLKSIZE, SCHFS_NAME_MAX — statfs reports them */ #include "gfx.h" /* gfx_cols/gfx_rows — TIOCGWINSZ reports them */ -#include "vt.h" /* the terminals TIOC*/VT_* act on */ +#include "vt.h" /* the terminals the TIOC and VT_ ioctls act on */ #include "pciuser.h" #include "keymap.h" /* KB_*LAYOUT — the keyboard layout catalogue */ #include "netuser.h"