Index: kern/sys_pipe.c =================================================================== RCS file: /cvs/src/sys/kern/sys_pipe.c,v retrieving revision 1.13 diff -u -r1.13 sys_pipe.c --- kern/sys_pipe.c 3 Nov 2003 17:11:21 -0000 1.13 +++ kern/sys_pipe.c 17 Feb 2004 06:58:48 -0000 @@ -52,6 +52,7 @@ #include #include +#include #include #include #include @@ -67,6 +68,10 @@ #include #include #include +#include +#include +#include +#include #include #include @@ -118,6 +123,7 @@ static struct filterops pipe_wfiltops = { 1, NULL, filt_pipedetach, filt_pipewrite }; +MALLOC_DEFINE(M_PIPE, "pipe", "pipe structures"); /* * Default pipe buffer size(s), this can be kind-of large now because pipe @@ -144,9 +150,36 @@ * Limit the number of "big" pipes */ #define LIMITBIGPIPES 32 -static int nbigpipe; +#define PIPEQ_MAX_CACHE 16 /* per-cpu pipe structure cache */ -static int amountpipekva; +static int pipe_maxbig = LIMITBIGPIPES; +static int pipe_maxcache = PIPEQ_MAX_CACHE; +static int pipe_nbig; +static int pipe_kva; +static int pipe_bcache_alloc; +static int pipe_bkmem_alloc; +static int pipe_dcache_alloc; +static int pipe_dkmem_alloc; + +SYSCTL_NODE(_kern, OID_AUTO, pipe, CTLFLAG_RW, 0, "Pipe operation"); +SYSCTL_INT(_kern_pipe, OID_AUTO, nbig, + CTLFLAG_RD, &pipe_nbig, 0, "numer of big pipes allocated"); +SYSCTL_INT(_kern_pipe, OID_AUTO, kva, + CTLFLAG_RD, &pipe_kva, 0, "kva reserved by pipes"); +SYSCTL_INT(_kern_pipe, OID_AUTO, maxcache, + CTLFLAG_RW, &pipe_maxcache, 0, "max pipes cached per-cpu"); +SYSCTL_INT(_kern_pipe, OID_AUTO, maxbig, + CTLFLAG_RW, &pipe_maxbig, 0, "max number of big pipes"); +#if !defined(NO_PIPE_SYSCTL_STATS) +SYSCTL_INT(_kern_pipe, OID_AUTO, bcache_alloc, + CTLFLAG_RW, &pipe_bcache_alloc, 0, "pipe buffer from pcpu cache"); +SYSCTL_INT(_kern_pipe, OID_AUTO, dcache_alloc, + CTLFLAG_RW, &pipe_dcache_alloc, 0, "pipe direct buf from pcpu cache"); +SYSCTL_INT(_kern_pipe, OID_AUTO, bkmem_alloc, + CTLFLAG_RW, &pipe_bkmem_alloc, 0, "pipe buffer from kmem"); +SYSCTL_INT(_kern_pipe, OID_AUTO, dkmem_alloc, + CTLFLAG_RW, &pipe_dkmem_alloc, 0, "pipe direct buf from kmem"); +#endif static void pipeclose (struct pipe *cpipe); static void pipe_free_kmem (struct pipe *cpipe); @@ -162,8 +195,6 @@ #endif static int pipespace (struct pipe *cpipe, int size); -static vm_zone_t pipe_zone; - /* * The pipe system call for the DTYPE_PIPE type of pipes * @@ -184,9 +215,6 @@ KKASSERT(p); fdp = p->p_fd; - if (pipe_zone == NULL) - pipe_zone = zinit("PIPE", sizeof(struct pipe), 0, 0, 4); - rpipe = wpipe = NULL; if (pipe_create(&rpipe) || pipe_create(&wpipe)) { pipeclose(rpipe); @@ -247,98 +275,76 @@ * If it fails it will return ENOMEM. */ static int -pipespace(cpipe, size) - struct pipe *cpipe; - int size; +pipespace(struct pipe *cpipe, int size) { struct vm_object *object; caddr_t buffer; int npages, error; - npages = round_page(size)/PAGE_SIZE; - /* - * Create an object, I don't like the idea of paging to/from - * kernel_object. - * XXX -- minor change needed here for NetBSD/OpenBSD VM systems. - */ - object = vm_object_allocate(OBJT_DEFAULT, npages); - buffer = (caddr_t) vm_map_min(kernel_map); + npages = round_page(size) / PAGE_SIZE; + object = cpipe->pipe_buffer.object; /* - * Insert the object into the kernel map, and allocate kva for it. - * The map entry is, by default, pageable. - * XXX -- minor change needed here for NetBSD/OpenBSD VM systems. + * [re]create the object if necessary and reserve space for it + * in the kernel_map. The object and memory are pageable. On + * success, free the old resources before assigning the new + * ones. */ - error = vm_map_find(kernel_map, object, 0, - (vm_offset_t *) &buffer, size, 1, - VM_PROT_ALL, VM_PROT_ALL, 0); - - if (error != KERN_SUCCESS) { - vm_object_deallocate(object); - return (ENOMEM); + if (object == NULL || object->size != npages) { + object = vm_object_allocate(OBJT_DEFAULT, npages); + buffer = (caddr_t) vm_map_min(kernel_map); + + error = vm_map_find(kernel_map, object, 0, + (vm_offset_t *) &buffer, size, 1, + VM_PROT_ALL, VM_PROT_ALL, 0); + + if (error != KERN_SUCCESS) { + vm_object_deallocate(object); + return (ENOMEM); + } + pipe_kva += size; + pipe_free_kmem(cpipe); + cpipe->pipe_buffer.object = object; + cpipe->pipe_buffer.buffer = buffer; + cpipe->pipe_buffer.size = size; + ++pipe_bkmem_alloc; + } else { + ++pipe_bcache_alloc; + if (cpipe->pipe_map.kva) + ++pipe_dcache_alloc; } - - /* free old resources if we're resizing */ - pipe_free_kmem(cpipe); - cpipe->pipe_buffer.object = object; - cpipe->pipe_buffer.buffer = buffer; - cpipe->pipe_buffer.size = size; cpipe->pipe_buffer.in = 0; cpipe->pipe_buffer.out = 0; cpipe->pipe_buffer.cnt = 0; - amountpipekva += cpipe->pipe_buffer.size; return (0); } /* - * initialize and allocate VM and memory for pipe + * Initialize and allocate VM and memory for pipe, pulling the pipe from + * our per-cpu cache if possible. For now make sure it is sized for the + * smaller PIPE_SIZE default. */ static int pipe_create(cpipep) struct pipe **cpipep; { + globaldata_t gd = mycpu; struct pipe *cpipe; int error; - *cpipep = zalloc(pipe_zone); - if (*cpipep == NULL) - return (ENOMEM); - - cpipe = *cpipep; - - /* so pipespace()->pipe_free_kmem() doesn't follow junk pointer */ - cpipe->pipe_buffer.object = NULL; -#ifndef PIPE_NODIRECT - cpipe->pipe_map.kva = NULL; -#endif - /* - * protect so pipeclose() doesn't follow a junk pointer - * if pipespace() fails. - */ - bzero(&cpipe->pipe_sel, sizeof(cpipe->pipe_sel)); - cpipe->pipe_state = 0; - cpipe->pipe_peer = NULL; - cpipe->pipe_busy = 0; - -#ifndef PIPE_NODIRECT - /* - * pipe data structure initializations to support direct pipe I/O - */ - cpipe->pipe_map.cnt = 0; - cpipe->pipe_map.kva = 0; - cpipe->pipe_map.pos = 0; - cpipe->pipe_map.npages = 0; - /* cpipe->pipe_map.ms[] = invalid */ -#endif - - error = pipespace(cpipe, PIPE_SIZE); - if (error) + if ((cpipe = gd->gd_pipeq) != NULL) { + gd->gd_pipeq = cpipe->pipe_peer; + --gd->gd_pipeqcount; + cpipe->pipe_peer = NULL; + } else { + cpipe = malloc(sizeof(struct pipe), M_PIPE, M_WAITOK|M_ZERO); + } + *cpipep = cpipe; + if ((error = pipespace(cpipe, PIPE_SIZE)) != 0) return (error); - vfs_timestamp(&cpipe->pipe_ctime); cpipe->pipe_atime = cpipe->pipe_ctime; cpipe->pipe_mtime = cpipe->pipe_ctime; - return (0); } @@ -593,7 +599,8 @@ */ wpipe->pipe_map.kva = kmem_alloc_pageable(kernel_map, wpipe->pipe_buffer.size + PAGE_SIZE); - amountpipekva += wpipe->pipe_buffer.size + PAGE_SIZE; + pipe_kva += wpipe->pipe_buffer.size + PAGE_SIZE; + ++pipe_dkmem_alloc; } pmap_qenter(wpipe->pipe_map.kva, wpipe->pipe_map.ms, wpipe->pipe_map.npages); @@ -623,12 +630,12 @@ if (wpipe->pipe_map.kva) { pmap_qremove(wpipe->pipe_map.kva, wpipe->pipe_map.npages); - if (amountpipekva > MAXPIPEKVA) { + if (pipe_kva > MAXPIPEKVA) { vm_offset_t kva = wpipe->pipe_map.kva; wpipe->pipe_map.kva = 0; kmem_free(kernel_map, kva, wpipe->pipe_buffer.size + PAGE_SIZE); - amountpipekva -= wpipe->pipe_buffer.size + PAGE_SIZE; + pipe_kva -= wpipe->pipe_buffer.size + PAGE_SIZE; } } for (i = 0; i < wpipe->pipe_map.npages; i++) @@ -777,14 +784,14 @@ * so. */ if ((uio->uio_resid > PIPE_SIZE) && - (nbigpipe < LIMITBIGPIPES) && + (pipe_nbig < pipe_maxbig) && (wpipe->pipe_state & PIPE_DIRECTW) == 0 && (wpipe->pipe_buffer.size <= PIPE_SIZE) && (wpipe->pipe_buffer.cnt == 0)) { if ((error = pipelock(wpipe,1)) == 0) { if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) - nbigpipe++; + pipe_nbig++; pipeunlock(wpipe); } } @@ -822,7 +829,7 @@ */ if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && (fp->f_flag & FNONBLOCK) == 0 && - (wpipe->pipe_map.kva || (amountpipekva < LIMITPIPEKVA)) && + (wpipe->pipe_map.kva || (pipe_kva < LIMITPIPEKVA)) && (uio->uio_iov->iov_len >= PIPE_MINDIRECT)) { error = pipe_direct_write( wpipe, uio); if (error) @@ -1145,19 +1152,19 @@ static void pipe_free_kmem(struct pipe *cpipe) { - if (cpipe->pipe_buffer.buffer != NULL) { if (cpipe->pipe_buffer.size > PIPE_SIZE) - --nbigpipe; - amountpipekva -= cpipe->pipe_buffer.size; + --pipe_nbig; + pipe_kva -= cpipe->pipe_buffer.size; kmem_free(kernel_map, (vm_offset_t)cpipe->pipe_buffer.buffer, cpipe->pipe_buffer.size); cpipe->pipe_buffer.buffer = NULL; + cpipe->pipe_buffer.object = NULL; } #ifndef PIPE_NODIRECT if (cpipe->pipe_map.kva != NULL) { - amountpipekva -= cpipe->pipe_buffer.size + PAGE_SIZE; + pipe_kva -= cpipe->pipe_buffer.size + PAGE_SIZE; kmem_free(kernel_map, cpipe->pipe_map.kva, cpipe->pipe_buffer.size + PAGE_SIZE); @@ -1175,38 +1182,55 @@ static void pipeclose(struct pipe *cpipe) { + globaldata_t gd; struct pipe *ppipe; - if (cpipe) { - - pipeselwakeup(cpipe); + if (cpipe == NULL) + return; - /* - * If the other side is blocked, wake it up saying that - * we want to close it down. - */ - while (cpipe->pipe_busy) { - wakeup(cpipe); - cpipe->pipe_state |= PIPE_WANT | PIPE_EOF; - tsleep(cpipe, 0, "pipecl", 0); - } + pipeselwakeup(cpipe); - /* - * Disconnect from peer - */ - if ((ppipe = cpipe->pipe_peer) != NULL) { - pipeselwakeup(ppipe); + /* + * If the other side is blocked, wake it up saying that + * we want to close it down. + */ + while (cpipe->pipe_busy) { + wakeup(cpipe); + cpipe->pipe_state |= PIPE_WANT | PIPE_EOF; + tsleep(cpipe, 0, "pipecl", 0); + } - ppipe->pipe_state |= PIPE_EOF; - wakeup(ppipe); - KNOTE(&ppipe->pipe_sel.si_note, 0); - ppipe->pipe_peer = NULL; - } - /* - * free resources - */ + /* + * Disconnect from peer + */ + if ((ppipe = cpipe->pipe_peer) != NULL) { + pipeselwakeup(ppipe); + + ppipe->pipe_state |= PIPE_EOF; + wakeup(ppipe); + KNOTE(&ppipe->pipe_sel.si_note, 0); + ppipe->pipe_peer = NULL; + } + + /* + * free or cache resources + */ + gd = mycpu; + if (gd->gd_pipeqcount >= pipe_maxcache || + cpipe->pipe_buffer.size != PIPE_SIZE + ) { pipe_free_kmem(cpipe); - zfree(pipe_zone, cpipe); + free(cpipe, M_PIPE); + } else { + KKASSERT(cpipe->pipe_map.npages == 0); + + cpipe->pipe_state = 0; + cpipe->pipe_busy = 0; + cpipe->pipe_map.cnt = 0; + cpipe->pipe_map.pos = 0; + cpipe->pipe_peer = gd->gd_pipeq; + gd->gd_pipeq = cpipe; + ++gd->gd_pipeqcount; } } Index: sys/globaldata.h =================================================================== RCS file: /cvs/src/sys/sys/globaldata.h,v retrieving revision 1.25 diff -u -r1.25 globaldata.h --- sys/globaldata.h 15 Feb 2004 02:14:42 -0000 1.25 +++ sys/globaldata.h 17 Feb 2004 02:27:18 -0000 @@ -81,6 +81,7 @@ struct sysmsg; struct privatespace; struct vm_map_entry; +struct pipe; struct globaldata { struct privatespace *gd_prvspace; /* self-reference */ @@ -115,6 +116,9 @@ struct systimer gd_schedclock; /* scheduler periodic */ volatile __uint32_t gd_time_seconds; /* uptime in seconds */ volatile sysclock_t gd_cpuclock_base; /* cpuclock relative base */ + + struct pipe *gd_pipeq; /* cache pipe structures */ + int gd_pipeqcount; /* number of structures */ /* extended by */ }; Index: sys/pipe.h =================================================================== RCS file: /cvs/src/sys/sys/pipe.h,v retrieving revision 1.2 diff -u -r1.2 pipe.h --- sys/pipe.h 17 Jun 2003 04:28:58 -0000 1.2 +++ sys/pipe.h 17 Feb 2004 06:56:24 -0000 @@ -64,7 +64,7 @@ u_int out; /* out pointer */ u_int size; /* size of buffer */ caddr_t buffer; /* kva of buffer */ - struct vm_object *object; /* VM object containing buffer */ + struct vm_object *object; /* VM object containing buffer */ }; /*