From: Dipankar Sarma <dipankar@in.ibm.com>
fget() shows up on profiles, especially on SMP. Dipankar's patch
special-cases the situation wherein there are no sharers of current->files.
In this situation we know that no other process can close this file, so it
is not necessary to increment the file's refcount.
It's ugly as sin, but makes a substantial difference.
The test is
dd if=/dev/zero of=foo bs=1 count=1M
On 4CPU P3 xeon with 1MB L2 cache and 512MB ram:
kernel sys time std-dev
------------ -------- -------
UP - vanilla 2.104 0.028
UP - file 1.867 0.019
SMP - vanilla 2.976 0.023
SMP - file 2.719 0.026
fs/file_table.c | 30 +++++++++++++++++++++++++++++-
fs/read_write.c | 49 +++++++++++++++++++++++++++++--------------------
include/linux/file.h | 8 ++++++++
3 files changed, 66 insertions(+), 21 deletions(-)
diff -puN fs/file_table.c~fget-speedup fs/file_table.c
--- 25/fs/file_table.c~fget-speedup 2003-05-04 23:41:42.000000000 -0700
+++ 25-akpm/fs/file_table.c 2003-05-04 23:41:46.000000000 -0700
@@ -141,7 +141,7 @@ void close_private_file(struct file *fil
security_file_free(file);
}
-void fput(struct file * file)
+void fput(struct file *file)
{
if (atomic_dec_and_test(&file->f_count))
__fput(file);
@@ -190,6 +190,34 @@ struct file *fget(unsigned int fd)
return file;
}
+/*
+ * Lightweight file lookup - no refcnt increment if fd table isn't shared.
+ * You can use this only if it is guranteed that the current task already
+ * holds a refcnt to that file. That check has to be done at fget() only
+ * and a flag is returned to be passed to the corresponding fput_light().
+ * There must not be a cloning between an fget_light/fput_light pair.
+ */
+struct file *fget_light(unsigned int fd, int *fput_needed)
+{
+ struct file *file;
+ struct files_struct *files = current->files;
+
+ *fput_needed = 0;
+ if (likely((atomic_read(&files->count) == 1))) {
+ file = fcheck(fd);
+ } else {
+ read_lock(&files->file_lock);
+ file = fcheck(fd);
+ if (file) {
+ get_file(file);
+ *fput_needed = 1;
+ }
+ read_unlock(&files->file_lock);
+ }
+ return file;
+}
+
+
void put_filp(struct file *file)
{
if (atomic_dec_and_test(&file->f_count)) {
diff -puN fs/read_write.c~fget-speedup fs/read_write.c
--- 25/fs/read_write.c~fget-speedup 2003-05-04 23:41:42.000000000 -0700
+++ 25-akpm/fs/read_write.c 2003-05-04 23:41:42.000000000 -0700
@@ -115,9 +115,10 @@ asmlinkage off_t sys_lseek(unsigned int
{
off_t retval;
struct file * file;
+ int fput_needed;
retval = -EBADF;
- file = fget(fd);
+ file = fget_light(fd, &fput_needed);
if (!file)
goto bad;
@@ -128,7 +129,7 @@ asmlinkage off_t sys_lseek(unsigned int
if (res != (loff_t)retval)
retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
}
- fput(file);
+ fput_light(file, fput_needed);
bad:
return retval;
}
@@ -141,9 +142,10 @@ asmlinkage long sys_llseek(unsigned int
int retval;
struct file * file;
loff_t offset;
+ int fput_needed;
retval = -EBADF;
- file = fget(fd);
+ file = fget_light(fd, &fput_needed);
if (!file)
goto bad;
@@ -161,7 +163,7 @@ asmlinkage long sys_llseek(unsigned int
retval = 0;
}
out_putf:
- fput(file);
+ fput_light(file, fput_needed);
bad:
return retval;
}
@@ -251,11 +253,12 @@ asmlinkage ssize_t sys_read(unsigned int
{
struct file *file;
ssize_t ret = -EBADF;
+ int fput_needed;
- file = fget(fd);
+ file = fget_light(fd, &fput_needed);
if (file) {
ret = vfs_read(file, buf, count, &file->f_pos);
- fput(file);
+ fput_light(file, fput_needed);
}
return ret;
@@ -265,11 +268,12 @@ asmlinkage ssize_t sys_write(unsigned in
{
struct file *file;
ssize_t ret = -EBADF;
+ int fput_needed;
- file = fget(fd);
+ file = fget_light(fd, &fput_needed);
if (file) {
ret = vfs_write(file, buf, count, &file->f_pos);
- fput(file);
+ fput_light(file, fput_needed);
}
return ret;
@@ -280,14 +284,15 @@ asmlinkage ssize_t sys_pread64(unsigned
{
struct file *file;
ssize_t ret = -EBADF;
+ int fput_needed;
if (pos < 0)
return -EINVAL;
- file = fget(fd);
+ file = fget_light(fd, &fput_needed);
if (file) {
ret = vfs_read(file, buf, count, &pos);
- fput(file);
+ fput_light(file, fput_needed);
}
return ret;
@@ -298,14 +303,15 @@ asmlinkage ssize_t sys_pwrite64(unsigned
{
struct file *file;
ssize_t ret = -EBADF;
+ int fput_needed;
if (pos < 0)
return -EINVAL;
- file = fget(fd);
+ file = fget_light(fd, &fput_needed);
if (file) {
ret = vfs_write(file, buf, count, &pos);
- fput(file);
+ fput_light(file, fput_needed);
}
return ret;
@@ -479,11 +485,12 @@ sys_readv(unsigned long fd, const struct
{
struct file *file;
ssize_t ret = -EBADF;
+ int fput_needed;
- file = fget(fd);
+ file = fget_light(fd, &fput_needed);
if (file) {
ret = vfs_readv(file, vec, vlen, &file->f_pos);
- fput(file);
+ fput_light(file, fput_needed);
}
return ret;
@@ -494,11 +501,12 @@ sys_writev(unsigned long fd, const struc
{
struct file *file;
ssize_t ret = -EBADF;
+ int fput_needed;
- file = fget(fd);
+ file = fget_light(fd, &fput_needed);
if (file) {
ret = vfs_writev(file, vec, vlen, &file->f_pos);
- fput(file);
+ fput_light(file, fput_needed);
}
return ret;
@@ -511,12 +519,13 @@ static ssize_t do_sendfile(int out_fd, i
struct inode * in_inode, * out_inode;
loff_t pos;
ssize_t retval;
+ int fput_needed_in, fput_needed_out;
/*
* Get input file, and verify that it is ok..
*/
retval = -EBADF;
- in_file = fget(in_fd);
+ in_file = fget_light(in_fd, &fput_needed_in);
if (!in_file)
goto out;
if (!(in_file->f_mode & FMODE_READ))
@@ -539,7 +548,7 @@ static ssize_t do_sendfile(int out_fd, i
* Get output file, and verify that it is ok..
*/
retval = -EBADF;
- out_file = fget(out_fd);
+ out_file = fget_light(out_fd, &fput_needed_out);
if (!out_file)
goto fput_in;
if (!(out_file->f_mode & FMODE_WRITE))
@@ -579,9 +588,9 @@ static ssize_t do_sendfile(int out_fd, i
retval = -EOVERFLOW;
fput_out:
- fput(out_file);
+ fput_light(out_file, fput_needed_out);
fput_in:
- fput(in_file);
+ fput_light(in_file, fput_needed_in);
out:
return retval;
}
diff -puN include/linux/file.h~fget-speedup include/linux/file.h
--- 25/include/linux/file.h~fget-speedup 2003-05-04 23:41:42.000000000 -0700
+++ 25-akpm/include/linux/file.h 2003-05-04 23:41:46.000000000 -0700
@@ -35,7 +35,15 @@ struct files_struct {
extern void FASTCALL(__fput(struct file *));
extern void FASTCALL(fput(struct file *));
+
+static inline void fput_light(struct file *file, int fput_needed)
+{
+ if (unlikely(fput_needed))
+ fput(file);
+}
+
extern struct file * FASTCALL(fget(unsigned int fd));
+extern struct file * FASTCALL(fget_light(unsigned int fd, int *fput_needed));
extern void FASTCALL(set_close_on_exec(unsigned int fd, int flag));
extern void put_filp(struct file *);
extern int get_unused_fd(void);
_