Skip to content

Commit

Permalink
Enabled writeback caching on Linux, greatly improving write performance.
Browse files Browse the repository at this point in the history
The effect is particularly exaggerated (42x) for small writes.

Fixes #113.

% go install -v && gcsfuse --temp-dir /mnt/ssd0 jacobsa-standard-asia ~/mp
% go build ./benchmarks/write_locally && ./write_locally --dir ~/mp --write_size 4096

Before:

    Wrote 67052 times (261.92 MiB) in 10.000022709s (6705.2 Hz, 26.19 MiB/s)

After:

    Wrote 2860685 times (10.91 GiB) in 10.002095148s (286008.6 Hz, 1.09 GiB/s)

% go install -v && gcsfuse --temp-dir /mnt/ssd0 jacobsa-standard-asia ~/mp
% go build ./benchmarks/write_locally && ./write_locally --dir ~/mp

Before:

    Wrote 3724 times (3.64 GiB) in 10.00007858s (372.4 Hz, 372.40 MiB/s)

After:

    Wrote 12764 times (12.46 GiB) in 10.000301535s (1276.4 Hz, 1.25 GiB/s)
  • Loading branch information
jacobsa committed Aug 13, 2015
2 parents 8ecfe19 + 9033692 commit 2cda84d
Show file tree
Hide file tree
Showing 18 changed files with 293 additions and 162 deletions.
15 changes: 8 additions & 7 deletions internal/fs/caching_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/jacobsa/gcloud/gcs/gcsutil"
. "github.com/jacobsa/oglematchers"
. "github.com/jacobsa/ogletest"
"github.com/jacobsa/timeutil"
)

////////////////////////////////////////////////////////////////////////
Expand All @@ -44,14 +45,14 @@ type cachingTestCommon struct {
func (t *cachingTestCommon) SetUp(ti *TestInfo) {
// Wrap the bucket in a stat caching layer for the purposes of the file
// system.
t.uncachedBucket = gcsfake.NewFakeBucket(&t.clock, "some_bucket")
t.uncachedBucket = gcsfake.NewFakeBucket(timeutil.RealClock(), "some_bucket")

const statCacheCapacity = 1000
statCache := gcscaching.NewStatCache(statCacheCapacity)
t.bucket = gcscaching.NewFastStatBucket(
ttl,
statCache,
&t.clock,
&t.cacheClock,
t.uncachedBucket)

// Enable directory type caching.
Expand Down Expand Up @@ -149,7 +150,7 @@ func (t *CachingTest) FileChangedRemotely() {
ExpectEq(len("taco"), fi.Size())

// After the TTL elapses, we should see the new version.
t.clock.AdvanceTime(ttl + time.Millisecond)
t.cacheClock.AdvanceTime(ttl + time.Millisecond)

fi, err = os.Stat(path.Join(t.Dir, name))
AssertEq(nil, err)
Expand Down Expand Up @@ -183,7 +184,7 @@ func (t *CachingTest) DirectoryRemovedRemotely() {
ExpectTrue(fi.IsDir())

// After the TTL elapses, we should see it disappear.
t.clock.AdvanceTime(ttl + time.Millisecond)
t.cacheClock.AdvanceTime(ttl + time.Millisecond)

_, err = os.Stat(path.Join(t.Dir, name))
ExpectTrue(os.IsNotExist(err), "err: %v", err)
Expand Down Expand Up @@ -217,7 +218,7 @@ func (t *CachingTest) ConflictingNames_RemoteModifier() {
ExpectTrue(os.IsNotExist(err), "err: %v", err)

// After the TTL elapses, we should see both.
t.clock.AdvanceTime(ttl + time.Millisecond)
t.cacheClock.AdvanceTime(ttl + time.Millisecond)

fi, err = os.Stat(path.Join(t.Dir, name))
AssertEq(nil, err)
Expand Down Expand Up @@ -282,7 +283,7 @@ func (t *CachingTest) TypeOfNameChanges_RemoteModifier() {
ExpectTrue(os.IsNotExist(err), "err: %v", err)

// After the TTL elapses, we should see it turn into a file.
t.clock.AdvanceTime(ttl + time.Millisecond)
t.cacheClock.AdvanceTime(ttl + time.Millisecond)

fi, err = os.Stat(path.Join(t.Dir, name))
AssertEq(nil, err)
Expand Down Expand Up @@ -405,7 +406,7 @@ func (t *CachingWithImplicitDirsTest) SymlinksAreTypeCached() {
ExpectEq(filePerms|os.ModeSymlink, fi.Mode())

// After the TTL elapses, we should see the directory.
t.clock.AdvanceTime(ttl + time.Millisecond)
t.cacheClock.AdvanceTime(ttl + time.Millisecond)
fi, err = os.Lstat(path.Join(t.Dir, "foo"))

AssertEq(nil, err)
Expand Down
18 changes: 6 additions & 12 deletions internal/fs/foreign_modifications_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func (t *ForeignModsTest) ReadDir_EmptyRoot() {

func (t *ForeignModsTest) ReadDir_ContentsInRoot() {
// Set up contents.
createTime := t.clock.Now()
createTime := t.mtimeClock.Now()
AssertEq(
nil,
t.createObjects(
Expand All @@ -113,9 +113,6 @@ func (t *ForeignModsTest) ReadDir_ContentsInRoot() {
"baz": "burrito",
}))

// Make sure the time below doesn't match.
t.clock.AdvanceTime(time.Second)

/////////////////////////
// ReadDir
/////////////////////////
Expand All @@ -141,7 +138,7 @@ func (t *ForeignModsTest) ReadDir_ContentsInRoot() {
ExpectEq("baz", e.Name())
ExpectEq(len("burrito"), e.Size())
ExpectEq(filePerms, e.Mode())
ExpectThat(e.ModTime(), timeutil.TimeEq(createTime))
ExpectThat(e, fusetesting.MtimeIsWithin(createTime, timeSlop))
ExpectFalse(e.IsDir())
ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
Expand All @@ -152,7 +149,7 @@ func (t *ForeignModsTest) ReadDir_ContentsInRoot() {
ExpectEq("foo", e.Name())
ExpectEq(len("taco"), e.Size())
ExpectEq(filePerms, e.Mode())
ExpectThat(e.ModTime(), timeutil.TimeEq(createTime))
ExpectThat(e, fusetesting.MtimeIsWithin(createTime, timeSlop))
ExpectFalse(e.IsDir())
ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
Expand All @@ -176,7 +173,7 @@ func (t *ForeignModsTest) ReadDir_EmptySubDirectory() {

func (t *ForeignModsTest) ReadDir_ContentsInSubDirectory() {
// Set up contents.
createTime := t.clock.Now()
createTime := t.mtimeClock.Now()
AssertEq(
nil,
t.createObjects(
Expand All @@ -194,9 +191,6 @@ func (t *ForeignModsTest) ReadDir_ContentsInSubDirectory() {
"dir/baz": "burrito",
}))

// Make sure the time below doesn't match.
t.clock.AdvanceTime(time.Second)

// Wait for the directory to show up in the file system.
_, err := fusetesting.ReadDirPicky(path.Join(t.mfs.Dir()))
AssertEq(nil, err)
Expand All @@ -223,7 +217,7 @@ func (t *ForeignModsTest) ReadDir_ContentsInSubDirectory() {
ExpectEq("baz", e.Name())
ExpectEq(len("burrito"), e.Size())
ExpectEq(filePerms, e.Mode())
ExpectThat(e.ModTime(), timeutil.TimeEq(createTime))
ExpectThat(e, fusetesting.MtimeIsWithin(createTime, timeSlop))
ExpectFalse(e.IsDir())
ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
Expand All @@ -234,7 +228,7 @@ func (t *ForeignModsTest) ReadDir_ContentsInSubDirectory() {
ExpectEq("foo", e.Name())
ExpectEq(len("taco"), e.Size())
ExpectEq(filePerms, e.Mode())
ExpectThat(e.ModTime(), timeutil.TimeEq(createTime))
ExpectThat(e, fusetesting.MtimeIsWithin(createTime, timeSlop))
ExpectFalse(e.IsDir())
ExpectEq(1, e.Sys().(*syscall.Stat_t).Nlink)
ExpectEq(currentUid(), e.Sys().(*syscall.Stat_t).Uid)
Expand Down
26 changes: 16 additions & 10 deletions internal/fs/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,9 @@ import (
)

type ServerConfig struct {
// A clock used for modification times and cache expiration.
Clock timeutil.Clock
// A clock used for cache expiration. It is *not* used for inode times, for
// which we use the wall clock.
CacheClock timeutil.Clock

// The bucket that the file system is to export.
Bucket gcs.Bucket
Expand Down Expand Up @@ -133,7 +134,8 @@ func NewServer(cfg *ServerConfig) (server fuse.Server, err error) {

// Set up the basic struct.
fs := &fileSystem{
clock: cfg.Clock,
mtimeClock: timeutil.RealClock(),
cacheClock: cfg.CacheClock,
bucket: cfg.Bucket,
syncer: syncer,
tempDir: cfg.TempDir,
Expand Down Expand Up @@ -163,7 +165,8 @@ func NewServer(cfg *ServerConfig) (server fuse.Server, err error) {
fs.implicitDirs,
fs.dirTypeCacheTTL,
cfg.Bucket,
fs.clock)
fs.mtimeClock,
fs.cacheClock)

root.Lock()
root.IncrementLookupCount()
Expand Down Expand Up @@ -216,9 +219,10 @@ type fileSystem struct {
// Dependencies
/////////////////////////

clock timeutil.Clock
bucket gcs.Bucket
syncer gcsx.Syncer
mtimeClock timeutil.Clock
cacheClock timeutil.Clock
bucket gcs.Bucket
syncer gcsx.Syncer

/////////////////////////
// Constant data
Expand Down Expand Up @@ -508,7 +512,8 @@ func (fs *fileSystem) mintInode(name string, o *gcs.Object) (in inode.Inode) {
fs.implicitDirs,
fs.dirTypeCacheTTL,
fs.bucket,
fs.clock)
fs.mtimeClock,
fs.cacheClock)

// Implicit directories
case inode.IsDirName(name):
Expand All @@ -523,7 +528,8 @@ func (fs *fileSystem) mintInode(name string, o *gcs.Object) (in inode.Inode) {
fs.implicitDirs,
fs.dirTypeCacheTTL,
fs.bucket,
fs.clock)
fs.mtimeClock,
fs.cacheClock)

case inode.IsSymlink(o):
in = inode.NewSymlinkInode(
Expand All @@ -547,7 +553,7 @@ func (fs *fileSystem) mintInode(name string, o *gcs.Object) (in inode.Inode) {
fs.bucket,
fs.syncer,
fs.tempDir,
fs.clock)
fs.mtimeClock)
}

// Place it in our map of IDs to inodes.
Expand Down
14 changes: 8 additions & 6 deletions internal/fs/fs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,9 @@ type fsTest struct {

// Dependencies. If bucket is set before SetUp is called, it will be used
// rather than creating a default one.
clock timeutil.SimulatedClock
bucket gcs.Bucket
mtimeClock timeutil.Clock
cacheClock timeutil.SimulatedClock
bucket gcs.Bucket

// Mount information
mfs *fuse.MountedFileSystem
Expand All @@ -96,13 +97,14 @@ func (t *fsTest) SetUp(ti *TestInfo) {
var err error
t.ctx = ti.Ctx

// Set up the clock.
t.clock.SetTime(time.Date(2015, 4, 5, 2, 15, 0, 0, time.Local))
t.serverCfg.Clock = &t.clock
// Set up the clocks.
t.mtimeClock = timeutil.RealClock()
t.cacheClock.SetTime(time.Date(2015, 4, 5, 2, 15, 0, 0, time.Local))
t.serverCfg.CacheClock = &t.cacheClock

// And the bucket.
if t.bucket == nil {
t.bucket = gcsfake.NewFakeBucket(&t.clock, "some_bucket")
t.bucket = gcsfake.NewFakeBucket(t.mtimeClock, "some_bucket")
}

t.serverCfg.Bucket = t.bucket
Expand Down
31 changes: 17 additions & 14 deletions internal/fs/inode/dir.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,9 @@ type dirInode struct {
// Dependencies
/////////////////////////

bucket gcs.Bucket
clock timeutil.Clock
bucket gcs.Bucket
mtimeClock timeutil.Clock
cacheClock timeutil.Clock

/////////////////////////
// Constant data
Expand Down Expand Up @@ -210,7 +211,8 @@ func NewDirInode(
implicitDirs bool,
typeCacheTTL time.Duration,
bucket gcs.Bucket,
clock timeutil.Clock) (d DirInode) {
mtimeClock timeutil.Clock,
cacheClock timeutil.Clock) (d DirInode) {
if !IsDirName(name) {
panic(fmt.Sprintf("Unexpected name: %s", name))
}
Expand All @@ -219,7 +221,8 @@ func NewDirInode(
const typeCacheCapacity = 1 << 16
typed := &dirInode{
bucket: bucket,
clock: clock,
mtimeClock: mtimeClock,
cacheClock: cacheClock,
id: id,
implicitDirs: implicitDirs,
name: name,
Expand Down Expand Up @@ -463,7 +466,7 @@ func (d *dirInode) filterMissingChildDirs(

// First add any names that we already know are directories according to our
// cache, removing them from the input.
now := d.clock.Now()
now := d.cacheClock.Now()
var tmp []string
for _, name := range in {
if d.cache.IsDir(now, name) {
Expand Down Expand Up @@ -532,7 +535,7 @@ func (d *dirInode) filterMissingChildDirs(
err = b.Join()

// Update the cache with everything we learned.
now = d.clock.Now()
now = d.cacheClock.Now()
for _, name := range filteredSlice {
d.cache.NoteDir(now, name)
}
Expand Down Expand Up @@ -604,7 +607,7 @@ func (d *dirInode) LookUpChild(
name string) (result LookUpResult, err error) {
// Consult the cache about the type of the child. This may save us work
// below.
now := d.clock.Now()
now := d.cacheClock.Now()
cacheSaysFile := d.cache.IsFile(now, name)
cacheSaysDir := d.cache.IsDir(now, name)

Expand Down Expand Up @@ -651,7 +654,7 @@ func (d *dirInode) LookUpChild(
}

// Update the cache.
now = d.clock.Now()
now = d.cacheClock.Now()
if fileResult.Exists() {
d.cache.NoteFile(now, name)
}
Expand Down Expand Up @@ -727,7 +730,7 @@ func (d *dirInode) ReadEntries(
newTok = listing.ContinuationToken

// Update the type cache with everything we learned.
now := d.clock.Now()
now := d.cacheClock.Now()
for _, e := range entries {
switch e.Type {
case fuseutil.DT_File:
Expand All @@ -746,15 +749,15 @@ func (d *dirInode) CreateChildFile(
ctx context.Context,
name string) (o *gcs.Object, err error) {
metadata := map[string]string{
FileMtimeMetadataKey: d.clock.Now().UTC().Format(time.RFC3339Nano),
FileMtimeMetadataKey: d.mtimeClock.Now().UTC().Format(time.RFC3339Nano),
}

o, err = d.createNewObject(ctx, path.Join(d.Name(), name), metadata)
if err != nil {
return
}

d.cache.NoteFile(d.clock.Now(), name)
d.cache.NoteFile(d.cacheClock.Now(), name)

return
}
Expand All @@ -781,7 +784,7 @@ func (d *dirInode) CloneToChildFile(
}

// Update the type cache.
d.cache.NoteFile(d.clock.Now(), name)
d.cache.NoteFile(d.cacheClock.Now(), name)

return
}
Expand All @@ -800,7 +803,7 @@ func (d *dirInode) CreateChildSymlink(
return
}

d.cache.NoteFile(d.clock.Now(), name)
d.cache.NoteFile(d.cacheClock.Now(), name)

return
}
Expand All @@ -814,7 +817,7 @@ func (d *dirInode) CreateChildDir(
return
}

d.cache.NoteDir(d.clock.Now(), name)
d.cache.NoteDir(d.cacheClock.Now(), name)

return
}
Expand Down
1 change: 1 addition & 0 deletions internal/fs/inode/dir_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ func (t *DirTest) resetInode(implicitDirs bool) {
implicitDirs,
typeCacheTTL,
t.bucket,
&t.clock,
&t.clock)

t.in.Lock()
Expand Down
6 changes: 4 additions & 2 deletions internal/fs/inode/explicit_dir.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,17 @@ func NewExplicitDirInode(
implicitDirs bool,
typeCacheTTL time.Duration,
bucket gcs.Bucket,
clock timeutil.Clock) (d ExplicitDirInode) {
mtimeClock timeutil.Clock,
cacheClock timeutil.Clock) (d ExplicitDirInode) {
wrapped := NewDirInode(
id,
o.Name,
attrs,
implicitDirs,
typeCacheTTL,
bucket,
clock)
mtimeClock,
cacheClock)

d = &explicitDirInode{
dirInode: wrapped.(*dirInode),
Expand Down
Loading

0 comments on commit 2cda84d

Please sign in to comment.