finder/fs/fs.go

// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"sync"
	"time"
)

var OsFs FileSystem = osFs{}

func NewMockFs(files map[string][]byte) *MockFs {
	workDir := "/cwd"
	fs := &MockFs{
		Clock:   NewClock(time.Unix(2, 2)),
		workDir: workDir,
	}
	fs.root = *fs.newDir()
	fs.MkDirs(workDir)

	for path, bytes := range files {
		dir := filepath.Dir(path)
		fs.MkDirs(dir)
		fs.WriteFile(path, bytes, 0777)
	}

	return fs
}

type FileSystem interface {
	// getting information about files
	Open(name string) (file io.ReadCloser, err error)
	Lstat(path string) (stats os.FileInfo, err error)
	ReadDir(path string) (contents []DirEntryInfo, err error)

	InodeNumber(info os.FileInfo) (number uint64, err error)
	DeviceNumber(info os.FileInfo) (number uint64, err error)
	PermTime(info os.FileInfo) (time time.Time, err error)

	// changing contents of the filesystem
	Rename(oldPath string, newPath string) (err error)
	WriteFile(path string, data []byte, perm os.FileMode) (err error)
	Remove(path string) (err error)
	RemoveAll(path string) (err error)

	// metadata about the filesystem
	ViewId() (id string) // Some unique id of the user accessing the filesystem
}

// DentryInfo is a subset of the functionality available through os.FileInfo that might be able
// to be gleaned through only a syscall.Getdents without requiring a syscall.Lstat of every file.
type DirEntryInfo interface {
	Name() string
	Mode() os.FileMode // the file type encoded as an os.FileMode
	IsDir() bool
}

type dirEntryInfo struct {
	name       string
	mode       os.FileMode
	modeExists bool
}

var _ DirEntryInfo = os.FileInfo(nil)

func (d *dirEntryInfo) Name() string      { return d.name }
func (d *dirEntryInfo) Mode() os.FileMode { return d.mode }
func (d *dirEntryInfo) IsDir() bool       { return d.mode.IsDir() }
func (d *dirEntryInfo) String() string    { return d.name + ": " + d.mode.String() }

// osFs implements FileSystem using the local disk.
type osFs struct{}

var _ FileSystem = (*osFs)(nil)

func (osFs) Open(name string) (io.ReadCloser, error) { return os.Open(name) }

func (osFs) Lstat(path string) (stats os.FileInfo, err error) {
	return os.Lstat(path)
}

func (osFs) ReadDir(path string) (contents []DirEntryInfo, err error) {
	entries, err := readdir(path)
	if err != nil {
		return nil, err
	}
	for _, entry := range entries {
		contents = append(contents, entry)
	}

	return contents, nil
}

func (osFs) Rename(oldPath string, newPath string) error {
	return os.Rename(oldPath, newPath)
}

func (osFs) WriteFile(path string, data []byte, perm os.FileMode) error {
	return ioutil.WriteFile(path, data, perm)
}

func (osFs) Remove(path string) error {
	return os.Remove(path)
}

func (osFs) RemoveAll(path string) error {
	return os.RemoveAll(path)
}

func (osFs) ViewId() (id string) {
	user, err := user.Current()
	if err != nil {
		return ""
	}
	username := user.Username

	hostname, err := os.Hostname()
	if err != nil {
		return ""
	}

	return username + "@" + hostname
}

type Clock struct {
	time time.Time
}

func NewClock(startTime time.Time) *Clock {
	return &Clock{time: startTime}

}

func (c *Clock) Tick() {
	c.time = c.time.Add(time.Microsecond)
}

func (c *Clock) Time() time.Time {
	return c.time
}

// given "/a/b/c/d", pathSplit returns ("/a/b/c", "d")
func pathSplit(path string) (dir string, leaf string) {
	dir, leaf = filepath.Split(path)
	if dir != "/" && len(dir) > 0 {
		dir = dir[:len(dir)-1]
	}
	return dir, leaf
}

// MockFs supports singlethreaded writes and multithreaded reads
type MockFs struct {
	// configuration
	viewId       string //
	deviceNumber uint64

	// implementation
	root            mockDir
	Clock           *Clock
	workDir         string
	nextInodeNumber uint64

	// history of requests, for tests to check
	StatCalls      []string
	ReadDirCalls   []string
	aggregatesLock sync.Mutex
}

var _ FileSystem = (*MockFs)(nil)

type mockInode struct {
	modTime     time.Time
	permTime    time.Time
	sys         interface{}
	inodeNumber uint64
	readErr     error
}

func (m mockInode) ModTime() time.Time {
	return m.modTime
}

func (m mockInode) Sys() interface{} {
	return m.sys
}

type mockFile struct {
	bytes []byte

	mockInode
}

type mockLink struct {
	target string

	mockInode
}

type mockDir struct {
	mockInode

	subdirs  map[string]*mockDir
	files    map[string]*mockFile
	symlinks map[string]*mockLink
}

func (m *MockFs) resolve(path string, followLastLink bool) (result string, err error) {
	if !filepath.IsAbs(path) {
		path = filepath.Join(m.workDir, path)
	}
	path = filepath.Clean(path)

	return m.followLinks(path, followLastLink, 10)
}

// note that followLinks can return a file path that doesn't exist
func (m *MockFs) followLinks(path string, followLastLink bool, count int) (canonicalPath string, err error) {
	if path == "/" {
		return path, nil
	}

	parentPath, leaf := pathSplit(path)
	if parentPath == path {
		err = fmt.Errorf("Internal error: %v yields itself as a parent", path)
		panic(err.Error())
		return "", fmt.Errorf("Internal error: %v yields itself as a parent", path)
	}

	parentPath, err = m.followLinks(parentPath, true, count)
	if err != nil {
		return "", err
	}

	parentNode, err := m.getDir(parentPath, false)
	if err != nil {
		return "", err
	}
	if parentNode.readErr != nil {
		return "", &os.PathError{
			Op:   "read",
			Path: path,
			Err:  parentNode.readErr,
		}
	}

	link, isLink := parentNode.symlinks[leaf]
	if isLink && followLastLink {
		if count <= 0 {
			// probably a loop
			return "", &os.PathError{
				Op:   "read",
				Path: path,
				Err:  fmt.Errorf("too many levels of symbolic links"),
			}
		}

		if link.readErr != nil {
			return "", &os.PathError{
				Op:   "read",
				Path: path,
				Err:  link.readErr,
			}
		}

		target := m.followLink(link, parentPath)
		return m.followLinks(target, followLastLink, count-1)
	}
	return path, nil
}

func (m *MockFs) followLink(link *mockLink, parentPath string) (result string) {
	return filepath.Clean(filepath.Join(parentPath, link.target))
}

func (m *MockFs) getFile(parentDir *mockDir, fileName string) (file *mockFile, err error) {
	file, isFile := parentDir.files[fileName]
	if !isFile {
		_, isDir := parentDir.subdirs[fileName]
		_, isLink := parentDir.symlinks[fileName]
		if isDir || isLink {
			return nil, &os.PathError{
				Op:   "open",
				Path: fileName,
				Err:  os.ErrInvalid,
			}
		}

		return nil, &os.PathError{
			Op:   "open",
			Path: fileName,
			Err:  os.ErrNotExist,
		}
	}
	if file.readErr != nil {
		return nil, &os.PathError{
			Op:   "open",
			Path: fileName,
			Err:  file.readErr,
		}
	}
	return file, nil
}

func (m *MockFs) getInode(parentDir *mockDir, name string) (inode *mockInode, err error) {
	file, isFile := parentDir.files[name]
	if isFile {
		return &file.mockInode, nil
	}
	link, isLink := parentDir.symlinks[name]
	if isLink {
		return &link.mockInode, nil
	}
	dir, isDir := parentDir.subdirs[name]
	if isDir {
		return &dir.mockInode, nil
	}
	return nil, &os.PathError{
		Op:   "stat",
		Path: name,
		Err:  os.ErrNotExist,
	}

}

func (m *MockFs) Open(path string) (io.ReadCloser, error) {
	path, err := m.resolve(path, true)
	if err != nil {
		return nil, err
	}

	if err != nil {
		return nil, err
	}

	parentPath, base := pathSplit(path)
	parentDir, err := m.getDir(parentPath, false)
	if err != nil {
		return nil, err
	}
	file, err := m.getFile(parentDir, base)
	if err != nil {
		return nil, err
	}
	return struct {
		io.Closer
		*bytes.Reader
	}{
		ioutil.NopCloser(nil),
		bytes.NewReader(file.bytes),
	}, nil

}

// a mockFileInfo is for exporting file stats in a way that satisfies the FileInfo interface
type mockFileInfo struct {
	path         string
	size         int64
	modTime      time.Time // time at which the inode's contents were modified
	permTime     time.Time // time at which the inode's permissions were modified
	isDir        bool
	inodeNumber  uint64
	deviceNumber uint64
}

func (m *mockFileInfo) Name() string {
	return m.path
}

func (m *mockFileInfo) Size() int64 {
	return m.size
}

func (m *mockFileInfo) Mode() os.FileMode {
	return 0
}

func (m *mockFileInfo) ModTime() time.Time {
	return m.modTime
}

func (m *mockFileInfo) IsDir() bool {
	return m.isDir
}

func (m *mockFileInfo) Sys() interface{} {
	return nil
}

func (m *MockFs) dirToFileInfo(d *mockDir, path string) (info *mockFileInfo) {
	return &mockFileInfo{
		path:         path,
		size:         1,
		modTime:      d.modTime,
		permTime:     d.permTime,
		isDir:        true,
		inodeNumber:  d.inodeNumber,
		deviceNumber: m.deviceNumber,
	}

}

func (m *MockFs) fileToFileInfo(f *mockFile, path string) (info *mockFileInfo) {
	return &mockFileInfo{
		path:         path,
		size:         1,
		modTime:      f.modTime,
		permTime:     f.permTime,
		isDir:        false,
		inodeNumber:  f.inodeNumber,
		deviceNumber: m.deviceNumber,
	}
}

func (m *MockFs) linkToFileInfo(l *mockLink, path string) (info *mockFileInfo) {
	return &mockFileInfo{
		path:         path,
		size:         1,
		modTime:      l.modTime,
		permTime:     l.permTime,
		isDir:        false,
		inodeNumber:  l.inodeNumber,
		deviceNumber: m.deviceNumber,
	}
}

func (m *MockFs) Lstat(path string) (stats os.FileInfo, err error) {
	// update aggregates
	m.aggregatesLock.Lock()
	m.StatCalls = append(m.StatCalls, path)
	m.aggregatesLock.Unlock()

	// resolve symlinks
	path, err = m.resolve(path, false)
	if err != nil {
		return nil, err
	}

	// special case for root dir
	if path == "/" {
		return m.dirToFileInfo(&m.root, "/"), nil
	}

	// determine type and handle appropriately
	parentPath, baseName := pathSplit(path)
	dir, err := m.getDir(parentPath, false)
	if err != nil {
		return nil, err
	}
	subdir, subdirExists := dir.subdirs[baseName]
	if subdirExists {
		return m.dirToFileInfo(subdir, path), nil
	}
	file, fileExists := dir.files[baseName]
	if fileExists {
		return m.fileToFileInfo(file, path), nil
	}
	link, linkExists := dir.symlinks[baseName]
	if linkExists {
		return m.linkToFileInfo(link, path), nil
	}
	// not found
	return nil, &os.PathError{
		Op:   "stat",
		Path: path,
		Err:  os.ErrNotExist,
	}
}

func (m *MockFs) InodeNumber(info os.FileInfo) (number uint64, err error) {
	mockInfo, ok := info.(*mockFileInfo)
	if ok {
		return mockInfo.inodeNumber, nil
	}
	return 0, fmt.Errorf("%v is not a mockFileInfo", info)
}
func (m *MockFs) DeviceNumber(info os.FileInfo) (number uint64, err error) {
	mockInfo, ok := info.(*mockFileInfo)
	if ok {
		return mockInfo.deviceNumber, nil
	}
	return 0, fmt.Errorf("%v is not a mockFileInfo", info)
}
func (m *MockFs) PermTime(info os.FileInfo) (when time.Time, err error) {
	mockInfo, ok := info.(*mockFileInfo)
	if ok {
		return mockInfo.permTime, nil
	}
	return time.Date(0, 0, 0, 0, 0, 0, 0, nil),
		fmt.Errorf("%v is not a mockFileInfo", info)
}

func (m *MockFs) ReadDir(path string) (contents []DirEntryInfo, err error) {
	// update aggregates
	m.aggregatesLock.Lock()
	m.ReadDirCalls = append(m.ReadDirCalls, path)
	m.aggregatesLock.Unlock()

	// locate directory
	path, err = m.resolve(path, true)
	if err != nil {
		return nil, err
	}
	results := []DirEntryInfo{}
	dir, err := m.getDir(path, false)
	if err != nil {
		return nil, err
	}
	if dir.readErr != nil {
		return nil, &os.PathError{
			Op:   "read",
			Path: path,
			Err:  dir.readErr,
		}
	}
	// describe its contents
	for name, subdir := range dir.subdirs {
		dirInfo := m.dirToFileInfo(subdir, name)
		results = append(results, dirInfo)
	}
	for name, file := range dir.files {
		info := m.fileToFileInfo(file, name)
		results = append(results, info)
	}
	for name, link := range dir.symlinks {
		info := m.linkToFileInfo(link, name)
		results = append(results, info)
	}
	return results, nil
}

func (m *MockFs) Rename(sourcePath string, destPath string) error {
	// validate source parent exists
	sourcePath, err := m.resolve(sourcePath, false)
	if err != nil {
		return err
	}
	sourceParentPath := filepath.Dir(sourcePath)
	sourceParentDir, err := m.getDir(sourceParentPath, false)
	if err != nil {
		return err
	}
	if sourceParentDir == nil {
		return &os.PathError{
			Op:   "move",
			Path: sourcePath,
			Err:  os.ErrNotExist,
		}
	}
	if sourceParentDir.readErr != nil {
		return &os.PathError{
			Op:   "move",
			Path: sourcePath,
			Err:  sourceParentDir.readErr,
		}
	}

	// validate dest parent exists
	destPath, err = m.resolve(destPath, false)
	destParentPath := filepath.Dir(destPath)
	destParentDir, err := m.getDir(destParentPath, false)
	if err != nil {
		return err
	}
	if destParentDir == nil {
		return &os.PathError{
			Op:   "move",
			Path: destParentPath,
			Err:  os.ErrNotExist,
		}
	}
	if destParentDir.readErr != nil {
		return &os.PathError{
			Op:   "move",
			Path: destParentPath,
			Err:  destParentDir.readErr,
		}
	}
	// check the source and dest themselves
	sourceBase := filepath.Base(sourcePath)
	destBase := filepath.Base(destPath)

	file, sourceIsFile := sourceParentDir.files[sourceBase]
	dir, sourceIsDir := sourceParentDir.subdirs[sourceBase]
	link, sourceIsLink := sourceParentDir.symlinks[sourceBase]

	// validate that the source exists
	if !sourceIsFile && !sourceIsDir && !sourceIsLink {
		return &os.PathError{
			Op:   "move",
			Path: sourcePath,
			Err:  os.ErrNotExist,
		}

	}

	// validate the destination doesn't already exist as an incompatible type
	_, destWasFile := destParentDir.files[destBase]
	_, destWasDir := destParentDir.subdirs[destBase]
	_, destWasLink := destParentDir.symlinks[destBase]

	if destWasDir {
		return &os.PathError{
			Op:   "move",
			Path: destPath,
			Err:  errors.New("destination exists as a directory"),
		}
	}

	if sourceIsDir && (destWasFile || destWasLink) {
		return &os.PathError{
			Op:   "move",
			Path: destPath,
			Err:  errors.New("destination exists as a file"),
		}
	}

	if destWasFile {
		delete(destParentDir.files, destBase)
	}
	if destWasDir {
		delete(destParentDir.subdirs, destBase)
	}
	if destWasLink {
		delete(destParentDir.symlinks, destBase)
	}

	if sourceIsFile {
		destParentDir.files[destBase] = file
		delete(sourceParentDir.files, sourceBase)
	}
	if sourceIsDir {
		destParentDir.subdirs[destBase] = dir
		delete(sourceParentDir.subdirs, sourceBase)
	}
	if sourceIsLink {
		destParentDir.symlinks[destBase] = link
		delete(destParentDir.symlinks, sourceBase)
	}

	destParentDir.modTime = m.Clock.Time()
	sourceParentDir.modTime = m.Clock.Time()
	return nil
}

func (m *MockFs) newInodeNumber() uint64 {
	result := m.nextInodeNumber
	m.nextInodeNumber++
	return result
}

func (m *MockFs) WriteFile(filePath string, data []byte, perm os.FileMode) error {
	filePath, err := m.resolve(filePath, true)
	if err != nil {
		return err
	}
	parentPath := filepath.Dir(filePath)
	parentDir, err := m.getDir(parentPath, false)
	if err != nil || parentDir == nil {
		return &os.PathError{
			Op:   "write",
			Path: parentPath,
			Err:  os.ErrNotExist,
		}
	}
	if parentDir.readErr != nil {
		return &os.PathError{
			Op:   "write",
			Path: parentPath,
			Err:  parentDir.readErr,
		}
	}

	baseName := filepath.Base(filePath)
	_, exists := parentDir.files[baseName]
	if !exists {
		parentDir.modTime = m.Clock.Time()
		parentDir.files[baseName] = m.newFile()
	} else {
		readErr := parentDir.files[baseName].readErr
		if readErr != nil {
			return &os.PathError{
				Op:   "write",
				Path: filePath,
				Err:  readErr,
			}
		}
	}
	file := parentDir.files[baseName]
	file.bytes = data
	file.modTime = m.Clock.Time()
	return nil
}

func (m *MockFs) newFile() *mockFile {
	newFile := &mockFile{}
	newFile.inodeNumber = m.newInodeNumber()
	newFile.modTime = m.Clock.Time()
	newFile.permTime = newFile.modTime
	return newFile
}

func (m *MockFs) newDir() *mockDir {
	newDir := &mockDir{
		subdirs:  make(map[string]*mockDir, 0),
		files:    make(map[string]*mockFile, 0),
		symlinks: make(map[string]*mockLink, 0),
	}
	newDir.inodeNumber = m.newInodeNumber()
	newDir.modTime = m.Clock.Time()
	newDir.permTime = newDir.modTime
	return newDir
}

func (m *MockFs) newLink(target string) *mockLink {
	newLink := &mockLink{
		target: target,
	}
	newLink.inodeNumber = m.newInodeNumber()
	newLink.modTime = m.Clock.Time()
	newLink.permTime = newLink.modTime

	return newLink
}
func (m *MockFs) MkDirs(path string) error {
	_, err := m.getDir(path, true)
	return err
}

// getDir doesn't support symlinks
func (m *MockFs) getDir(path string, createIfMissing bool) (dir *mockDir, err error) {
	cleanedPath := filepath.Clean(path)
	if cleanedPath == "/" {
		return &m.root, nil
	}

	parentPath, leaf := pathSplit(cleanedPath)
	if len(parentPath) >= len(path) {
		return &m.root, nil
	}
	parent, err := m.getDir(parentPath, createIfMissing)
	if err != nil {
		return nil, err
	}
	if parent.readErr != nil {
		return nil, &os.PathError{
			Op:   "stat",
			Path: path,
			Err:  parent.readErr,
		}
	}
	childDir, dirExists := parent.subdirs[leaf]
	if !dirExists {
		if createIfMissing {
			// confirm that a file with the same name doesn't already exist
			_, fileExists := parent.files[leaf]
			if fileExists {
				return nil, &os.PathError{
					Op:   "mkdir",
					Path: path,
					Err:  os.ErrExist,
				}
			}
			// create this directory
			childDir = m.newDir()
			parent.subdirs[leaf] = childDir
			parent.modTime = m.Clock.Time()
		} else {
			return nil, &os.PathError{
				Op:   "stat",
				Path: path,
				Err:  os.ErrNotExist,
			}
		}
	}
	return childDir, nil

}

func (m *MockFs) Remove(path string) (err error) {
	path, err = m.resolve(path, false)
	parentPath, leaf := pathSplit(path)
	if len(leaf) == 0 {
		return fmt.Errorf("Cannot remove %v\n", path)
	}
	parentDir, err := m.getDir(parentPath, false)
	if err != nil {
		return err
	}
	if parentDir == nil {
		return &os.PathError{
			Op:   "remove",
			Path: path,
			Err:  os.ErrNotExist,
		}
	}
	if parentDir.readErr != nil {
		return &os.PathError{
			Op:   "remove",
			Path: path,
			Err:  parentDir.readErr,
		}
	}
	_, isDir := parentDir.subdirs[leaf]
	if isDir {
		return &os.PathError{
			Op:   "remove",
			Path: path,
			Err:  os.ErrInvalid,
		}
	}
	_, isLink := parentDir.symlinks[leaf]
	if isLink {
		delete(parentDir.symlinks, leaf)
	} else {
		_, isFile := parentDir.files[leaf]
		if !isFile {
			return &os.PathError{
				Op:   "remove",
				Path: path,
				Err:  os.ErrNotExist,
			}
		}
		delete(parentDir.files, leaf)
	}
	parentDir.modTime = m.Clock.Time()
	return nil
}

func (m *MockFs) Symlink(oldPath string, newPath string) (err error) {
	newPath, err = m.resolve(newPath, false)
	if err != nil {
		return err
	}

	newParentPath, leaf := pathSplit(newPath)
	newParentDir, err := m.getDir(newParentPath, false)
	if newParentDir.readErr != nil {
		return &os.PathError{
			Op:   "link",
			Path: newPath,
			Err:  newParentDir.readErr,
		}
	}
	if err != nil {
		return err
	}
	newParentDir.symlinks[leaf] = m.newLink(oldPath)
	return nil
}

func (m *MockFs) RemoveAll(path string) (err error) {
	path, err = m.resolve(path, false)
	if err != nil {
		return err
	}
	parentPath, leaf := pathSplit(path)
	if len(leaf) == 0 {
		return fmt.Errorf("Cannot remove %v\n", path)
	}
	parentDir, err := m.getDir(parentPath, false)
	if err != nil {
		return err
	}
	if parentDir == nil {
		return &os.PathError{
			Op:   "removeAll",
			Path: path,
			Err:  os.ErrNotExist,
		}
	}
	if parentDir.readErr != nil {
		return &os.PathError{
			Op:   "removeAll",
			Path: path,
			Err:  parentDir.readErr,
		}

	}
	_, isFile := parentDir.files[leaf]
	_, isLink := parentDir.symlinks[leaf]
	if isFile || isLink {
		return m.Remove(path)
	}
	_, isDir := parentDir.subdirs[leaf]
	if !isDir {
		if !isDir {
			return &os.PathError{
				Op:   "removeAll",
				Path: path,
				Err:  os.ErrNotExist,
			}
		}
	}

	delete(parentDir.subdirs, leaf)
	parentDir.modTime = m.Clock.Time()
	return nil
}

func (m *MockFs) SetReadable(path string, readable bool) error {
	var readErr error
	if !readable {
		readErr = os.ErrPermission
	}
	return m.SetReadErr(path, readErr)
}

func (m *MockFs) SetReadErr(path string, readErr error) error {
	path, err := m.resolve(path, false)
	if err != nil {
		return err
	}
	parentPath, leaf := filepath.Split(path)
	parentDir, err := m.getDir(parentPath, false)
	if err != nil {
		return err
	}
	if parentDir.readErr != nil {
		return &os.PathError{
			Op:   "chmod",
			Path: parentPath,
			Err:  parentDir.readErr,
		}
	}

	inode, err := m.getInode(parentDir, leaf)
	if err != nil {
		return err
	}
	inode.readErr = readErr
	inode.permTime = m.Clock.Time()
	return nil
}

func (m *MockFs) ClearMetrics() {
	m.ReadDirCalls = []string{}
	m.StatCalls = []string{}
}

func (m *MockFs) ViewId() (id string) {
	return m.viewId
}

func (m *MockFs) SetViewId(id string) {
	m.viewId = id
}
func (m *MockFs) SetDeviceNumber(deviceNumber uint64) {
	m.deviceNumber = deviceNumber
}