Skip to content

Commit

Permalink
Merge pull request restic#5143 from MichaelEischer/fs-handle-interface
Browse files Browse the repository at this point in the history 
fs: rework FS interface to be handle based
  • Loading branch information
@MichaelEischer
MichaelEischer authored Nov 30, 2024
2 parents 8644bb1 + b51bf0c commit 8642049
Show file tree
Hide file tree
Showing 31 changed files with 810 additions and 478 deletions.
162 changes: 96 additions & 66 deletions internal/archiver/archiver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,11 @@ func (s *ItemStats) Add(other ItemStats) {
s.TreeSizeInRepo += other.TreeSizeInRepo
}

// ToNoder returns a restic.Node for a File.
type ToNoder interface {
ToNode(ignoreXattrListError bool) (*restic.Node, error)
}

type archiverRepo interface {
restic.Loader
restic.BlobSaver
Expand Down Expand Up @@ -257,8 +262,8 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I
}

// nodeFromFileInfo returns the restic node from an os.FileInfo.
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) {
node, err := arch.FS.NodeFromFileInfo(filename, fi, ignoreXattrListError)
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*restic.Node, error) {
node, err := meta.ToNode(ignoreXattrListError)
if !arch.WithAtime {
node.AccessTime = node.ModTime
}
Expand Down Expand Up @@ -308,20 +313,14 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {

// saveDir stores a directory in the repo and returns the node. snPath is the
// path within the current snapshot.
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete fileCompleteFunc) (d futureNode, err error) {
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous *restic.Tree, complete fileCompleteFunc) (d futureNode, err error) {
debug.Log("%v %v", snPath, dir)

treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi, false)
treeNode, names, err := arch.dirToNodeAndEntries(snPath, dir, meta)
if err != nil {
return futureNode{}, err
}

names, err := fs.Readdirnames(arch.FS, dir, fs.O_NOFOLLOW)
if err != nil {
return futureNode{}, err
}
sort.Strings(names)

nodes := make([]futureNode, 0, len(names))

for _, name := range names {
Expand Down Expand Up @@ -359,6 +358,29 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi
return fn, nil
}

func (arch *Archiver) dirToNodeAndEntries(snPath, dir string, meta fs.File) (node *restic.Node, names []string, err error) {
err = meta.MakeReadable()
if err != nil {
return nil, nil, fmt.Errorf("openfile for readdirnames failed: %w", err)
}

node, err = arch.nodeFromFileInfo(snPath, dir, meta, false)
if err != nil {
return nil, nil, err
}
if node.Type != restic.NodeTypeDir {
return nil, nil, fmt.Errorf("directory %q changed type, refusing to archive", snPath)
}

names, err = meta.Readdirnames(-1)
if err != nil {
return nil, nil, fmt.Errorf("readdirnames %v failed: %w", dir, err)
}
sort.Strings(names)

return node, names, nil
}

// futureNode holds a reference to a channel that returns a FutureNodeResult
// or a reference to an already existing result. If the result is available
// immediately, then storing a reference directly requires less memory than
Expand Down Expand Up @@ -435,21 +457,39 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
return futureNode{}, false, err
}

filterError := func(err error) (futureNode, bool, error) {
err = arch.error(abstarget, err)
if err != nil {
return futureNode{}, false, errors.WithStack(err)
}
return futureNode{}, true, nil
}
// exclude files by path before running Lstat to reduce number of lstat calls
if !arch.SelectByName(abstarget) {
debug.Log("%v is excluded by path", target)
return futureNode{}, true, nil
}

meta, err := arch.FS.OpenFile(target, fs.O_NOFOLLOW, true)
if err != nil {
debug.Log("open metadata for %v returned error: %v", target, err)
return filterError(err)
}
closeFile := true
defer func() {
if closeFile {
cerr := meta.Close()
if err == nil {
err = cerr
}
}
}()

// get file info and run remaining select functions that require file information
fi, err := arch.FS.Lstat(target)
fi, err := meta.Stat()
if err != nil {
debug.Log("lstat() for %v returned error: %v", target, err)
err = arch.error(abstarget, err)
if err != nil {
return futureNode{}, false, errors.WithStack(err)
}
return futureNode{}, true, nil
return filterError(err)
}
if !arch.Select(abstarget, fi, arch.FS) {
debug.Log("%v is excluded", target)
Expand All @@ -467,7 +507,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
debug.Log("%v hasn't changed, using old list of blobs", target)
arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start))
arch.CompleteBlob(previous.Size)
node, err := arch.nodeFromFileInfo(snPath, target, fi, false)
node, err := arch.nodeFromFileInfo(snPath, target, meta, false)
if err != nil {
return futureNode{}, false, err
}
Expand All @@ -494,40 +534,28 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous

// reopen file and do an fstat() on the open file to check it is still
// a file (and has not been exchanged for e.g. a symlink)
file, err := arch.FS.OpenFile(target, fs.O_RDONLY|fs.O_NOFOLLOW, 0)
err := meta.MakeReadable()
if err != nil {
debug.Log("Openfile() for %v returned error: %v", target, err)
err = arch.error(abstarget, err)
if err != nil {
return futureNode{}, false, errors.WithStack(err)
}
return futureNode{}, true, nil
debug.Log("MakeReadable() for %v returned error: %v", target, err)
return filterError(err)
}

fi, err = file.Stat()
fi, err := meta.Stat()
if err != nil {
debug.Log("stat() on opened file %v returned error: %v", target, err)
_ = file.Close()
err = arch.error(abstarget, err)
if err != nil {
return futureNode{}, false, errors.WithStack(err)
}
return futureNode{}, true, nil
return filterError(err)
}

// make sure it's still a file
if !fi.Mode().IsRegular() {
err = errors.Errorf("file %v changed type, refusing to archive", fi.Name())
_ = file.Close()
err = arch.error(abstarget, err)
if err != nil {
return futureNode{}, false, err
}
return futureNode{}, true, nil
err = errors.Errorf("file %q changed type, refusing to archive", target)
return filterError(err)
}

closeFile = false

// Save will close the file, we don't need to do that
fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() {
fn = arch.fileSaver.Save(ctx, snPath, target, meta, func() {
arch.StartFile(snPath)
}, func() {
arch.trackItem(snPath, nil, nil, ItemStats{}, 0)
Expand All @@ -547,7 +575,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
return futureNode{}, false, err
}

fn, err = arch.saveDir(ctx, snPath, target, fi, oldSubtree,
fn, err = arch.saveDir(ctx, snPath, target, meta, oldSubtree,
func(node *restic.Node, stats ItemStats) {
arch.trackItem(snItem, previous, node, stats, time.Since(start))
})
Expand All @@ -563,7 +591,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
default:
debug.Log(" %v other", target)

node, err := arch.nodeFromFileInfo(snPath, target, fi, false)
node, err := arch.nodeFromFileInfo(snPath, target, meta, false)
if err != nil {
return futureNode{}, false, err
}
Expand Down Expand Up @@ -614,22 +642,6 @@ func join(elem ...string) string {
return path.Join(elem...)
}

// statDir returns the file info for the directory. Symbolic links are
// resolved. If the target directory is not a directory, an error is returned.
func (arch *Archiver) statDir(dir string) (os.FileInfo, error) {
fi, err := arch.FS.Stat(dir)
if err != nil {
return nil, errors.WithStack(err)
}

tpe := fi.Mode() & (os.ModeType | os.ModeCharDevice)
if tpe != os.ModeDir {
return fi, errors.Errorf("path is not a directory: %v", dir)
}

return fi, nil
}

// saveTree stores a Tree in the repo, returned is the tree. snPath is the path
// within the current snapshot.
func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous *restic.Tree, complete fileCompleteFunc) (futureNode, int, error) {
Expand All @@ -640,15 +652,8 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
return futureNode{}, 0, errors.Errorf("FileInfoPath for %v is empty", snPath)
}

fi, err := arch.statDir(atree.FileInfoPath)
if err != nil {
return futureNode{}, 0, err
}

debug.Log("%v, dir node data loaded from %v", snPath, atree.FileInfoPath)
// in some cases reading xattrs for directories above the backup source is not allowed
// thus ignore errors for such folders.
node, err = arch.nodeFromFileInfo(snPath, atree.FileInfoPath, fi, true)
var err error
node, err = arch.dirPathToNode(snPath, atree.FileInfoPath)
if err != nil {
return futureNode{}, 0, err
}
Expand Down Expand Up @@ -719,6 +724,31 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
return fn, len(nodes), nil
}

func (arch *Archiver) dirPathToNode(snPath, target string) (node *restic.Node, err error) {
meta, err := arch.FS.OpenFile(target, 0, true)
if err != nil {
return nil, err
}
defer func() {
cerr := meta.Close()
if err == nil {
err = cerr
}
}()

debug.Log("%v, reading dir node data from %v", snPath, target)
// in some cases reading xattrs for directories above the backup source is not allowed
// thus ignore errors for such folders.
node, err = arch.nodeFromFileInfo(snPath, target, meta, true)
if err != nil {
return nil, err
}
if node.Type != restic.NodeTypeDir {
return nil, errors.Errorf("path is not a directory: %v", target)
}
return node, err
}

// resolveRelativeTargets replaces targets that only contain relative
// directories ("." or "../../") with the contents of the directory. Each
// element of target is processed with fs.Clean().
Expand Down
Loading

0 comments on commit 8642049

Please sign in to comment.