435 lines
8.1 KiB
Go
435 lines
8.1 KiB
Go
// Copyright 2016 The Internal Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// Package file provides an os.File-like interface of a memory mapped file.
|
|
package file
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/cznic/fileutil"
|
|
"github.com/cznic/internal/buffer"
|
|
"github.com/cznic/mathutil"
|
|
"github.com/edsrzf/mmap-go"
|
|
)
|
|
|
|
const copyBufSize = 1 << 20 // 1 MB.
|
|
|
|
var (
|
|
_ Interface = (*mem)(nil)
|
|
_ Interface = (*file)(nil)
|
|
|
|
_ os.FileInfo = stat{}
|
|
|
|
sysPage = os.Getpagesize()
|
|
)
|
|
|
|
// Interface is a os.File-like entity.
|
|
type Interface interface {
|
|
io.ReaderAt
|
|
io.ReaderFrom
|
|
io.WriterAt
|
|
io.WriterTo
|
|
|
|
Close() error
|
|
Stat() (os.FileInfo, error)
|
|
Sync() error
|
|
Truncate(int64) error
|
|
}
|
|
|
|
// Open returns a new Interface backed by f, or an error, if any.
|
|
func Open(f *os.File) (Interface, error) { return newFile(f, 1<<30, 20) }
|
|
|
|
// OpenMem returns a new Interface, or an error, if any. The Interface content
|
|
// is volatile, it's backed only by process' memory.
|
|
func OpenMem(name string) (Interface, error) { return newMem(name, 18), nil }
|
|
|
|
type memMap map[int64]*[]byte
|
|
|
|
type mem struct {
|
|
m memMap
|
|
modTime time.Time
|
|
name string
|
|
pgBits uint
|
|
pgMask int
|
|
pgSize int
|
|
size int64
|
|
}
|
|
|
|
func newMem(name string, pgBits uint) *mem {
|
|
pgSize := 1 << pgBits
|
|
return &mem{
|
|
m: memMap{},
|
|
modTime: time.Now(),
|
|
name: name,
|
|
pgBits: pgBits,
|
|
pgMask: pgSize - 1,
|
|
pgSize: pgSize,
|
|
}
|
|
}
|
|
|
|
func (f *mem) IsDir() bool { return false }
|
|
func (f *mem) Mode() os.FileMode { return os.ModeTemporary + 0600 }
|
|
func (f *mem) ModTime() time.Time { return f.modTime }
|
|
func (f *mem) Name() string { return f.name }
|
|
func (f *mem) ReadFrom(r io.Reader) (n int64, err error) { return readFrom(f, r) }
|
|
func (f *mem) Size() (n int64) { return f.size }
|
|
func (f *mem) Stat() (os.FileInfo, error) { return f, nil }
|
|
func (f *mem) Sync() error { return nil }
|
|
func (f *mem) Sys() interface{} { return nil }
|
|
func (f *mem) WriteTo(w io.Writer) (n int64, err error) { return writeTo(f, w) }
|
|
|
|
func (f *mem) Close() error {
|
|
f.Truncate(0)
|
|
f.m = nil
|
|
return nil
|
|
}
|
|
|
|
func (f *mem) ReadAt(b []byte, off int64) (n int, err error) {
|
|
avail := f.size - off
|
|
pi := off >> f.pgBits
|
|
po := int(off) & f.pgMask
|
|
rem := len(b)
|
|
if int64(rem) >= avail {
|
|
rem = int(avail)
|
|
err = io.EOF
|
|
}
|
|
var zeroPage *[]byte
|
|
for rem != 0 && avail > 0 {
|
|
pg := f.m[pi]
|
|
if pg == nil {
|
|
if zeroPage == nil {
|
|
zeroPage = buffer.CGet(f.pgSize)
|
|
defer buffer.Put(zeroPage)
|
|
}
|
|
pg = zeroPage
|
|
}
|
|
nc := copy(b[:mathutil.Min(rem, f.pgSize)], (*pg)[po:])
|
|
pi++
|
|
po = 0
|
|
rem -= nc
|
|
n += nc
|
|
b = b[nc:]
|
|
}
|
|
return n, err
|
|
}
|
|
|
|
func (f *mem) Truncate(size int64) (err error) {
|
|
if size < 0 {
|
|
return fmt.Errorf("invalid truncate size: %d", size)
|
|
}
|
|
|
|
first := size >> f.pgBits
|
|
if size&int64(f.pgMask) != 0 {
|
|
first++
|
|
}
|
|
last := f.size >> f.pgBits
|
|
if f.size&int64(f.pgMask) != 0 {
|
|
last++
|
|
}
|
|
for ; first <= last; first++ {
|
|
if p := f.m[first]; p != nil {
|
|
buffer.Put(p)
|
|
}
|
|
delete(f.m, first)
|
|
}
|
|
|
|
f.size = size
|
|
return nil
|
|
}
|
|
|
|
func (f *mem) WriteAt(b []byte, off int64) (n int, err error) {
|
|
pi := off >> f.pgBits
|
|
po := int(off) & f.pgMask
|
|
n = len(b)
|
|
rem := n
|
|
var nc int
|
|
for rem != 0 {
|
|
pg := f.m[pi]
|
|
if pg == nil {
|
|
pg = buffer.CGet(f.pgSize)
|
|
f.m[pi] = pg
|
|
}
|
|
nc = copy((*pg)[po:], b)
|
|
pi++
|
|
po = 0
|
|
rem -= nc
|
|
b = b[nc:]
|
|
}
|
|
f.size = mathutil.MaxInt64(f.size, off+int64(n))
|
|
return n, nil
|
|
}
|
|
|
|
type stat struct {
|
|
os.FileInfo
|
|
size int64
|
|
}
|
|
|
|
func (s stat) Size() int64 { return s.size }
|
|
|
|
type fileMap map[int64]mmap.MMap
|
|
|
|
type file struct {
|
|
f *os.File
|
|
m fileMap
|
|
maxPages int
|
|
pgBits uint
|
|
pgMask int
|
|
pgSize int
|
|
size int64
|
|
fsize int64
|
|
}
|
|
|
|
func newFile(f *os.File, maxSize int64, pgBits uint) (*file, error) {
|
|
if maxSize < 0 {
|
|
panic("internal error")
|
|
}
|
|
|
|
pgSize := 1 << pgBits
|
|
switch {
|
|
case sysPage > pgSize:
|
|
pgBits = uint(mathutil.Log2Uint64(uint64(sysPage)))
|
|
default:
|
|
pgBits = uint(mathutil.Log2Uint64(uint64(pgSize / sysPage * sysPage)))
|
|
}
|
|
pgSize = 1 << pgBits
|
|
fi := &file{
|
|
f: f,
|
|
m: fileMap{},
|
|
maxPages: int(mathutil.MinInt64(
|
|
1024,
|
|
mathutil.MaxInt64(maxSize/int64(pgSize), 1)),
|
|
),
|
|
pgBits: pgBits,
|
|
pgMask: pgSize - 1,
|
|
pgSize: pgSize,
|
|
}
|
|
info, err := f.Stat()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err = fi.Truncate(info.Size()); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return fi, nil
|
|
}
|
|
|
|
func (f *file) ReadFrom(r io.Reader) (n int64, err error) { return readFrom(f, r) }
|
|
func (f *file) Sync() (err error) { return f.f.Sync() }
|
|
func (f *file) WriteTo(w io.Writer) (n int64, err error) { return writeTo(f, w) }
|
|
|
|
func (f *file) Close() (err error) {
|
|
for _, p := range f.m {
|
|
if err = p.Unmap(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if err = f.f.Truncate(f.size); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err = f.f.Sync(); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err = f.f.Close(); err != nil {
|
|
return err
|
|
}
|
|
|
|
f.m = nil
|
|
f.f = nil
|
|
return nil
|
|
}
|
|
|
|
func (f *file) page(index int64) (mmap.MMap, error) {
|
|
if len(f.m) == f.maxPages {
|
|
for i, p := range f.m {
|
|
if err := p.Unmap(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
delete(f.m, i)
|
|
break
|
|
}
|
|
}
|
|
|
|
off := index << f.pgBits
|
|
fsize := off + int64(f.pgSize)
|
|
if fsize > f.fsize {
|
|
if err := f.f.Truncate(fsize); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
f.fsize = fsize
|
|
}
|
|
p, err := mmap.MapRegion(f.f, f.pgSize, mmap.RDWR, 0, off)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
f.m[index] = p
|
|
return p, nil
|
|
}
|
|
|
|
func (f *file) ReadAt(b []byte, off int64) (n int, err error) {
|
|
avail := f.size - off
|
|
pi := off >> f.pgBits
|
|
po := int(off) & f.pgMask
|
|
rem := len(b)
|
|
if int64(rem) >= avail {
|
|
rem = int(avail)
|
|
err = io.EOF
|
|
}
|
|
for rem != 0 && avail > 0 {
|
|
pg := f.m[pi]
|
|
if pg == nil {
|
|
if pg, err = f.page(pi); err != nil {
|
|
return n, err
|
|
}
|
|
}
|
|
nc := copy(b[:mathutil.Min(rem, f.pgSize)], pg[po:])
|
|
pi++
|
|
po = 0
|
|
rem -= nc
|
|
n += nc
|
|
b = b[nc:]
|
|
}
|
|
return n, err
|
|
}
|
|
|
|
func (f *file) Stat() (os.FileInfo, error) {
|
|
fi, err := f.f.Stat()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return stat{fi, f.size}, nil
|
|
}
|
|
|
|
func (f *file) Truncate(size int64) (err error) {
|
|
if size < 0 {
|
|
return fmt.Errorf("invalid truncate size: %d", size)
|
|
}
|
|
|
|
first := size >> f.pgBits
|
|
if size&int64(f.pgMask) != 0 {
|
|
first++
|
|
}
|
|
last := f.size >> f.pgBits
|
|
if f.size&int64(f.pgMask) != 0 {
|
|
last++
|
|
}
|
|
for ; first <= last; first++ {
|
|
if p := f.m[first]; p != nil {
|
|
if err := p.Unmap(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
delete(f.m, first)
|
|
}
|
|
|
|
f.size = size
|
|
fsize := (size + int64(f.pgSize) - 1) &^ int64(f.pgMask)
|
|
if fsize != f.fsize {
|
|
if err := f.f.Truncate(fsize); err != nil {
|
|
return err
|
|
}
|
|
|
|
}
|
|
f.fsize = fsize
|
|
return nil
|
|
}
|
|
|
|
func (f *file) WriteAt(b []byte, off int64) (n int, err error) {
|
|
pi := off >> f.pgBits
|
|
po := int(off) & f.pgMask
|
|
n = len(b)
|
|
rem := n
|
|
var nc int
|
|
for rem != 0 {
|
|
pg := f.m[pi]
|
|
if pg == nil {
|
|
pg, err = f.page(pi)
|
|
if err != nil {
|
|
return n, err
|
|
}
|
|
}
|
|
nc = copy(pg[po:], b)
|
|
pi++
|
|
po = 0
|
|
rem -= nc
|
|
b = b[nc:]
|
|
}
|
|
f.size = mathutil.MaxInt64(f.size, off+int64(n))
|
|
return n, nil
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------
|
|
|
|
func readFrom(f Interface, r io.Reader) (n int64, err error) {
|
|
f.Truncate(0)
|
|
p := buffer.Get(copyBufSize)
|
|
b := *p
|
|
defer buffer.Put(p)
|
|
|
|
var off int64
|
|
var werr error
|
|
for {
|
|
rn, rerr := r.Read(b)
|
|
if rn != 0 {
|
|
_, werr = f.WriteAt(b[:rn], off)
|
|
n += int64(rn)
|
|
off += int64(rn)
|
|
}
|
|
if rerr != nil {
|
|
if !fileutil.IsEOF(rerr) {
|
|
err = rerr
|
|
}
|
|
break
|
|
}
|
|
|
|
if werr != nil {
|
|
err = werr
|
|
break
|
|
}
|
|
}
|
|
return n, err
|
|
}
|
|
|
|
func writeTo(f Interface, w io.Writer) (n int64, err error) {
|
|
p := buffer.Get(copyBufSize)
|
|
b := *p
|
|
defer buffer.Put(p)
|
|
|
|
var off int64
|
|
var werr error
|
|
for {
|
|
rn, rerr := f.ReadAt(b, off)
|
|
if rn != 0 {
|
|
_, werr = w.Write(b[:rn])
|
|
n += int64(rn)
|
|
off += int64(rn)
|
|
}
|
|
if rerr != nil {
|
|
if !fileutil.IsEOF(rerr) {
|
|
err = rerr
|
|
}
|
|
break
|
|
}
|
|
|
|
if werr != nil {
|
|
err = werr
|
|
break
|
|
}
|
|
}
|
|
return n, err
|
|
}
|