VRS
A file format for sensor data.
Loading...
Searching...
No Matches
Public Member Functions | Static Public Member Functions | Protected Member Functions | Protected Attributes | List of all members
vrs::DiskFileT< FileChunk > Class Template Reference

FileHandler implementation for disk files, with chunked file support. More...

#include <DiskFile.h>

Inheritance diagram for vrs::DiskFileT< FileChunk >:
vrs::WriteFileHandler vrs::FileHandler vrs::FileDelegator vrs::AtomicDiskFile

Public Member Functions

std::unique_ptr< FileHandlermakeNew () const override
 Make a new DiskFile object, with a default state.
 
const string & getFileHandlerName () const override
 
int openSpec (const FileSpec &fileSpec) override
 Open a file in read-only mode.
 
bool isOpened () const override
 Tell if a file is actually open.
 
int create (const string &newFilePath, const map< string, string > &options={}) override
 Create a new file.
 
void forgetFurtherChunks (int64_t fileSize) override
 Call this method to forget any chunk beyond this file size.
 
int64_t getTotalSize () const override
 Get the total size of all the chunks considered.
 
vector< std::pair< string, int64_t > > getFileChunks () const override
 Get the list of chunks, path + size.
 
int close () override
 Close the file.
 
int skipForward (int64_t offset) override
 Skip a number of bytes further in the file, in a chunk aware way.
 
int setPos (int64_t offset) override
 Set the file position at an arbitrary position, in a chunk aware way.
 
int read (void *buffer, size_t length) override
 Read a number of bytes, in a chunk aware way.
 
template<typename T , std::enable_if_t< std::is_trivially_copyable< T >::value, int > = 0>
int read (T &object)
 Helper to read trivially copyable objects, in a chunk aware way.
 
size_t getLastRWSize () const override
 Get the number of bytes actually moved by the last read or write operation.
 
bool reopenForUpdatesSupported () const override
 Tell if modifying files is supported by this FileHandler implementation.
 
int reopenForUpdates () override
 Switch from read-only to read-write mode.
 
bool isReadOnly () const override
 Find out if the file is in read-only mode.
 
int write (const void *buffer, size_t length) override
 Write to the current chunk, possibly expanding it.
 
template<typename T , std::enable_if_t< std::is_trivially_copyable< T >::value, int > = 0>
int write (const T &object)
 Helper for trivially copyable objects.
 
int overwrite (const void *buffer, size_t length) override
 
template<typename T , std::enable_if_t< std::is_trivially_copyable< T >::value, int > = 0>
int overwrite (const T &object)
 Helper for trivially copyable objects.
 
int addChunk () override
 Append a new chunk to the current file.
 
int truncate () override
 Truncate chunk to the current file position. Use with care.
 
int getLastError () const override
 Get the last error code. 0 means no error.
 
bool isEof () const override
 Tell if we are at the end of the last chunk.
 
int64_t getPos () const override
 Get the absolute position in the file, in a chunk aware way.
 
int64_t getChunkPos () const override
 Get position in the current chunk.
 
int getChunkRange (int64_t &outChunkOffset, int64_t &outChunkSize) const override
 Get range of the current chunk.
 
bool getCurrentChunk (string &outChunkPath, size_t &outChunkIndex) const override
 Get the path of the current chunk, or an empty string if no chunk is open.
 
bool isRemoteFileSystem () const override
 
template<typename T , std::enable_if_t< std::is_trivially_copyable< T >::value, int > = 0>
int writeZstdFile (const string &path, const T &object)
 
template<typename T , std::enable_if_t< std::is_trivially_copyable< T >::value, int > = 0>
int writeZstdFile (const string &path, const vector< T > &v)
 
template<typename T , std::enable_if_t< std::is_trivially_copyable< T >::value, int > = 0>
int readZstdFile (const string &path, T &object)
 
int parseUri (FileSpec &inOutFileSpec, size_t colonIndex) const override
 
const string & staticName ()
 
- Public Member Functions inherited from vrs::WriteFileHandler
virtual int create (const FileSpec &spec)
 
virtual int createSplitFile (const FileSpec &spec, const string &initialFilePath)
 
virtual void addSplitHead (MAYBE_UNUSED FileSpec &inOutSpec)
 
template<typename T , std::enable_if_t< std::is_trivially_copyable< T >::value, int > = 0>
int write (const T &object)
 Helper for trivially copyable objects.
 
template<typename T , std::enable_if_t< std::is_trivially_copyable< T >::value, int > = 0>
int overwrite (const T &object)
 Helper for trivially copyable objects.
 
- Public Member Functions inherited from vrs::FileHandler
virtual const string & getWriteFileHandlerName () const
 
virtual int open (const string &filePath)
 
int delegateOpen (const FileSpec &fileSpec, unique_ptr< FileHandler > &outNewDelegate) override
 
virtual bool isAvailableOrPrefetch (MAYBE_UNUSED size_t length)
 
template<typename T , std::enable_if_t< std::is_trivially_copyable< T >::value, int > = 0>
int read (T &object)
 Helper to read trivially copyable objects, in a chunk aware way.
 
virtual bool setCachingStrategy (CachingStrategy)
 
virtual CachingStrategy getCachingStrategy () const
 
virtual bool prefetchReadSequence (MAYBE_UNUSED const vector< std::pair< size_t, size_t > > &sequence, MAYBE_UNUSED bool clearSequence=true)
 
virtual bool setStatsCallback (const CacheStatsCallbackFunction &)
 
virtual bool purgeCache ()
 
bool isFileHandlerMatch (const FileSpec &fileSpec) const
 
virtual bool showProgress () const
 Tell if the file handler is probably slow, and extra progress information might be useful.
 
- Public Member Functions inherited from vrs::FileDelegator
virtual int parseUri (FileSpec &inOutFileSpec, MAYBE_UNUSED size_t colonIndex) const
 

Static Public Member Functions

static const std::string & staticName ()
 
static int writeZstdFile (const string &path, const void *data, size_t dataSize)
 
static int writeZstdFile (const string &path, const string &string)
 
static int readZstdFile (const string &path, vector< char > &outContent)
 Read a compressed buffer or a string (automatically adjusts the size)
 
static int readZstdFile (const string &path, string &outString)
 
static int readZstdFile (const string &path, void *data, size_t dataSize)
 Read a compressed buffer of an exact size, fails if the size isn't perfectly right.
 
static string readTextFile (const string &path)
 
static int writeTextFile (const string &path, const string &text)
 
- Static Public Member Functions inherited from vrs::WriteFileHandler
static unique_ptr< WriteFileHandlermake (const string &fileHandlerName)
 Create a new WriteFileHandler from a name.
 
- Static Public Member Functions inherited from vrs::FileHandler
static unique_ptr< FileHandlermakeOpen (const string &filePath)
 Open a file in read-only mode. Returns an open file handler, or nullptr on error.
 
static unique_ptr< FileHandlermakeOpen (const FileSpec &fileSpec)
 

Protected Member Functions

int checkChunks (const vector< string > &chunks)
 
int openChunk (FileChunk *chunk)
 
int closeChunk (FileChunk *chunk)
 
int addChunk (const string &chunkFilePath)
 
bool isLastChunk () const
 
bool trySetPosInCurrentChunk (int64_t offset)
 

Protected Attributes

map< string, string > options_
 
std::unique_ptr< vector< FileChunk > > chunks_
 
FileChunk * currentChunk_ {}
 
int filesOpenCount_ {}
 
size_t lastRWSize_ {}
 
int lastError_ {}
 
bool readOnly_ {true}
 

Additional Inherited Members

- Public Types inherited from vrs::FileHandler
using CacheStatsCallbackFunction = std::function< void(const CacheStats &stats)>
 

Detailed Description

template<class FileChunk>
class vrs::DiskFileT< FileChunk >

FileHandler implementation for disk files, with chunked file support.

Member Function Documentation

◆ addChunk()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::addChunk ( )
overridevirtual

Append a new chunk to the current file.

Implements vrs::WriteFileHandler.

◆ close()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::close ( )
overridevirtual

Close the file.

Implements vrs::FileHandler.

◆ create()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::create ( const string &  newFilePath,
const map< string, string > &  options = {} 
)
overridevirtual

Create a new file.

Implements vrs::WriteFileHandler.

◆ forgetFurtherChunks()

template<class FileChunk >
void vrs::DiskFileT< FileChunk >::forgetFurtherChunks ( int64_t  fileSize)
overridevirtual

Call this method to forget any chunk beyond this file size.

Implements vrs::FileHandler.

◆ getChunkPos()

template<class FileChunk >
int64_t vrs::DiskFileT< FileChunk >::getChunkPos ( ) const
overridevirtual

Get position in the current chunk.

Implements vrs::FileHandler.

◆ getChunkRange()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::getChunkRange ( int64_t &  outChunkOffset,
int64_t &  outChunkSize 
) const
overridevirtual

Get range of the current chunk.

Implements vrs::FileHandler.

◆ getCurrentChunk()

template<class FileChunk >
bool vrs::DiskFileT< FileChunk >::getCurrentChunk ( string &  outChunkPath,
size_t &  outChunkIndex 
) const
overridevirtual

Get the path of the current chunk, or an empty string if no chunk is open.

Implements vrs::WriteFileHandler.

◆ getFileChunks()

template<class FileChunk >
vector< pair< string, int64_t > > vrs::DiskFileT< FileChunk >::getFileChunks ( ) const
overridevirtual

Get the list of chunks, path + size.

Implements vrs::FileHandler.

◆ getFileHandlerName()

template<class FileChunk >
const string & vrs::DiskFileT< FileChunk >::getFileHandlerName ( ) const
overridevirtual

Implements vrs::FileHandler.

◆ getLastError()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::getLastError ( ) const
overridevirtual

Get the last error code. 0 means no error.

Implements vrs::FileHandler.

◆ getLastRWSize()

template<class FileChunk >
size_t vrs::DiskFileT< FileChunk >::getLastRWSize ( ) const
overridevirtual

Get the number of bytes actually moved by the last read or write operation.

Implements vrs::FileHandler.

◆ getPos()

template<class FileChunk >
int64_t vrs::DiskFileT< FileChunk >::getPos ( ) const
overridevirtual

Get the absolute position in the file, in a chunk aware way.

Implements vrs::FileHandler.

◆ getTotalSize()

template<class FileChunk >
int64_t vrs::DiskFileT< FileChunk >::getTotalSize ( ) const
overridevirtual

Get the total size of all the chunks considered.

Implements vrs::FileHandler.

◆ isEof()

template<class FileChunk >
bool vrs::DiskFileT< FileChunk >::isEof ( ) const
overridevirtual

Tell if we are at the end of the last chunk.

Implements vrs::FileHandler.

◆ isOpened()

template<class FileChunk >
bool vrs::DiskFileT< FileChunk >::isOpened ( ) const
overridevirtual

Tell if a file is actually open.

Implements vrs::FileHandler.

◆ isReadOnly()

template<class FileChunk >
bool vrs::DiskFileT< FileChunk >::isReadOnly ( ) const
overridevirtual

Find out if the file is in read-only mode.

Implements vrs::WriteFileHandler.

◆ isRemoteFileSystem()

template<class FileChunk >
bool vrs::DiskFileT< FileChunk >::isRemoteFileSystem ( ) const
overridevirtual

Tell if the file handler is handling remote data. Readers might need caching. Writers might not support modifying written data (and require a split head).

Implements vrs::FileHandler.

◆ makeNew()

template<class FileChunk >
unique_ptr< FileHandler > vrs::DiskFileT< FileChunk >::makeNew ( ) const
overridevirtual

Make a new DiskFile object, with a default state.

Implements vrs::FileHandler.

◆ openSpec()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::openSpec ( const FileSpec fileSpec)
overridevirtual

Open a file in read-only mode.

Implements vrs::FileHandler.

◆ overwrite()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::overwrite ( const void *  buffer,
size_t  length 
)
overridevirtual

Write a number of bytes to the file, in a chunk aware way, only ever extending the file's last chunk.

Implements vrs::WriteFileHandler.

◆ read()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::read ( void *  buffer,
size_t  length 
)
overridevirtual

Read a number of bytes, in a chunk aware way.

Implements vrs::FileHandler.

◆ readTextFile()

template<class FileChunk >
string vrs::DiskFileT< FileChunk >::readTextFile ( const string &  path)
static

Read a local file, expected to contain some text

Parameters
pathlocal file to read
Returns
The content of the file, if found, or the empty string otherwise. Will log errors if some unexpected access error happens, but will be silent and return an empty string if the file doesn't exist.

◆ reopenForUpdates()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::reopenForUpdates ( )
overridevirtual

Switch from read-only to read-write mode.

Implements vrs::WriteFileHandler.

◆ reopenForUpdatesSupported()

template<class FileChunk >
bool vrs::DiskFileT< FileChunk >::reopenForUpdatesSupported ( ) const
overridevirtual

Tell if modifying files is supported by this FileHandler implementation.

Implements vrs::WriteFileHandler.

◆ setPos()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::setPos ( int64_t  offset)
overridevirtual

Set the file position at an arbitrary position, in a chunk aware way.

Implements vrs::FileHandler.

◆ skipForward()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::skipForward ( int64_t  offset)
overridevirtual

Skip a number of bytes further in the file, in a chunk aware way.

Implements vrs::FileHandler.

◆ truncate()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::truncate ( )
overridevirtual

Truncate chunk to the current file position. Use with care.

Implements vrs::WriteFileHandler.

◆ write()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::write ( const void *  buffer,
size_t  length 
)
overridevirtual

Write to the current chunk, possibly expanding it.

Implements vrs::WriteFileHandler.

◆ writeTextFile()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::writeTextFile ( const string &  path,
const string &  text 
)
static

Write string to a local file

Parameters
pathlocal file path
texttext to write
Returns
A status code, 0 for success.

◆ writeZstdFile()

template<class FileChunk >
int vrs::DiskFileT< FileChunk >::writeZstdFile ( const string &  path,
const void *  data,
size_t  dataSize 
)
static

Helper methods to write a blob of data, or a string, to disk in one shot. Note that the data is compressed, so that we have high confidence it wasn't edited.


The documentation for this class was generated from the following files: