VRS
A file format for sensor data.
Loading...
Searching...
No Matches
DiskFile.h
1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#pragma once
18
19#include <cstdio>
20
21#include <vrs/VrsExport.h>
22#include <vrs/WriteFileHandler.h>
23#include <vrs/os/Platform.h>
24
25namespace vrs {
26
27using std::string;
28using std::vector;
29
30class DiskFileChunk;
31
33template <class FileChunk>
34class VRS_API DiskFileT : public WriteFileHandler {
35 public:
36 static const string& staticName();
37
38 DiskFileT();
39 ~DiskFileT() override;
40
42 std::unique_ptr<FileHandler> makeNew() const override;
43
44 const string& getFileHandlerName() const override;
45
47 int openSpec(const FileSpec& fileSpec) override;
48
50 bool isOpened() const override;
51
53 int create(const string& newFilePath, const FileSpec::Extras& options = {}) override;
55 void forgetFurtherChunks(int64_t fileSize) override;
57 int64_t getTotalSize() const override;
59 vector<std::pair<string, int64_t>> getFileChunks() const override;
61 int close() override;
62
64 int skipForward(int64_t offset) override;
66 int setPos(int64_t offset) override;
67
69 int read(void* buffer, size_t length) override;
71 template <typename T, std::enable_if_t<std::is_trivially_copyable<T>::value, int> = 0>
72 int read(T& object) {
73 return read(&object, sizeof(object));
74 }
75
76 template <typename T, std::enable_if_t<std::is_trivially_copyable<T>::value, int> = 0>
77 int read(vector<T>& vec) {
78 return read(vec.data(), vec.size() * sizeof(T));
79 }
80
81 size_t getLastRWSize() const override;
82
84 bool reopenForUpdatesSupported() const override;
86 int reopenForUpdates() override;
88 bool isReadOnly() const override;
90 int write(const void* buffer, size_t length) override;
92 template <typename T, std::enable_if_t<std::is_trivially_copyable<T>::value, int> = 0>
93 int write(const T& object) {
94 return write(&object, sizeof(object));
95 }
96
98 int overwrite(const void* buffer, size_t length) override;
100 template <typename T, std::enable_if_t<std::is_trivially_copyable<T>::value, int> = 0>
101 int overwrite(const T& object) {
102 return overwrite(&object, sizeof(object));
103 }
104
105 int addChunk() override;
107 int truncate() override;
108
110 int getLastError() const override;
112 bool isEof() const override;
114 int64_t getPos() const override;
116 int64_t getChunkPos() const override;
118 int getChunkRange(int64_t& outChunkOffset, int64_t& outChunkSize) const override;
120 bool getCurrentChunk(string& outChunkPath, size_t& outChunkIndex) const override;
121
122 bool isRemoteFileSystem() const override;
123
126 static int writeZstdFile(const string& path, const void* data, size_t dataSize);
127 static int writeZstdFile(const string& path, const string& string);
128 template <typename T, std::enable_if_t<std::is_trivially_copyable<T>::value, int> = 0>
129 int writeZstdFile(const string& path, const T& object) {
130 return writeZstdFile(path, &object, sizeof(T));
131 }
132 template <typename T, std::enable_if_t<std::is_trivially_copyable<T>::value, int> = 0>
133 int writeZstdFile(const string& path, const vector<T>& v) {
134 return writeZstdFile(path, v.data(), v.size() * sizeof(T));
135 }
137 static int readZstdFile(const string& path, vector<char>& outContent);
138 static int readZstdFile(const string& path, string& outString);
139 template <typename T, std::enable_if_t<std::is_trivially_copyable<T>::value, int> = 0>
140 int readZstdFile(const string& path, T& object) {
141 return readZstdFile(path, &object, sizeof(T));
142 }
144 static int readZstdFile(const string& path, void* data, size_t dataSize);
145
151 static string readTextFile(const string& path);
156 static int writeTextFile(const string& path, const string& text);
157
158 int parseUri(FileSpec& inOutFileSpec, size_t colonIndex) const override;
159
160 protected:
161 int checkChunks(const vector<string>& chunks);
162 int openChunk(FileChunk* chunk);
163 int closeChunk(FileChunk* chunk);
164 int addChunk(const string& chunkFilePath);
165 bool isLastChunk() const;
166 bool trySetPosInCurrentChunk(int64_t offset);
167
168 FileSpec::Extras options_; // optional options provided in openSpec or createFile
169 std::unique_ptr<vector<FileChunk>> chunks_; // all the chunks, when a VRS file is opened.
170 FileChunk* currentChunk_{}; // always points to the current chunk within chunks_.
171 int filesOpenCount_{};
172
173 size_t lastRWSize_{};
174 int lastError_{};
175 bool readOnly_{true};
176};
177
178using DiskFile = DiskFileT<DiskFileChunk>;
179
180#if (IS_LINUX_PLATFORM() && IS_X86_PLATFORM()) || IS_ANDROID_PLATFORM() || \
181 (IS_WINDOWS_PLATFORM() && IS_VRS_FB_INTERNAL())
182#define VRS_ASYNC_DISKFILE_SUPPORTED() true
183
184class AsyncDiskFileChunk;
185using AsyncDiskFile = DiskFileT<AsyncDiskFileChunk>;
186
187#else
188#define VRS_ASYNC_DISKFILE_SUPPORTED() false
189
190#endif
191
207class VRS_API AtomicDiskFile : public DiskFile {
208 public:
209 ~AtomicDiskFile() override;
210
211 int create(const string& newFilePath, const FileSpec::Extras& options = {}) override;
212 int close() override;
213
214 void abort() override;
215
216 private:
217 string finalName_;
218};
219
220} // namespace vrs
Definition DiskFile.h:207
int close() override
Definition DiskFile.cpp:641
int create(const string &newFilePath, const FileSpec::Extras &options={}) override
Definition DiskFile.cpp:636
void abort() override
Definition DiskFile.cpp:656
Definition DiskFileChunk.hpp:30
FileHandler implementation for disk files, with chunked file support.
Definition DiskFile.h:34
int read(T &object)
Helper to read trivially copyable objects, in a chunk aware way.
Definition DiskFile.h:72
int close() override
Close the file.
Definition DiskFile.cpp:72
static int writeTextFile(const string &path, const string &text)
Definition DiskFile.cpp:583
bool isEof() const override
Tell if we are at the end of the last chunk.
Definition DiskFile.cpp:341
static int writeZstdFile(const string &path, const void *data, size_t dataSize)
Definition DiskFile.cpp:497
int write(const void *buffer, size_t length) override
Write to the current chunk, possibly expanding it.
Definition DiskFile.cpp:265
int overwrite(const T &object)
Helper for trivially copyable objects.
Definition DiskFile.h:101
int openSpec(const FileSpec &fileSpec) override
Open a file in read-only mode.
Definition DiskFile.cpp:95
int64_t getTotalSize() const override
Get the total size of all the chunks considered.
Definition DiskFile.cpp:182
int getLastError() const override
Get the last error code. 0 means no error.
Definition DiskFile.cpp:336
std::unique_ptr< FileHandler > makeNew() const override
Make a new DiskFile object, with a default state.
Definition DiskFile.cpp:62
static string readTextFile(const string &path)
Definition DiskFile.cpp:567
int64_t getChunkPos() const override
Get position in the current chunk.
Definition DiskFile.cpp:460
bool isOpened() const override
Tell if a file is actually open.
Definition DiskFile.cpp:110
int truncate() override
Truncate chunk to the current file position. Use with care.
Definition DiskFile.cpp:316
int read(vector< T > &vec)
Helper to read a vector of trivially copyable objects, in a chunk aware way.
Definition DiskFile.h:77
int reopenForUpdates() override
Switch from read-only to read-write mode.
Definition DiskFile.cpp:243
bool reopenForUpdatesSupported() const override
Tell if modifying files is supported by this FileHandler implementation.
Definition DiskFile.cpp:238
int addChunk() override
Append a new chunk to the current file.
Definition DiskFile.cpp:379
int write(const T &object)
Helper for trivially copyable objects.
Definition DiskFile.h:93
int read(void *buffer, size_t length) override
Read a number of bytes, in a chunk aware way.
Definition DiskFile.cpp:201
static int readZstdFile(const string &path, void *data, size_t dataSize)
Read a compressed buffer of an exact size, fails if the size isn't perfectly right.
Definition DiskFile.cpp:548
bool getCurrentChunk(string &outChunkPath, size_t &outChunkIndex) const override
Get the path of the current chunk, or an empty string if no chunk is open.
Definition DiskFile.cpp:482
int setPos(int64_t offset) override
Set the file position at an arbitrary position, in a chunk aware way.
Definition DiskFile.cpp:147
int overwrite(const void *buffer, size_t length) override
Definition DiskFile.cpp:282
int64_t getPos() const override
Get the absolute position in the file, in a chunk aware way.
Definition DiskFile.cpp:453
size_t getLastRWSize() const override
Get the number of bytes actually moved by the last read or write operation.
Definition DiskFile.cpp:233
int create(const string &newFilePath, const FileSpec::Extras &options={}) override
Create a new file.
Definition DiskFile.cpp:115
int skipForward(int64_t offset) override
Skip a number of bytes further in the file, in a chunk aware way.
Definition DiskFile.cpp:134
bool isRemoteFileSystem() const override
Definition DiskFile.cpp:492
bool isReadOnly() const override
Find out if the file is in read-only mode.
Definition DiskFile.cpp:260
vector< std::pair< string, int64_t > > getFileChunks() const override
Get the list of chunks, path + size.
Definition DiskFile.cpp:191
static int readZstdFile(const string &path, vector< char > &outContent)
Read a compressed buffer or a string (automatically adjusts the size).
Definition DiskFile.cpp:538
void forgetFurtherChunks(int64_t fileSize) override
Call this method to forget any chunk beyond this file size.
Definition DiskFile.cpp:123
int getChunkRange(int64_t &outChunkOffset, int64_t &outChunkSize) const override
Get range of the current chunk.
Definition DiskFile.cpp:467
Definition Compressor.cpp:130
Generalized file descriptor class, allowing the efficient representation of complex file objects,...
Definition FileSpec.h:43
map< string, string, std::less<> > Extras
Definition FileSpec.h:46