Ocean
Loading...
Searching...
No Matches
NewTextureGenerator.h
Go to the documentation of this file.
1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 *
4 * This source code is licensed under the MIT license found in the
5 * LICENSE file in the root directory of this source tree.
6 */
7
8#ifndef META_OCEAN_TRACKING_MAPTEXTURING_NEW_TEXTURE_GENERATOR_H
9#define META_OCEAN_TRACKING_MAPTEXTURING_NEW_TEXTURE_GENERATOR_H
10
13
14#include "ocean/base/Frame.h"
15#include "ocean/base/Thread.h"
16
18
20#include "ocean/math/Frustum.h"
23
24namespace Ocean
25{
26
27namespace Tracking
28{
29
30namespace MapTexturing
31{
32
33/**
34 *
35 * @ingroup trackingmaptexturing
36 */
37class OCEAN_TRACKING_MAPTEXTURING_EXPORT NewTextureGenerator : protected Thread
38{
39 public:
40
41 /**
42 * Definition of an unordered map mapping triangle ids to votes.
43 */
44 typedef std::unordered_map<Index64, Scalar> TriangleVoteMap;
45
46 class OCEAN_TRACKING_MAPTEXTURING_EXPORT Keyframe
47 {
48 public:
49
50 /**
51 * Creates a new keyframe object.
52 * @param anyCamera The camera profile of the keyframe, must be valid
53 * @param camera_T_world The camera pose of this keyframe, must be valid
54 * @param frame The color frame of this keyframe, must be valid
55 * @param depth The depth frame of this keyframe
56 */
57 Keyframe(SharedAnyCamera anyCamera, const HomogenousMatrix4& camera_T_world, Frame&& frame, Frame&& depth);
58
59 bool updateVotes(const BlockedMeshes& blockedMeshes);
60
61 inline bool needsToBeStored() const;
62
63 inline bool isValid() const;
64
65 static Scalar determineTriangleVote(const AnyCamera& anyCamera, const HomogenousMatrix4& world_T_camera, const HomogenousMatrix4& flippedCamera_T_world, const Triangle3& triangle, const Frame& depth);
66
67 public:
68
69 /// The camera profile of this keyframe.
71
72 /// The camera pose of this keyframe.
74
75 /// The color frame of this keyframe.
77
78 /// The depth frame of this keyframe.
80
81 /// The map mapping triangle ids to their corresponding usage votes for this keyframe.
83
84 /// The set holding the ids of all triangles which could be textured with this keyfame.
86
87 bool needsToBeStored_ = true;
88 };
89
91 {
92 public:
93
94 explicit TexturedRegion(const size_t meshes) :
95 vertices_(meshes),
96 textureCoordinates_(meshes),
97 islandIds_(meshes)
98 {
99
100 }
101
102 public:
103
105
106 std::vector<Vectors3> vertices_;
107
108 std::vector<Vectors2> textureCoordinates_;
109
110 std::vector<Indices32> islandIds_;
111 };
112
113 /**
114 * This class holds the relevant information for a textured mesh.
115 */
117 {
118 public:
119
120 /**
121 * Default constructor.
122 */
123 TexturedMesh() = default;
124
125 /**
126 * Creates a new textured mesh object.
127 * @param vertices The vertices of the mesh
128 * @param textureCoordinates The per-vertex texture coordinates, one for each vertex
129 */
130 inline TexturedMesh(Vectors3&& vertices, Vectors2&& textureCoordinates);
131
132 public:
133
134 /// The vertices of this mesh.
136
137 /// The per-vertex texture coordinates of this mesh, one for each vertex.
139
140 /// The ids of the keyframes which are associated with this textured mesh.
142 };
143
144 typedef std::vector<TexturedMesh> TexturedMeshes;
145
146 typedef std::unordered_map<VectorI3, TexturedMesh, VectorI3> TexturedMeshMap;
147
148 typedef std::vector<Keyframe> Keyframes;
149
150 typedef std::unordered_map<Index32, Keyframe> KeyframeMap;
151
152 typedef std::unordered_map<Index32, TexturedRegion> TexturedRegionMap;
153
154 protected:
155
156 /**
157 * Definition of individual execution modes.
158 */
159 enum ExecutionMode : unsigned int
160 {
161 /// The generator is currently idling and can process new meshes.
163 /// The generator is currently processing new meshes.
165 /// The generator has processed new meshes which can be received/accessed.
166 EM_MESES_PROCESSED
167 };
168
169 typedef std::unordered_map<VectorI3, UnorderedIndexSet32, VectorI3> UsedKeyframeIdsPerMeshMap;
170
171 /**
172 * Definition of an unordered map mapping ids of keyframes to usage counters.
173 */
174 typedef std::unordered_map<Index32, Index32> KeyframeIdCounterMap;
175
176 /**
177 * Definition of an unordered set holding block ids.
178 */
179 typedef std::unordered_set<VectorI3, VectorI3> BlockCoordinateSet;
180
181 public:
182
183 bool processMeshes(Frustums&& cameraFrustums, Vectors3&& vertices, Rendering::TriangleFaces&& triangleFaces, Keyframes&& newKeyframes);
184
185 /**
186 * Returns whether the generator is ready to process new meshes.
187 * @return True, if so; False, if the generator is still processing meshes
188 */
189 inline bool readyToProcessMeshes() const;
190
191 /**
192 * Returns the latest textured meshes.
193 * @param texturedMeshMap The resulting map holding the textured meshes
194 * @param textureFrame The resulting texture associated with the meshes
195 * @return True, if meshes existed; False, if the generator has now new meshes or is still processing the meshes
196 */
197 inline bool latestTexturedMeshes(TexturedMeshMap& texturedMeshMap, Frame& textureFrame);
198
199 /**
200 * Returns the current memory usage of this texture generator.
201 * The memory usage is mainly determined by the keyframes the generator stores.
202 * @return The generator's current memory usage, in bytes
203 */
204 uint64_t memoryUsage() const;
205
206 /**
207 * Down-samples a given depth frame by taking the minimal depth value of a 4-neighborhood.
208 * @param depthFrame The depth frame to down-sample, must have pixel format FORMAT_F32, must be valid
209 * @param iterations The number of down-sample iterations, with range [1, infinity)
210 * @return The down-sampled depth frame
211 */
212 static Frame downsampleDepthFrame(const Frame& depthFrame, const unsigned int iterations = 2u);
213
214 protected:
215
216 void convertToTexture(const TexturedRegionMap& texturedRegionMap, TexturedMeshMap& texturedMeshMap, Frame& textureFrame);
217
218 BlockedMeshes extractVisibleMeshes(const Frustums& cameraFrustums, const Vectors3& vertices, const Rendering::TriangleFaces& triangleFaces) const;
219
220 /**
221 * The thread run function.
222 */
223 void threadRun() override;
224
226
227 static void spreadBestVotesAcrossKeyframes(KeyframeMap& keyframeMap, const TriangleVoteMap& bestVoteMap, const Scalar acceptanceRatio = Scalar(0.2));
228
229 static bool extractRegionsFromKeyframes(KeyframeMap& keyframeMap, const BlockedMeshes& blockMeshes, TexturedRegionMap& texturedRegionMap);
230
231 static bool extractRegionsFromMostVotedKeyframe(UnorderedIndexSet32& candidateKeyframeIds, KeyframeMap& keyframeMap, const BlockedMeshes& blockMeshes, TexturedRegionMap& texturedRegionMap);
232
233 static TexturedRegion extractRegionsFromKeyframe(const Keyframe& keyframe, const BlockedMeshes& blockedMeshes);
234
235 protected:
236
237 /// The current execution mode.
238 ExecutionMode executionMode_ = EM_IDLE;
239
240 /// The vertices of the meshes which are currently processed.
242
243 /// The triangle faces of the meshes which are currently processed.
245
246 /// The frustums of the cameras which have been used to determine vertices which are currently processed.
248
250
251 /// The counter for unique keyframe ids.
252 Index32 keyframeIdCounter_ = 0u;
253
254 /// The map mapping keyframe ids to keyframes.
256
257 /// The set holding the ids of all currently existing blocks.
259
261
262 /// The map mapping ids of keyframes to usage counters.
264
265 /// The map holding the latest textured meshes.
267
268 /// The latest texture associated with the latest textured meshes.
270
271 /// The generator's lock.
272 mutable Lock lock_;
273};
274
276{
277 return needsToBeStored_;
278}
279
281{
282 return camera_ != nullptr;
283}
284
285inline NewTextureGenerator::TexturedMesh::TexturedMesh(Vectors3&& vertices, Vectors2&& textureCoordinates) :
286 vertices_(std::move(vertices)),
287 textureCoordinates_(std::move(textureCoordinates))
288{
289 ocean_assert(vertices_.size() == textureCoordinates_.size());
290}
291
293{
294 const ScopedLock scopedLock(lock_);
295
296 return executionMode_ == EM_IDLE;
297}
298
299inline bool NewTextureGenerator::latestTexturedMeshes(TexturedMeshMap& texturedMeshMap, Frame& textureFrame)
300{
301 const ScopedLock scopedLock(lock_);
302
304 {
305 return false;
306 }
307
308 texturedMeshMap = std::move(latestTexturedMeshMap_);
309 textureFrame = std::move(latestTextureFrame_);
310
312
313 return true;
314}
315
316}
317
318}
319
320}
321
322#endif // META_OCEAN_TRACKING_MAPTEXTURING_NEW_TEXTURE_GENERATOR_H
This class implements the abstract base class for all AnyCamera objects.
Definition AnyCamera.h:130
This class implements Ocean's image class.
Definition Frame.h:1808
This class implements a recursive lock object.
Definition Lock.h:31
This class implements a scoped lock object for recursive lock objects.
Definition Lock.h:135
This class implements a thread.
Definition Thread.h:115
Frame depth_
The depth frame of this keyframe.
Definition NewTextureGenerator.h:79
bool needsToBeStored_
Definition NewTextureGenerator.h:87
TriangleVoteMap triangleVoteMap_
The map mapping triangle ids to their corresponding usage votes for this keyframe.
Definition NewTextureGenerator.h:82
Frame frame_
The color frame of this keyframe.
Definition NewTextureGenerator.h:76
bool needsToBeStored() const
Definition NewTextureGenerator.h:275
SharedAnyCamera camera_
The camera profile of this keyframe.
Definition NewTextureGenerator.h:70
UnorderedIndexSet64 acceptableTrianglesSet_
The set holding the ids of all triangles which could be textured with this keyfame.
Definition NewTextureGenerator.h:85
HomogenousMatrix4 world_T_camera_
The camera pose of this keyframe.
Definition NewTextureGenerator.h:73
bool isValid() const
Definition NewTextureGenerator.h:280
static Scalar determineTriangleVote(const AnyCamera &anyCamera, const HomogenousMatrix4 &world_T_camera, const HomogenousMatrix4 &flippedCamera_T_world, const Triangle3 &triangle, const Frame &depth)
Keyframe(SharedAnyCamera anyCamera, const HomogenousMatrix4 &camera_T_world, Frame &&frame, Frame &&depth)
Creates a new keyframe object.
bool updateVotes(const BlockedMeshes &blockedMeshes)
This class holds the relevant information for a textured mesh.
Definition NewTextureGenerator.h:117
UnorderedIndexSet32 usedKeyframeIds_
The ids of the keyframes which are associated with this textured mesh.
Definition NewTextureGenerator.h:141
Vectors3 vertices_
The vertices of this mesh.
Definition NewTextureGenerator.h:135
Vectors2 textureCoordinates_
The per-vertex texture coordinates of this mesh, one for each vertex.
Definition NewTextureGenerator.h:138
std::vector< Vectors2 > textureCoordinates_
Definition NewTextureGenerator.h:108
TexturedRegion(const size_t meshes)
Definition NewTextureGenerator.h:94
std::vector< Indices32 > islandIds_
Definition NewTextureGenerator.h:110
std::vector< Vectors3 > vertices_
Definition NewTextureGenerator.h:106
CV::PixelBoundingBoxes islandBoundingBoxes_
Definition NewTextureGenerator.h:104
Definition NewTextureGenerator.h:38
static TriangleVoteMap determineBestVotesAcrossKeyframes(const KeyframeMap &keyframeMap)
void convertToTexture(const TexturedRegionMap &texturedRegionMap, TexturedMeshMap &texturedMeshMap, Frame &textureFrame)
static void spreadBestVotesAcrossKeyframes(KeyframeMap &keyframeMap, const TriangleVoteMap &bestVoteMap, const Scalar acceptanceRatio=Scalar(0.2))
std::unordered_set< VectorI3, VectorI3 > BlockCoordinateSet
Definition of an unordered set holding block ids.
Definition NewTextureGenerator.h:179
uint64_t memoryUsage() const
Returns the current memory usage of this texture generator.
std::unordered_map< VectorI3, UnorderedIndexSet32, VectorI3 > UsedKeyframeIdsPerMeshMap
Definition NewTextureGenerator.h:169
std::unordered_map< VectorI3, TexturedMesh, VectorI3 > TexturedMeshMap
Definition NewTextureGenerator.h:146
std::unordered_map< Index32, Keyframe > KeyframeMap
Definition NewTextureGenerator.h:150
Rendering::TriangleFaces activeTriangleFaces_
The triangle faces of the meshes which are currently processed.
Definition NewTextureGenerator.h:244
static TexturedRegion extractRegionsFromKeyframe(const Keyframe &keyframe, const BlockedMeshes &blockedMeshes)
bool processMeshes(Frustums &&cameraFrustums, Vectors3 &&vertices, Rendering::TriangleFaces &&triangleFaces, Keyframes &&newKeyframes)
void threadRun() override
The thread run function.
bool latestTexturedMeshes(TexturedMeshMap &texturedMeshMap, Frame &textureFrame)
Returns the latest textured meshes.
Definition NewTextureGenerator.h:299
std::unordered_map< Index64, Scalar > TriangleVoteMap
Definition of an unordered map mapping triangle ids to votes.
Definition NewTextureGenerator.h:44
std::vector< TexturedMesh > TexturedMeshes
Definition NewTextureGenerator.h:144
static Frame downsampleDepthFrame(const Frame &depthFrame, const unsigned int iterations=2u)
Down-samples a given depth frame by taking the minimal depth value of a 4-neighborhood.
BlockedMeshes extractVisibleMeshes(const Frustums &cameraFrustums, const Vectors3 &vertices, const Rendering::TriangleFaces &triangleFaces) const
std::vector< Keyframe > Keyframes
Definition NewTextureGenerator.h:148
BlockedMeshes blockedMeshes_
Definition NewTextureGenerator.h:249
UsedKeyframeIdsPerMeshMap usedKeyframeIdsPerMeshMap_
Definition NewTextureGenerator.h:260
ExecutionMode executionMode_
The current execution mode.
Definition NewTextureGenerator.h:238
static bool extractRegionsFromKeyframes(KeyframeMap &keyframeMap, const BlockedMeshes &blockMeshes, TexturedRegionMap &texturedRegionMap)
Frame latestTextureFrame_
The latest texture associated with the latest textured meshes.
Definition NewTextureGenerator.h:269
TexturedMeshMap latestTexturedMeshMap_
The map holding the latest textured meshes.
Definition NewTextureGenerator.h:266
std::unordered_map< Index32, TexturedRegion > TexturedRegionMap
Definition NewTextureGenerator.h:152
bool readyToProcessMeshes() const
Returns whether the generator is ready to process new meshes.
Definition NewTextureGenerator.h:292
KeyframeIdCounterMap keyframeIdCounterMap_
The map mapping ids of keyframes to usage counters.
Definition NewTextureGenerator.h:263
std::unordered_map< Index32, Index32 > KeyframeIdCounterMap
Definition of an unordered map mapping ids of keyframes to usage counters.
Definition NewTextureGenerator.h:174
BlockCoordinateSet blockCoordinateSet_
The set holding the ids of all currently existing blocks.
Definition NewTextureGenerator.h:258
Lock lock_
The generator's lock.
Definition NewTextureGenerator.h:272
Frustums activeCameraFrustums_
The frustums of the cameras which have been used to determine vertices which are currently processed.
Definition NewTextureGenerator.h:247
static bool extractRegionsFromMostVotedKeyframe(UnorderedIndexSet32 &candidateKeyframeIds, KeyframeMap &keyframeMap, const BlockedMeshes &blockMeshes, TexturedRegionMap &texturedRegionMap)
ExecutionMode
Definition of individual execution modes.
Definition NewTextureGenerator.h:160
@ EM_IDLE
The generator is currently idling and can process new meshes.
Definition NewTextureGenerator.h:162
@ EM_PROCESSING_MESHES
The generator is currently processing new meshes.
Definition NewTextureGenerator.h:164
@ EM_MESES_PROCESSED
The generator has processed new meshes which can be received/accessed.
Definition NewTextureGenerator.h:166
KeyframeMap keyframeMap_
The map mapping keyframe ids to keyframes.
Definition NewTextureGenerator.h:255
Vectors3 activeVertices_
The vertices of the meshes which are currently processed.
Definition NewTextureGenerator.h:241
This class implements a 3D triangle.
Definition Triangle3.h:80
std::unordered_set< Index32 > UnorderedIndexSet32
Definition of an unordered_set holding 32 bit indices.
Definition Base.h:126
std::unordered_set< Index64 > UnorderedIndexSet64
Definition of an unordered_set holding 64 bit indices.
Definition Base.h:132
uint32_t Index32
Definition of a 32 bit index value.
Definition Base.h:84
std::vector< PixelBoundingBox > PixelBoundingBoxes
Definition of a vector holding bounding box objects with only positive coordinate values.
Definition PixelBoundingBox.h:42
std::vector< Frustum > Frustums
Definition of a vector holding Frustum objects.
Definition Frustum.h:59
float Scalar
Definition of a scalar type.
Definition Math.h:129
std::shared_ptr< AnyCamera > SharedAnyCamera
Definition of a shared pointer holding an AnyCamera object with Scalar precision.
Definition AnyCamera.h:60
std::vector< Vector2 > Vectors2
Definition of a vector holding Vector2 objects.
Definition Vector2.h:64
std::vector< Vector3 > Vectors3
Definition of a vector holding Vector3 objects.
Definition Vector3.h:65
std::vector< TriangleFace > TriangleFaces
Definition of a vector holding triangle faces.
Definition TriangleFace.h:30
std::vector< BlockedMesh > BlockedMeshes
Definition of a vector holding BlockedMesh objects.
Definition BlockedMesh.h:36
The namespace covering the entire Ocean framework.
Definition Accessor.h:15