DataLayout.h

/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#pragma once
 
#include <array>
 
#include <functional>
#include <memory>
#include <ostream>
#include <string>
#include <vector>
 
#include "ForwardDefinitions.h"
 
namespace vrs {
 
using std::ostream;
using std::string;
using std::unique_ptr;
using std::vector;
 
class DataPiece;
class AutoDataLayoutEnd;
 
template <typename T>
class DataPieceValue;
template <typename T>
class DataPieceArray;
template <typename T>
class DataPieceVector;
template <typename T>
class DataPieceStringMap;
class DataPieceString;
 
namespace internal {
class DataLayouter;
}
 
struct JsonWrapper;
 
enum class DataPieceType : uint8_t {
  Undefined = 0, 
  Value = 1, 
  Array = 2, 
  Vector = 3, 
  String = 4, 
  StringMap = 5, 
 
  COUNT 
};
 
enum class JsonFormatProfile {
  VrsFormat, 
  ExternalCompact, 
  ExternalPretty, 
  Public, 
};
 
struct JsonFormatProfileSpec {
  bool publicNames = false; 
  bool prettyJson = false; 
  bool value = false; 
  bool name = true; 
  bool type = true; 
  bool shortType = false; 
  bool index = true; 
  bool defaults = true; 
  bool tags = true; 
  bool properties = true; 
  bool required = true; 
 
  // Default format
  JsonFormatProfileSpec() = default;
  explicit JsonFormatProfileSpec(JsonFormatProfile profile);
};
 
class DataLayout {
 protected:
  DataLayout() = default;
 
 public:
  DataLayout& operator=(const DataLayout&) = delete;
  DataLayout(const DataLayout&) = delete;
 
  virtual ~DataLayout();
  static const size_t kNotFound;
  static const size_t kVariableSize;
 
// Pack and use unit32_t because we're storing on disk, and size_t might be 32 or 64 bits
#pragma pack(push, 1)
  class IndexEntry {
   public:
    void setOffset(size_t offset) {
      offset_ = static_cast<uint32_t>(offset);
    }
    size_t getOffset() const {
      return static_cast<uint32_t>(offset_);
    }
    void setLength(size_t length) {
      length_ = static_cast<uint32_t>(length);
    }
    size_t getLength() const {
      return static_cast<uint32_t>(length_);
    }
 
   private:
    uint32_t offset_{}; 
    uint32_t length_{}; 
  };
#pragma pack(pop)
 
  ContentBlock getContentBlock() const;
 
  vector<int8_t>& getFixedData() {
    return fixedData_;
  }
  vector<int8_t>& getVarData() {
    return varData_;
  }
  size_t getFixedDataSizeNeeded() const {
    return fixedDataSizeNeeded_;
  }
 
  size_t getVarDataSizeFromIndex() const;
 
  size_t getVarDataSizeNeeded() const;
 
  void collectVariableDataAndUpdateIndex();
  void collectVariableDataAndUpdateIndex(void* destination);
  void getRawData(vector<int8_t>& outRawData) const;
  void stageCurrentValues();
  bool copyClonedDataPieceValues(const DataLayout& originalLayout);
 
  bool mapLayout(DataLayout& targetLayout);
  bool isMapped() const {
    return mappedDataLayout_ != nullptr;
  }
  bool hasAllRequiredPieces() const {
    return mappedDataLayout_ == nullptr || hasAllRequiredPieces_;
  }
  void requireAllPieces();
 
  void printLayout(ostream& out, const string& indent = "") const;
  void printLayoutCompact(ostream& out, const string& indent = "") const;
 
  string asJson(JsonFormatProfile profile) const;
  string asJson(const JsonFormatProfileSpec& profile = JsonFormatProfileSpec()) const;
 
  string getListOfPiecesSpec() const;
 
  bool isSame(const DataLayout& otherLayout) const;
 
  static unique_ptr<DataLayout> makeFromJson(const string& json);
 
  template <class T>
  const DataPieceValue<T>* findDataPieceValue(const string& label) const;
  template <class T>
  DataPieceValue<T>* findDataPieceValue(const string& label);
  template <class T>
  const DataPieceArray<T>* findDataPieceArray(const string& label, size_t arraySize) const;
  template <class T>
  DataPieceArray<T>* findDataPieceArray(const string& label, size_t arraySize);
  template <class T>
  const DataPieceVector<T>* findDataPieceVector(const string& label) const;
  template <class T>
  DataPieceVector<T>* findDataPieceVector(const string& label);
  template <class T>
  const DataPieceStringMap<T>* findDataPieceStringMap(const string& label) const;
  template <class T>
  DataPieceStringMap<T>* findDataPieceStringMap(const string& label);
  const DataPieceString* findDataPieceString(const string& label) const;
  DataPieceString* findDataPieceString(const string& label);
  void forEachDataPiece(
      const std::function<void(const DataPiece*)>&,
      DataPieceType type = DataPieceType::Undefined) const;
  void forEachDataPiece(
      const std::function<void(DataPiece*)>&,
      DataPieceType type = DataPieceType::Undefined);
 
  bool isVarDataIndexValid() const;
  size_t getDeclaredFixedDataPiecesCount() const {
    return fixedSizePieces_.size();
  }
  size_t getDeclaredVarDataPiecesCount() const {
    return varSizePieces_.size();
  }
  size_t getAvailableFixedDataPiecesCount() const;
  size_t getAvailableVarDataPiecesCount() const;
 
 protected:
  template <class T>
  T* getFixedData(size_t offset, size_t size) {
    if (mappedDataLayout_ != nullptr) {
      return mappedDataLayout_->getFixedData<T>(offset, size);
    }
    if (offset != kNotFound && offset + size <= fixedData_.size()) {
      return reinterpret_cast<T*>(fixedData_.data() + offset);
    }
    return nullptr;
  }
  const IndexEntry* getVarSizeIndex() const;
  IndexEntry* getVarSizeIndex();
  template <class T>
  T* getVarData(size_t varPieceIndex, size_t& outCount) {
    if (mappedDataLayout_ != nullptr) {
      return mappedDataLayout_->getVarData<T>(varPieceIndex, outCount);
    }
    if (varPieceIndex < varSizePieces_.size()) {
      const IndexEntry& indexEntry = getVarSizeIndex()[varPieceIndex];
      if (indexEntry.getOffset() + indexEntry.getLength() <= varData_.size()) {
        outCount = indexEntry.getLength() / sizeof(T);
        return reinterpret_cast<T*>(varData_.data() + indexEntry.getOffset());
      }
    }
    outCount = 0;
    return nullptr;
  }
 
  void initLayout();
 
  void serialize(JsonWrapper& rj, const JsonFormatProfileSpec& profile) const;
 
  static bool mapPieces(
      const vector<DataPiece*>& searchPieces,
      const vector<DataPiece*>& givenPieces);
 
  friend class internal::DataLayouter;
  friend class DataPiece;
  template <class T>
  friend class DataPieceValue;
  template <class T>
  friend class DataPieceArray;
  template <class T>
  friend class DataPieceVector;
  template <class T>
  friend class DataPieceStringMap;
  friend class DataPieceString;
 
  vector<DataPiece*> fixedSizePieces_;
  vector<DataPiece*> varSizePieces_;
  vector<int8_t> fixedData_;
  size_t fixedDataSizeNeeded_{};
  vector<int8_t> varData_;
  bool hasAllRequiredPieces_{true};
  DataLayout* mappedDataLayout_{};
};
 
class EmptyDataLayout : public DataLayout {
 public:
  EmptyDataLayout() {
    initLayout();
  }
};
 
class AutoDataLayout : public DataLayout {
 public:
  AutoDataLayout();
};
 
class AutoDataLayoutEnd {
 public:
  AutoDataLayoutEnd();
};
 
class ManualDataLayout : public DataLayout {
 public:
  explicit ManualDataLayout(const string& json);
 
  ManualDataLayout();
  explicit ManualDataLayout(const DataLayout& layout);
 
  ~ManualDataLayout() override;
 
  DataPiece* add(unique_ptr<DataPiece> dataPiece);
 
  void endLayout();
 
 private:
  vector<unique_ptr<DataPiece>> manualPieces;
  bool layoutInProgress_;
};
 
struct DataLayoutStruct {
  explicit DataLayoutStruct(const string& structName);
  static void dataLayoutStructEnd(const string& structName);
};
 
#define DATA_LAYOUT_STRUCT(DATA_LAYOUT_STRUCT_TYPE)                 \
  explicit DATA_LAYOUT_STRUCT_TYPE(const std::string& _structName_) \
      : DataLayoutStruct(_structName_) {                            \
    dataLayoutStructEnd(_structName_);                              \
  }
 
#define DATA_LAYOUT_STRUCT_WITH_INIT(DATA_LAYOUT_STRUCT_TYPE)       \
  explicit DATA_LAYOUT_STRUCT_TYPE(const std::string& _structName_) \
      : DataLayoutStruct(_structName_) {                            \
    dataLayoutStructEnd(_structName_);                              \
    init();                                                         \
  }
 
template <typename T, size_t Size>
struct DataLayoutStructArray : public vrs::DataLayoutStruct {
  DATA_LAYOUT_STRUCT(DataLayoutStructArray)
  std::array<T, Size> array{createArrayHelper<T>(std::make_index_sequence<Size>())};
 
  T& operator[](const size_t index) {
    return array[index];
  }
 
  constexpr const T& operator[](const size_t index) const {
    return array[index];
  }
 
  constexpr std::size_t size() const noexcept {
    return array.size();
  }
 
  template <typename S, size_t... Indices>
  static constexpr auto createArrayHelper(std::index_sequence<Indices...>) {
    return std::array<S, sizeof...(Indices)>{S{std::to_string(Indices)}...};
  }
};
 
template <class OptionalFields>
class OptionalDataPieces : public std::unique_ptr<OptionalFields> {
 public:
  explicit OptionalDataPieces(bool allocateFields)
      : std::unique_ptr<OptionalFields>(
            allocateFields ? std::make_unique<OptionalFields>() : nullptr) {}
};
 
} // namespace vrs