Program Listing for File data_helper.h

↰ Return to documentation for file (include/runtime/include/dataset/data_helper.h)

#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATA_HELPER_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATA_HELPER_H_

#include <sys/stat.h>
#include <fstream>
#include <iostream>
#include <map>
#include <memory>
#include <sstream>
#include <string>
#include <unordered_map>
#include <vector>

#include "include/api/dual_abi_helper.h"
#include "include/api/status.h"

namespace mindspore {
namespace dataset {

class DATASET_API DataHelper {
 public:
  DataHelper() = default;

  ~DataHelper() = default;

  Status CreateAlbum(const std::string &in_dir, const std::string &out_dir) {
    return CreateAlbumIF(StringToChar(in_dir), StringToChar(out_dir));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<std::string> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), VectorStringToChar(value), StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<bool> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int8_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint8_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int16_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint16_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int32_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint32_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int64_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint64_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<float> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<double> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const std::string &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), StringToChar(value), StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const bool &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const int8_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const uint8_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const int16_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const uint16_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const int32_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const uint32_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const int64_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const uint64_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const float &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const double &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  template <typename T>
  Status WriteBinFile(const std::string &in_file, const std::vector<T> &data) {
    try {
      std::ofstream o(in_file, std::ios::binary | std::ios::out);
      if (!o.is_open()) {
        return Status(kMDUnexpectedError, "Error opening Bin file to write");
      }
      size_t length = data.size();
      if (length == 0) {
        return Status(kMDUnexpectedError, "size of data is 0 when written into file.");
      }
      o.write(reinterpret_cast<const char *>(&data[0]), std::streamsize(length * sizeof(T)));
      o.close();
    }
    // Catch any exception and convert to Status return code
    catch (const std::exception &err) {
      return Status(kMDUnexpectedError, "Write bin file failed ");
    }
    return Status::OK();
  }

  template <typename T>
  Status WriteBinFile(const std::string &in_file, T *data, size_t length) {
    try {
      if (data == nullptr) {
        return Status(kMDUnexpectedError, "input data can not be null");
      }
      std::ofstream o(in_file, std::ios::binary | std::ios::out);
      if (!o.is_open()) {
        return Status(kMDUnexpectedError, "Error opening Bin file to write");
      }
      o.write(reinterpret_cast<const char *>(data), std::streamsize(length * sizeof(T)));
      if (!o.good()) {
        return Status(kMDUnexpectedError, "Error writing Bin file");
      }
      o.close();
    }
    // Catch any exception and convert to Status return code
    catch (const std::exception &err) {
      return Status(kMDUnexpectedError, "Write bin file failed");
    }
    return Status::OK();
  }

  size_t DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr, const size_t &buffer_size);

  Status RemoveKey(const std::string &in_file, const std::string &key, const std::string &out_file = "") {
    return RemoveKeyIF(StringToChar(in_file), StringToChar(key), StringToChar(out_file));
  }

  void Print(std::ostream &out) const;

  friend std::ostream &operator<<(std::ostream &out, const DataHelper &dh) {
    dh.Print(out);
    return out;
  }

 private:
  // Helper function for dual ABI support
  Status CreateAlbumIF(const std::vector<char> &in_dir, const std::vector<char> &out_dir);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<std::vector<char>> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<bool> &value,
                       const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<int8_t> &value,
                       const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<uint8_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<int16_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<uint16_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<int32_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<uint32_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<int64_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<uint64_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<float> &value,
                       const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<double> &value,
                       const std::vector<char> &out_file);

  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<char> &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const bool &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int8_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint8_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int16_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint16_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int32_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint32_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int64_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint64_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const float &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const double &value,
                       const std::vector<char> &out_file);
  Status RemoveKeyIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<char> &out_file);
};
}  // namespace dataset
}  // namespace mindspore

#endif  // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATA_HELPER_H_