Program Listing for File data_helper.h
↰ Return to documentation for file (include/runtime/include/dataset/data_helper.h
)
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATA_HELPER_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATA_HELPER_H_
#include <sys/stat.h>
#include <fstream>
#include <iostream>
#include <map>
#include <memory>
#include <sstream>
#include <string>
#include <unordered_map>
#include <vector>
#include "include/api/dual_abi_helper.h"
#include "include/api/status.h"
namespace mindspore {
namespace dataset {
class DATASET_API DataHelper {
public:
DataHelper() = default;
~DataHelper() = default;
Status CreateAlbum(const std::string &in_dir, const std::string &out_dir) {
return CreateAlbumIF(StringToChar(in_dir), StringToChar(out_dir));
}
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<std::string> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), VectorStringToChar(value), StringToChar(out_file));
}
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<bool> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int8_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint8_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int16_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint16_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int32_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint32_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int64_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint64_t> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<float> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<double> &value,
const std::string &out_file = "") {
return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateValue(const std::string &in_file, const std::string &key, const std::string &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), StringToChar(value), StringToChar(out_file));
}
Status UpdateValue(const std::string &in_file, const std::string &key, const bool &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateValue(const std::string &in_file, const std::string &key, const int8_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateValue(const std::string &in_file, const std::string &key, const uint8_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateValue(const std::string &in_file, const std::string &key, const int16_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateValue(const std::string &in_file, const std::string &key, const uint16_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateValue(const std::string &in_file, const std::string &key, const int32_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateValue(const std::string &in_file, const std::string &key, const uint32_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateValue(const std::string &in_file, const std::string &key, const int64_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateValue(const std::string &in_file, const std::string &key, const uint64_t &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateValue(const std::string &in_file, const std::string &key, const float &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
Status UpdateValue(const std::string &in_file, const std::string &key, const double &value,
const std::string &out_file = "") {
return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
}
template <typename T>
Status WriteBinFile(const std::string &in_file, const std::vector<T> &data) {
try {
std::ofstream o(in_file, std::ios::binary | std::ios::out);
if (!o.is_open()) {
return Status(kMDUnexpectedError, "Error opening Bin file to write");
}
size_t length = data.size();
if (length == 0) {
return Status(kMDUnexpectedError, "size of data is 0 when written into file.");
}
o.write(reinterpret_cast<const char *>(&data[0]), std::streamsize(length * sizeof(T)));
o.close();
}
// Catch any exception and convert to Status return code
catch (const std::exception &err) {
return Status(kMDUnexpectedError, "Write bin file failed ");
}
return Status::OK();
}
template <typename T>
Status WriteBinFile(const std::string &in_file, T *data, size_t length) {
try {
if (data == nullptr) {
return Status(kMDUnexpectedError, "input data can not be null");
}
std::ofstream o(in_file, std::ios::binary | std::ios::out);
if (!o.is_open()) {
return Status(kMDUnexpectedError, "Error opening Bin file to write");
}
o.write(reinterpret_cast<const char *>(data), std::streamsize(length * sizeof(T)));
if (!o.good()) {
return Status(kMDUnexpectedError, "Error writing Bin file");
}
o.close();
}
// Catch any exception and convert to Status return code
catch (const std::exception &err) {
return Status(kMDUnexpectedError, "Write bin file failed");
}
return Status::OK();
}
size_t DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr, const size_t &buffer_size);
Status RemoveKey(const std::string &in_file, const std::string &key, const std::string &out_file = "") {
return RemoveKeyIF(StringToChar(in_file), StringToChar(key), StringToChar(out_file));
}
void Print(std::ostream &out) const;
friend std::ostream &operator<<(std::ostream &out, const DataHelper &dh) {
dh.Print(out);
return out;
}
private:
// Helper function for dual ABI support
Status CreateAlbumIF(const std::vector<char> &in_dir, const std::vector<char> &out_dir);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<std::vector<char>> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<bool> &value,
const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<int8_t> &value,
const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<uint8_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<int16_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<uint16_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<int32_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<uint32_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<int64_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
const std::vector<uint64_t> &value, const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<float> &value,
const std::vector<char> &out_file);
Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<double> &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<char> &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const bool &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int8_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint8_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int16_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint16_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int32_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint32_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int64_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint64_t &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const float &value,
const std::vector<char> &out_file);
Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const double &value,
const std::vector<char> &out_file);
Status RemoveKeyIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<char> &out_file);
};
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATA_HELPER_H_