# Copyright 2022-2023 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
1. This file is an abstraction of the dataset loading class. It contains
some basic dataset operations(skip, filter, map, batch, ...).
2. Specific dataset loading classes can be found in datasets_vision.py, datasets_text.py,
datasets_audio.py, datasets_standard_format.py and dataets_user_defined.py files.
datasets_vision.py: contains vision dataset loading classes.
datasets_text.py: contains text dataset loading classes.
datasets_audio.py: contains audio dataset loading classes.
datasets_standard_format.py: contains standard format loading classes which
any other kinds of datasets can be converted to.
dataets_user_defined.py: contains basic classes that help users to define
flexible ways to load dataset.
"""
import atexit
import glob
import json
import os
import queue
import signal
import stat
import subprocess
import warnings
import gc
import time
import uuid
import multiprocessing
from enum import Enum
from importlib import import_module
import sys
import threading
import copy
import weakref
import platform
import psutil
import mindspore._c_dataengine as cde
from mindspore._c_expression import typing
from mindspore import log as logger
from mindspore.parallel._ps_context import _is_role_pserver, _is_role_sched, _get_ps_context,\
_enable_distributed_mindrt
from mindspore.dataset.engine.offload import GetOffloadModel
import mindspore.dataset.transforms.c_transforms as c_transforms
import mindspore.dataset.transforms.py_transforms as py_transforms
import mindspore.dataset.transforms as transforms
from mindspore.dataset.text.utils import SentencePieceModel, DE_C_INTER_SENTENCEPIECE_MODE
from mindspore.parallel._utils import _get_device_num
from mindspore.dataset.debug import DebugHook
from mindspore.dataset.engine import samplers
from .iterators import DictIterator, TupleIterator, DummyIterator, check_iterator_cleanup, _set_iterator_cleanup, \
ITERATORS_LIST, _unset_iterator_cleanup
from .queue import _SharedQueue, _Queue
from .validators import check_batch, check_shuffle, check_map, check_filter, check_repeat, check_skip, check_zip, \
check_rename, check_device_send, check_take, check_output_shape, check_project, \
check_sync_wait, check_zip_dataset, check_add_column, check_concat, check_split, check_bucket_batch_by_length, \
check_save, check_tuple_iterator, check_dict_iterator, check_schema, check_to_device_send, check_padded_batch, \
check_total_batch
from ..core.config import get_callback_timeout, _init_device_info, get_enable_shared_mem, get_num_parallel_workers, \
get_enable_watchdog, get_seed, set_seed, get_debug_mode, get_multiprocessing_timeout_interval, _get_debug_hook_list
from ..core.datatypes import mstype_to_detype
from ..core.validator_helpers import replace_none
from ..core.py_util_helpers import ExceptionHandler
from ..transforms.py_transforms_util import FuncWrapper, Implementation
from ..vision.transforms import ToNumpy
from ...mindrecord.config import _get_enc_key, _get_enc_mode, _get_hash_mode, encrypt, append_hash_to_file
try:
context = import_module("mindspore.context")
except ModuleNotFoundError:
context = None
if platform.system().lower() == "darwin" and multiprocessing.get_start_method() != "fork":
multiprocessing.set_start_method("fork", True)
OffloadToManualOffloadMode = {
None: cde.ManualOffloadMode.UNSPECIFIED,
False: cde.ManualOffloadMode.DISABLED,
True: cde.ManualOffloadMode.ENABLED
}
_train_dataset = None
def _set_training_dataset(dataset):
"""
Set the dataset to be used when training recovery has occurred.
Args:
dataset: the training dataset or iterator
"""
global _train_dataset
_train_dataset = dataset
def _get_training_dataset():
"""
Get the dataset to be used when training recovery has occurred.
Returns:
training dataset/iterator
"""
return _train_dataset
def _reset_training_dataset(global_step, dataset_size):
"""
Reset the training dataset to the given global step.
Args:
global_step (int): Number of global steps that have completed training.
Dataset will provide data from its next step after reset.
dataset_size (int): Number of steps per epoch.
"""
dataset = _get_training_dataset()
if dataset is not None:
dataset._reset(global_step, dataset_size) # pylint: disable=protected-access
else:
raise RuntimeError("Training dataset is not set.")
[docs]class Shuffle(str, Enum):
"""Specify the shuffle mode.
- ``Shuffle.GLOBAL`` : Shuffle both the files and samples.
- ``Shuffle.FILES`` : Shuffle files only.
- ``Shuffle.INFILE`` : Shuffle data within each file.
"""
GLOBAL: str = "global"
FILES: str = "files"
INFILE: str = "infile"
ShuffleToShuffleMode = {Shuffle.FILES: cde.ShuffleMode.FILES,
Shuffle.GLOBAL: cde.ShuffleMode.GLOBAL,
Shuffle.INFILE: cde.ShuffleMode.INFILE}
def shuffle_to_shuffle_mode(shuffle):
"""
Shuffle Enum to Shuffle Mode
Args:
shuffle (Shuffle): shuffle flag to shuffle mode in C layer
Returns:
ShuffleMode, shuffle mode
"""
shuffle_mode = cde.ShuffleMode.GLOBAL # Global shuffle
if not isinstance(shuffle, Shuffle):
if shuffle is None or shuffle:
shuffle_mode = cde.ShuffleMode.GLOBAL # Global shuffle
else:
shuffle_mode = cde.ShuffleMode.FALSE # No shuffle
else:
shuffle_mode = ShuffleToShuffleMode[shuffle]
return shuffle_mode
def shuffle_to_bool(shuffle):
"""
Shuffle Enum to bool
Args:
shuffle (Shuffle): shuffle flag to bool
Returns:
bool, True / False
"""
if shuffle is not None and not isinstance(shuffle, (bool, Shuffle)):
raise TypeError("shuffle must be of boolean or enum of 'Shuffle' values like 'Shuffle.GLOBAL' or "
"'Shuffle.FILES' or 'Shuffle.INFILE'.")
shuffle_bool = True
if not isinstance(shuffle, Shuffle):
if shuffle is None:
shuffle_bool = None
elif shuffle:
shuffle_bool = True
else:
shuffle_bool = False
else:
shuffle_bool = True
return shuffle_bool
@check_zip
def zip(datasets):
"""
Zip the datasets in the input tuple of datasets.
Args:
datasets (tuple[Dataset]): A tuple of datasets to be zipped together.
The number of datasets must be more than 1.
Returns:
Dataset, a new dataset with the above operation applied.
Raises:
ValueError: If the number of datasets is 1.
TypeError: If datasets is not a tuple.
Examples:
>>> # Create a dataset which is the combination of dataset_1 and dataset_2
>>> import mindspore.dataset as ds
>>>
>>> dataset_1 = ds.GeneratorDataset([1], "column1")
>>> dataset_2 = ds.GeneratorDataset([2], "column2")
>>> dataset = ds.zip((dataset_1, dataset_2))
"""
if len(datasets) <= 1:
raise ValueError(
"Can't zip empty or just one dataset!")
for dataset in datasets:
if not isinstance(dataset, Dataset):
raise TypeError("Invalid dataset, expected Dataset object, but got %s!" % type(dataset))
return ZipDataset(datasets)
def _get_operator_process():
"""
Inner implemented method, mainly for passing sub-process id in C layer
Returns:
dict, mapping dict of operation id and corresponding process id.
"""
global _OP_PROCESS
process_info = _OP_PROCESS
op_process = dict()
keys = process_info.keys()
fetched_all = True
for key in keys:
try:
op_process[key] = list(process_info[key][1])
item_full = (len(process_info[key][1]) == process_info[key][0])
except KeyError as err:
raise err
fetched_all = fetched_all and item_full
return op_process, fetched_all
def _set_dataset_permissions(file_name, num_files):
"""
set saved dataset files' permissions to 600
the rule of dataset filenames should be the same as those in C++.
"""
num_digits = len(str(num_files - 1))
if num_files == 1:
paths = [file_name]
else:
paths = ["{}{}".format(file_name, str(x).rjust(num_digits, '0')) for x in range(num_files)]
for item in paths:
if os.path.exists(item):
os.chmod(item, stat.S_IRUSR | stat.S_IWUSR)
index_file = item + ".db"
if os.path.exists(index_file):
os.chmod(index_file, stat.S_IRUSR | stat.S_IWUSR)
class Dataset:
"""
Abstract class to represent a dataset in DataEngine's data pipeline.
This class is the base class of SourceDataset and Dataset, and represents
a node in the data flow graph.
Dataset
-----------------------------------------------------------
| | | |
VisionBaseDataset TextBaseDataset AudioBaseDataset |
- - - |
| | | |
---------------------------------------- |
UnionBaseDataset |
|
SourceDataset
-
|
MappableDataset
DatasetOperation: MapDataset(UnionBaseDataset)
BatchDataset(UnionBaseDataset)
PaddedBatchDataset(UnionBaseDataset)
BucketBatchByLengthDataset(UnionBaseDataset)
ShuffleDataset(UnionBaseDataset)
FilterDataset(UnionBaseDataset)
RepeatDataset(UnionBaseDataset)
SkipDataset(UnionBaseDataset)
TakeDataset(UnionBaseDataset)
ZipDataset(UnionBaseDataset)
ConcatDataset(UnionBaseDataset)
RenameDataset(UnionBaseDataset)
ProjectDataset(UnionBaseDataset)
SyncWaitDataset(UnionBaseDataset)
Impl Dataset - vision: ImageFolderDataset(MappableDataset, VisionBaseDataset)
USPSDataset(SourceDataset, VisionBaseDataset)
Impl Dataset - text: TextFileDataset(SourceDataset, TextBaseDataset)
YahooAnswersDataset(SourceDataset, TextBaseDataset)
Impl Dataset - audio: LJSpeechDataset(MappableDataset, AudioBaseDataset)
TedliumDataset(MappableDataset, AudioBaseDataset)
Impl Dataset - standard: MindDataset(MappableDataset, UnionBaseDataset)
TFRecordDataset(SourceDataset, UnionBaseDataset)
Impl Dataset - user defined: GeneratorDataset(MappableDataset, UnionBaseDataset)
NumpySlicesDataset(GeneratorDataset)
Args:
num_parallel_workers (int, optional): Number of workers to process the dataset in parallel.
Default: ``None``.
"""
def __init__(self, children=None, num_parallel_workers=None, cache=None):
# Note: children and parent are internal variables, not recommended for external using.
self.children = replace_none(children, [])
if isinstance(self.children, tuple):
self.children = list(self.children)
if not isinstance(self.children, list):
self.children = [self.children]
self.parent = []
for child in self.children:
child.parent.append(weakref.ref(self))
self.num_parallel_workers = num_parallel_workers
self.cache = cache
self._device_iter = 0
self._input_indexs = ()
self.saved_output_types = None
self.saved_output_shapes = None
self.estimated_output_shapes = None
self.runtime_context = None
self._col_names = None
self.dataset_size = None
self._batch_size = None
self._num_classes = None
self._repeat_count = None
self._class_indexing = None
self._sync = False
self._global_step = None
@staticmethod
def _get_operator_id(dataset):
"""
Internal method to iterate the tree and obtain op_id of each operation.
Returns:
Dataset, the root dataset of the tree.
"""
op_name = dict()
generator_process = dict()
op_name[str(dataset)] = 0
op_id = 1
def process_name(datasets, operator_id):
if not datasets:
return 0
temp = []
for item in datasets:
for d in item.children:
temp.append(d)
op_name[str(d)] = operator_id
from mindspore.dataset.engine.datasets_user_defined import GeneratorDataset
if isinstance(d, GeneratorDataset) and d.sample_fn and d.sample_fn.pids:
generator_process[operator_id] = [d.num_parallel_workers, set(d.sample_fn.pids)]
operator_id = operator_id + 1
return process_name(temp, operator_id)
process_name([dataset], op_id)
if generator_process:
global _OP_PROCESS
_OP_PROCESS.update(generator_process)
return op_name
def create_ir_tree(self, getter_mode=False):
"""
Internal method to build an IR tree.
Args:
getter_mode (bool, optional): Whether to build IR tree in pull mode. Default: ``False``.
Returns:
Union[DatasetNode, Dataset], the root node of the IR tree and the root dataset of the IR tree.
"""
parent = self.parent
self.parent = []
dataset = copy.deepcopy(self)
global _OP_NAME
_OP_NAME = Dataset._get_operator_id(dataset)
ir_tree = dataset.parse_tree(getter_mode)
self.parent = parent
_init_device_info()
return ir_tree, dataset
def parse_tree(self, getter_mode=False):
"""
Internal method to parse the API tree into an IR tree.
Args:
getter_mode (bool, optional): Whether to build IR tree in pull mode. Default: ``False``.
Returns:
DatasetNode, the root node of the IR tree.
"""
if len(self.parent) > 1:
raise ValueError("The data pipeline is not a tree (i.e., one node has 2 consumers)")
ir_children = [d.parse_tree(getter_mode) for d in self.children]
# Bootstrap can only be performed on a copy of the original dataset node.
# Bootstrap on original dataset node will make all iterators share the same process pool
self.pre_parse(getter_mode)
self.iterator_bootstrap()
ir_node = self.parse(ir_children)
ir_node = self.post_parse(ir_node)
return ir_node
def __safe_deepcopy__(self, memodict, exclude=()):
if id(self) in memodict:
return memodict[id(self)]
cls = self.__class__
new_op = cls.__new__(cls)
memodict[id(self)] = new_op
for arg, value in self.__dict__.items():
if arg in exclude:
setattr(new_op, arg, value)
else:
try:
setattr(new_op, arg, copy.deepcopy(value, memodict))
except TypeError:
setattr(new_op, arg, value)
return new_op
@staticmethod
def _noop_mode():
if _is_role_sched():
return True
return False
def iterator_bootstrap(self):
pass
def __add__(self, datasets):
return self.concat(datasets)
[docs] def to_json(self, filename=""):
"""
Serialize a pipeline into JSON string and dump into file if filename is provided.
Args:
filename (str): filename of JSON file to be saved as. Default: ``""``.
Returns:
str, JSON string of the pipeline.
Examples:
>>> import mindspore.dataset as ds
>>> mnist_dataset_dir = "/path/to/mnist_dataset_directory"
>>> dataset = ds.MnistDataset(dataset_dir=mnist_dataset_dir)
>>> dataset_json = dataset.to_json("/path/to/mnist_dataset_pipeline.json")
"""
ir_tree, _ = self.create_ir_tree()
return json.loads(ir_tree.to_json(filename))
# The decorator has been deleted.
[docs] def bucket_batch_by_length(self, column_names, bucket_boundaries, bucket_batch_sizes, element_length_function=None,
pad_info=None, pad_to_bucket_boundary=False, drop_remainder=False):
"""
Bucket elements according to their lengths. Each bucket will be padded and batched when
they are full.
A length function is called on each row in the dataset. The row is then
bucketed based on its length and bucket boundaries. When a bucket reaches its
corresponding size specified in bucket_batch_sizes, the entire bucket will be
padded according to pad_info, and then form a batch.
Refer to the following figure for the execution process:
.. image:: bucket_batch_by_length_en.png
Args:
column_names (list[str]): Columns passed to element_length_function.
bucket_boundaries (list[int]): A list consisting of the upper boundaries
of the buckets. Must be strictly increasing. If there are n boundaries,
n+1 buckets are created: One bucket for [0, bucket_boundaries[0]), one
bucket for [bucket_boundaries[i], bucket_boundaries[i+1]) for each
0<i<n-1, and the last bucket for [bucket_boundaries[n-1], inf).
bucket_batch_sizes (list[int]): A list consisting of the batch sizes for
each bucket. Must contain len(bucket_boundaries)+1 elements.
element_length_function (Callable, optional): A function that takes in
M arguments where M = len(column_names) and returns an integer. If no value
provided, parameter M the len(column_names) must be 1, and the size of the first
dimension of that column will be taken as the length. Default: ``None``.
pad_info (dict, optional): The information about how to batch each column. The key
corresponds to the column name, and the value must be a tuple of 2 elements.
The first element corresponds to the shape to pad to, and the second
element corresponds to the value to pad with. If a column is not
specified, then that column will be padded to the longest in the current
batch, and 0 will be used as the padding value. Any None dimensions will
be padded to the longest in the current batch, unless if
`pad_to_bucket_boundary` is ``True``. If no padding is wanted, set `pad_info`
to ``None``. Default: ``None``.
pad_to_bucket_boundary (bool, optional): If ``True``, will pad each None
dimension in `pad_info` to the bucket_boundary minus 1. If there are any
elements that fall into the last bucket, an error will occur.
Default: ``False``.
drop_remainder (bool, optional): If ``True``, will drop the last batch for each
bucket if it is not a full batch. Default: ``False``.
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> # Create a dataset where certain counts rows are combined into a batch
>>> # and drops the last incomplete batch if there is one.
>>> import mindspore.dataset as ds
>>> import numpy as np
>>> def generate_2_columns(n):
... for i in range(n):
... yield (np.array([i]), np.array([j for j in range(i + 1)]))
>>>
>>> column_names = ["col1", "col2"]
>>> dataset = ds.GeneratorDataset(generate_2_columns(8), column_names)
>>> bucket_boundaries = [5, 10]
>>> bucket_batch_sizes = [2, 1, 1]
>>> element_length_function = (lambda col1, col2: max(len(col1), len(col2)))
>>> # Will pad col2 to shape [bucket_boundaries[i]] where i is the
>>> # index of the bucket that is currently being batched.
>>> pad_info = {"col2": ([None], -1)}
>>> pad_to_bucket_boundary = True
>>> dataset = dataset.bucket_batch_by_length(column_names, bucket_boundaries,
... bucket_batch_sizes,
... element_length_function, pad_info,
... pad_to_bucket_boundary)
"""
return BucketBatchByLengthDataset(self, column_names, bucket_boundaries, bucket_batch_sizes,
element_length_function, pad_info, pad_to_bucket_boundary, drop_remainder)
[docs] @check_batch
def batch(self, batch_size, drop_remainder=False, num_parallel_workers=None, **kwargs):
"""
Combine batch_size number of consecutive rows into batch which apply per_batch_map to the samples first.
For any column, all the elements within that column must have the same shape.
Refer to the following figure for the execution process:
.. image:: batch_en.png
Note:
The order of using repeat and batch reflects the number of batches and per_batch_map.
It is recommended that the repeat operation applied after the batch operation finished.
Args:
batch_size (Union[int, Callable]): The number of rows each batch is created with. An
int or callable object which takes exactly 1 parameter, BatchInfo.
drop_remainder (bool, optional): Determines whether or not to drop the last block
whose data row number is less than batch size. Default: ``False`` . If ``True`` ,
and if there are less than `batch_size` rows available to make the last batch,
then those rows will be dropped and not propagated to the child node.
num_parallel_workers (int, optional): Number of workers(threads) to process the dataset in parallel.
Default: ``None`` .
**kwargs:
- per_batch_map (Callable[[List[numpy.ndarray], ..., List[numpy.ndarray], BatchInfo], \
(List[numpy.ndarray], ..., List[numpy.ndarray])], optional): Per batch map callable.
Default: ``None``.
A callable which takes (List[numpy.ndarray], ..., List[numpy.ndarray], BatchInfo) as input parameters.
Each list[numpy.ndarray] represents a batch of numpy.ndarray on a given column. The number of lists
should match with the number of entries in input_columns. The last parameter of the callable should
always be a BatchInfo object. Per_batch_map should return
(list[numpy.ndarray], list[numpy.ndarray], ...). The length of each list in output should be the same
as the input. output_columns is required if the number of output lists is different from input.
- input_columns (Union[str, list[str]], optional): List of names of the input columns. The size of
the list should match with signature of `per_batch_map` callable. Default: ``None`` .
- output_columns (Union[str, list[str]], optional): List of names assigned to the columns
outputted by the last operation. This parameter is mandatory if len(input_columns) !=
len(output_columns). The size of this list must match the number of output
columns of the last operation. Default: ``None`` , output columns will have the same
name as the input columns, i.e., the columns will be replaced.
- python_multiprocessing (bool, optional): Parallelize Python function `per_batch_map` with
multi-processing or multi-threading mode, ``True`` means multi-processing,
``False`` means multi-threading If `per_batch_map` is a I/O bound task, use
multi-threading mode. If `per_batch_map` is a CPU bound task, it is recommended to use
multi-processing mode. Default: ``False`` , use python multi-threading mode.
- max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
allocation to copy data between processes, the total occupied shared memory will increase as
``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set
to -1, shared memory will be dynamically allocated with the actual size of data. This is only used if
``python_multiprocessing`` is set to True. If it is an int value, it represents
``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
If it is a list, the first element represents the ``input_columns`` use this value as the unit to
create shared memory, and the second element represents ``output_columns`` use this value as the unit
to create shared memory. Default: 16.
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> # 1) Create a dataset where every 5 rows are combined into a batch
>>> # and drops the last incomplete batch if there is one.
>>> import mindspore.dataset as ds
>>> from PIL import Image
>>>
>>> cifar10_dataset_dir = "/path/to/cifar10_dataset_directory"
>>> dataset = ds.Cifar10Dataset(dataset_dir=cifar10_dataset_dir, num_samples=10)
>>> dataset = dataset.batch(5, True)
>>>
>>> # 2) resize image according to its batch number, if it's 5-th batch, resize to (5^2, 5^2) = (25, 25)
>>> def np_resize(col, BatchInfo):
... output = col.copy()
... s = (BatchInfo.get_batch_num() + 1) ** 2
... index = 0
... for c in col:
... img = Image.fromarray(c.astype('uint8')).convert('RGB')
... img = img.resize((s, s))
... output[index] = np.array(img)
... index += 1
... return (output,)
>>> dataset = dataset.batch(batch_size=8, input_columns=["image"], per_batch_map=np_resize)
>>>
>>> # 3) Create a dataset where its batch size is dynamic
>>> # Define a callable batch size function and let batch size increase 1 each time.
>>> def add_one(BatchInfo):
... return BatchInfo.get_batch_num() + 1
>>> dataset = dataset.batch(batch_size=add_one, drop_remainder=True)
"""
return BatchDataset(self, batch_size, drop_remainder, num_parallel_workers, **kwargs)
[docs] @check_padded_batch
def padded_batch(self, batch_size, drop_remainder=False, num_parallel_workers=None, pad_info=None):
"""
Combine batch_size number of consecutive rows into batch which apply pad_info to the samples first.
Refer to the following figure for the execution process:
.. image:: padded_batch_en.png
Note:
The order of using repeat and padded_batch reflects the number of batches.
It is recommended that the repeat operation applied after the padded_batch operation finished.
Args:
batch_size (Union[int, Callable]): The number of rows each batch is created with. An
int or callable object which takes exactly 1 parameter, BatchInfo.
drop_remainder (bool, optional): Determines whether or not to drop the last block
whose data row number is less than batch size. Default: ``False``. If ``True``, and if there
are less than batch_size rows available to make the last batch, then those rows will
be dropped and not propagated to the child node.
num_parallel_workers (int, optional): Number of workers(threads) to process the dataset in parallel.
Default: ``None``.
pad_info (dict, optional): The pad information about how to batch each column. The key
corresponds to the column name, and the value must be a tuple of 2 elements.
The first element corresponds to the shape to pad to, and the second
element corresponds to the value to pad with. If a column is not
specified, then that column will be padded to the longest in the current
batch, and 0 will be used as the padding value. If ``pad_info={"col1": ([224, 224], 0)}``,
expand the data column named ``col1`` to shape (224, 224), and fill in the missing values with 0.
If ``pad_info={}``, all samples in the batch will be filled to the shape with the largest sample
in the current batch. If ``pad_info={"col1": (None, 100)}``, all samples in the batch will be filled
to the shape with the largest sample in the current batch, and fill in the missing values with 100.
If no padding is wanted, set `pad_info` to ``None``. Default: ``None``.
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> # 1) Pad every sample to the largest sample's shape and batch the samples
>>> import mindspore.dataset as ds
>>> dataset = ds.NumpySlicesDataset([[1], [1, 2], [1, 2, 3], [1, 2, 3, 4]], "column1")
>>> dataset = dataset.padded_batch(2, True, pad_info={})
>>>
>>> # 2) Create a dataset where every 3 rows are combined into a batch
>>> # and drops the last incomplete batch if there is one.
>>> dataset = ds.NumpySlicesDataset([i for i in range(10)], "column1")
>>> dataset = dataset.padded_batch(3, True)
>>>
>>> # 3) Create a dataset where its batch size is dynamic
>>> # Define a callable batch size function and let batch size increase 1 each time.
>>> def add_one(BatchInfo):
... return BatchInfo.get_batch_num() + 1
>>> dataset = dataset.padded_batch(batch_size=add_one, drop_remainder=True)
"""
return PaddedBatchDataset(self, batch_size, drop_remainder, num_parallel_workers, pad_info)
[docs] @check_sync_wait
def sync_wait(self, condition_name, num_batch=1, callback=None):
"""
Add a blocking condition to the input Dataset and a synchronize action will be applied.
Args:
condition_name (str): The condition name that is used to toggle sending next row.
num_batch (int): the number of batches without blocking at the start of each epoch.
Default: ``1``.
callback (function): The callback function that will be invoked when sync_update is called.
Default: ``None``.
Returns:
Dataset, a new dataset with the above operation applied.
Raises:
RuntimeError: If condition name already exists.
Examples:
>>> import mindspore.dataset as ds
>>> import numpy as np
>>> def gen():
... for i in range(100):
... yield (np.array(i),)
>>>
>>> class Augment:
... def __init__(self, loss):
... self.loss = loss
...
... def preprocess(self, input_):
... return input_
...
... def update(self, data):
... self.loss = data["loss"]
>>>
>>> batch_size = 4
>>> dataset = ds.GeneratorDataset(gen, column_names=["input"])
>>>
>>> aug = Augment(0)
>>> dataset = dataset.sync_wait(condition_name="policy", callback=aug.update)
>>> dataset = dataset.map(operations=[aug.preprocess], input_columns=["input"])
>>> dataset = dataset.batch(batch_size)
>>> count = 0
>>> for data in dataset.create_dict_iterator(num_epochs=1, output_numpy=True):
... assert data["input"][0] == count
... count += batch_size
... data = {"loss": count}
... dataset.sync_update(condition_name="policy", data=data)
"""
return SyncWaitDataset(self, condition_name, num_batch, callback)
[docs] @check_shuffle
def shuffle(self, buffer_size):
"""
Shuffle the dataset by creating a cache with the size of `buffer_size` .
1. Make a shuffle buffer that contains the first `buffer_size` rows.
2. Randomly select an element from the shuffle buffer to be the next row
propagated to the child node.
3. Get the next row (if any) from the parent node and put it in the shuffle buffer.
4. Repeat steps 2 and 3 until there are no more rows left in the shuffle buffer.
A random seed can be provided to be used on the first epoch via `dataset.config.set_seed` . In every subsequent
epoch, the seed is changed to a new one, randomly generated value.
Args:
buffer_size (int): The size of the buffer (must be larger than 1) for
shuffling. Setting `buffer_size` equal to the number of rows in the entire
dataset will result in a global shuffle.
Returns:
Dataset, a new dataset with the above operation applied.
Raises:
RuntimeError: If exist sync operations before shuffle.
Examples:
>>> import mindspore.dataset as ds
>>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
>>>
>>> # Optionally set the seed for fixed randomness
>>> ds.config.set_seed(58)
>>>
>>> # Create a shuffled dataset using a shuffle buffer of size 4
>>> dataset = dataset.shuffle(4)
"""
return ShuffleDataset(self, buffer_size)
[docs] def flat_map(self, func):
"""
Map `func` to each row in dataset and flatten the result.
Args:
func (function): A function that must take one `numpy.ndarray` as an argument and
return a `Dataset` .
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> import mindspore.dataset as ds
>>> # 1) flat_map on one column dataset
>>> dataset = ds.NumpySlicesDataset([[0, 1], [2, 3]], shuffle=False)
>>>
>>> def repeat(array):
... # create a NumpySlicesDataset with the array
... data = ds.NumpySlicesDataset(array, shuffle=False)
... # repeat the dataset twice
... data = data.repeat(2)
... return data
>>>
>>> dataset = dataset.flat_map(repeat)
>>> # [0, 1, 0, 1, 2, 3, 2, 3]
>>>
>>> # 2) flat_map on multi column dataset
>>> dataset = ds.NumpySlicesDataset(([[0, 1], [2, 3]], [[0, -1], [-2, -3]]), shuffle=False)
>>>
>>> def plus_and_minus(col1, col2):
... # apply different methods on columns
... data = ds.NumpySlicesDataset((col1 + 1, col2 - 1), shuffle=False)
... return data
>>>
>>> dataset = dataset.flat_map(plus_and_minus)
>>> # ([1, 2, 3, 4], [-1, -2, -3, -4])
Raises:
TypeError: If `func` is not a function.
TypeError: If `func` doesn't return a Dataset.
"""
dataset = None
if not hasattr(func, '__call__'):
logger.critical("func must be a function.")
raise TypeError("func must be a function.")
for row_data in self.create_tuple_iterator(num_epochs=1, output_numpy=True):
if dataset is None:
dataset = func(*row_data)
else:
dataset += func(*row_data)
if not isinstance(dataset, Dataset):
logger.critical("flat_map must return a Dataset object.")
raise TypeError("flat_map must return a Dataset object.")
return dataset
[docs] @check_map
def map(self, operations, input_columns=None, output_columns=None, column_order=None,
num_parallel_workers=None, **kwargs):
"""
Apply each operation in operations to this dataset.
Each operation will be passed one or more columns from the dataset as input, and one or
more columns will be outputted. The first operation will be passed the columns specified
in input_columns as input. If there is more than one operation in operations, the outputted
columns of the previous operation are used as the input columns for the next operation.
The columns outputted by the very last operation will be assigned names specified by
`output_columns` , and if not specified, the column name of output column is same as that of `input_columns` .
- If you use transformations (
`vision transform <https://mindspore.cn/docs/en/r2.3.1/api_python/mindspore.\
dataset.transforms.html#module-mindspore.dataset.vision>`_ ,
`nlp transform <https://mindspore.cn/docs/en/r2.3.1/api_python/mindspore.\
dataset.transforms.html#module-mindspore.dataset.text>`_ ,
`audio transform <https://mindspore.cn/docs/en/r2.3.1/api_python/mindspore.\
dataset.transforms.html#module-mindspore.dataset.audio>`_ )
provided by mindspore dataset, please use the following parameters:
.. image:: map_parameter_en.png
- If you use user-defined transform as PyFunc (Python Func), please use the following parameters:
.. image:: map_parameter_pyfunc_en.png
Args:
operations (Union[list[TensorOperation], list[functions]]): List of operations to be
applied on the dataset. Operations are applied in the order they appear in this list.
input_columns (Union[str, list[str]], optional): List of the names of the columns that will be passed to
the first operation as input. The size of this list must match the number of
input columns expected by the first operation. Default: ``None``, the first
operation will be passed however many columns that are required, starting from
the first column.
output_columns (Union[str, list[str]], optional): List of names assigned to the columns outputted by
the last operation. This parameter is mandatory if len(input_columns) !=
len(output_columns). The size of this list must match the number of output
columns of the last operation. Default: ``None``, output columns will have the same
name as the input columns, i.e., the columns will be replaced.
num_parallel_workers (int, optional): Number of threads used to process the dataset in
parallel. Default: ``None``, the value from the configuration will be used.
**kwargs:
- python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker processes.
This option could be beneficial if the Python operation is computational heavy. Default: ``False``.
- max_rowsize (Union[int, list[int]], optional): Maximum size of row in MB that is used for shared
memory allocation to copy data between processes, the total occupied shared memory will increase as
``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set
to -1, shared memory will be dynamically allocated with the actual size of data. This is only used if
``python_multiprocessing`` is set to True. If it is an int value, it represents
``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
If it is a list, the first element represents the ``input_columns`` use this value as the unit to
create shared memory, and the second element represents ``output_columns`` use this value as the unit
to create shared memory. Default: 16.
- cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
Default: ``None``, which means no cache is used.
- callbacks (DSCallback, list[DSCallback], optional): List of Dataset callbacks to be called.
Default: ``None``.
- offload (bool, optional): Flag to indicate whether offload is used. Default: ``None``.
Note:
- Input `operations` accepts TensorOperations defined in mindspore.dataset part, plus user-defined
Python functions (PyFuncs).
- Do not add network computing operators from mindspore.nn and mindspore.ops or others into this
`operations` .
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> import mindspore.dataset as ds
>>> import mindspore.dataset.vision as vision
>>> # dataset is an instance of Dataset which has 2 columns, "image" and "label".
>>> # image is of type bytes type which can be decoded to RGB
>>> # label is of type int32
>>> cifar10_dataset_dir = "/path/to/cifar10_dataset_directory"
>>> dataset = ds.Cifar10Dataset(dataset_dir=cifar10_dataset_dir)
>>>
>>> # Define two operations, where each operation accepts 1 input column and outputs 1 column.
>>> decode_op = vision.Decode(to_pil=False)
>>> random_jitter_op = vision.RandomColorAdjust(brightness=(0.8, 0.8), contrast=(1, 1),
... saturation=(1, 1), hue=(0, 0))
>>>
>>> # 1) Simple map example.
>>>
>>> # Apply decode_op on column "image".
>>> dataset = dataset.map(operations=[decode_op], input_columns=["image"])
>>>
>>> # Decode and rename column "image" to "decoded_image".
>>> dataset = dataset.map(operations=[decode_op], input_columns=["image"], output_columns=["decoded_image"])
>>>
>>> # A simple example for user defined python function transform.
>>> dataset = ds.NumpySlicesDataset(data=[[0, 1, 2]], column_names=["data"])
>>> dataset = dataset.map(operations=[(lambda x: x - 1)], input_columns=["data"])
>>>
>>> # 2) Map example with more than one operation.
>>>
>>> # Create a dataset where the images are decoded, then randomly color jittered.
>>> # decode_op takes column "image" as input and outputs one column. The column
>>> # outputted by decode_op is passed as input to random_jitter_op.
>>> # random_jitter_op will output one column. Column "image" will be replaced by
>>> # the column outputted by random_jitter_op (the very last operation). All other
>>> # columns are unchanged.
>>> dataset = dataset.map(operations=[decode_op, random_jitter_op], input_columns=["image"])
>>>
>>> # Rename the column outputted by random_jitter_op to "image_mapped".
>>> dataset = dataset.map(operations=[decode_op, random_jitter_op], input_columns=["image"],
... output_columns=["image_mapped"])
>>>
>>> # Map with multiple operations using pyfunc and rename column's name
>>> dataset = ds.NumpySlicesDataset(data=[[0, 1, 2]], column_names=["data"])
>>> dataset = dataset.map(operations=[(lambda x: x * x), (lambda x: x - 1)], input_columns=["data"],
... output_columns=["data_mapped"])
>>>
>>> # 3) Example where number of input columns is not equal to number of output columns.
>>>
>>> # operations[0] is a lambda that takes 2 columns as input and outputs 3 columns.
>>> # operations[1] is a lambda that takes 3 columns as input and outputs 1 column.
>>> # operations[2] is a lambda that takes 1 column as input and outputs 4 columns.
>>> #
>>> # Note: The number of output columns of operation[i] must equal the number of
>>> # input columns of operation[i+1]. Otherwise, this map call will also result
>>> # in an error.
>>> operations = [(lambda x, y: (x, x + y, x + y + 1)),
... (lambda x, y, z: x * y * z),
... (lambda x: (x % 2, x % 3, x % 5, x % 7))]
>>> dataset = ds.NumpySlicesDataset(data=([[0, 1, 2]], [[3, 4, 5]]), column_names=["x", "y"])
>>> dataset = dataset.map(operations, input_columns=["x", "y"],
... output_columns=["mod2", "mod3", "mod5", "mod7"])
"""
if hasattr(self, 'operator_mixed') and getattr(self, 'operator_mixed') is True:
num_parallel_workers = 1
logger.warning(
"Input 'operations' of 'map' includes network computing operators like in mindspore.nn, mindspore.ops, "
"mindspore.numpy module and etc, which do not support multi-thread compiling, recommend to replace it "
"with python implemented operator like numpy etc. Here decrease 'num_parallel_workers' into 1.")
return MapDataset(self, operations, input_columns, output_columns, num_parallel_workers, **kwargs)
[docs] @check_filter
def filter(self, predicate, input_columns=None, num_parallel_workers=None):
"""
Filter dataset by prediction.
Args:
predicate (callable): Python callable which returns a boolean value. If False then filter the element.
input_columns (Union[str, list[str]], optional): List of names of the input columns. If not provided
or provided with ``None``, the predicate will be applied on all columns in the dataset.
Default: ``None``.
num_parallel_workers (int, optional): Number of workers to process the dataset
in parallel. Default: ``None``.
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> # generator data(0 ~ 19)
>>> # filter the data that greater than or equal to 11
>>> import mindspore.dataset as ds
>>> dataset = ds.GeneratorDataset([i for i in range(20)], "data")
>>> dataset = dataset.filter(predicate=lambda data: data < 11, input_columns = ["data"])
"""
return FilterDataset(self, predicate, input_columns, num_parallel_workers)
[docs] @check_repeat
def repeat(self, count=None):
"""
Repeat this dataset `count` times. Repeat infinitely if the `count` is ``None`` or ``-1``.
Note:
The order of using repeat and batch reflects the number of batches. It is recommended that
the repeat operation is used after the batch operation.
Args:
count (int): Number of times the dataset is going to be repeated. Default: ``None``.
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> import mindspore.dataset as ds
>>>
>>> # Create a dataset with 10 elements
>>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
>>> ori_size = dataset.get_dataset_size()
>>>
>>> # Repeat the dataset 50 times.
>>> dataset = dataset.repeat(50)
>>> repeated_size = dataset.get_dataset_size()
>>> print("ori_size", ori_size, ", repeated_size", repeated_size)
ori_size 10 , repeated_size 500
>>>
>>> # Since the original dataset size is less than batch_size, thus no data is returned
>>> dataset1 = ds.GeneratorDataset([i for i in range(10)], "column1")
>>> dataset1 = dataset1.batch(batch_size=20, drop_remainder=True)
>>> dataset1 = dataset1.repeat(6)
>>>
>>> # Repeat the original dataset to 60 elements, thus 3 batches are returned
>>> dataset2 = ds.GeneratorDataset([i for i in range(10)], "column1")
>>> dataset2 = dataset2.repeat(6)
>>> dataset2 = dataset2.batch(batch_size=20, drop_remainder=True)
>>> print("dataset1 size", dataset1.get_dataset_size(), ", dataset2 size", dataset2.get_dataset_size())
dataset1 size 0 , dataset2 size 3
"""
return RepeatDataset(self, count)
[docs] @check_skip
def skip(self, count):
"""
Skip the first N elements of this dataset.
Args:
count (int): Number of elements in the dataset to be skipped.
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> import mindspore.dataset as ds
>>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
>>> # Skip first 3 elements of dataset and retain 7 elements.
>>> dataset = dataset.skip(3)
"""
return SkipDataset(self, count)
[docs] @check_take
def take(self, count=-1):
"""
Take the first specified number of samples from the dataset.
Args:
count (int, optional): The desired number of samples to take. If the value exceeds
the total number of samples in the dataset, all data will be returned.
Default: ``-1`` , will return all data.
Note:
When there are operations that will change the number of samples of the dataset in
the data pipeline, the location of the `take` operation can change its effect.
For example, `batch` operation will combine the successive samples of the specified
`batch_size` into 1 sample, so `.batch(batch_size).take(1)` will be equivalent to
`.take(batch_size).batch(batch_size)`.
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> import mindspore.dataset as ds
>>> mnist_dataset_dir = "/path/to/mnist_dataset_directory"
>>> dataset = ds.MnistDataset(dataset_dir=mnist_dataset_dir)
>>> # Take 50 samples from MNIST dataset.
>>> dataset = dataset.take(50)
"""
return TakeDataset(self, count)
def _get_absolute_split_sizes(self, sizes):
"""
Internal method called by split to calculate absolute split sizes and to
do some error checking after calculating absolute split sizes.
Returns:
int, absolute split sizes of the dataset.
"""
# Call get_dataset_size here and check input here because
# don't want to call this once in check_split and another time in
# here again
dataset_size = self.get_dataset_size()
if dataset_size is None or dataset_size <= 0:
raise RuntimeError("dataset_size is unknown, unable to split.")
if not isinstance(sizes, list):
raise RuntimeError("sizes must be a list.")
all_int = all(isinstance(item, int) for item in sizes)
if all_int:
sizes_sum = sum(sizes)
if sizes_sum != dataset_size:
raise RuntimeError("Sum of split sizes {} is not equal to dataset size {}."
.format(sizes_sum, dataset_size))
return sizes
absolute_sizes = []
for item in sizes:
absolute_size = int(round(item * dataset_size))
if absolute_size == 0:
raise RuntimeError("Split percentage {} is too small.".format(item))
absolute_sizes.append(absolute_size)
absolute_sizes_sum = sum(absolute_sizes)
# if we still need more rows, give them to the first split.
# if we have too many rows, remove the extras from the first split that has
# enough rows.
size_difference = int(dataset_size - absolute_sizes_sum)
if size_difference > 0:
absolute_sizes[0] += size_difference
else:
for i, _ in enumerate(absolute_sizes):
if absolute_sizes[i] + size_difference > 0:
absolute_sizes[i] += size_difference
break
if sum(absolute_sizes) != dataset_size:
raise RuntimeError("Sum of calculated split sizes {} is not equal to dataset size {}."
.format(absolute_sizes_sum, dataset_size))
return absolute_sizes
[docs] @check_split
def split(self, sizes, randomize=True):
"""
Split the dataset into smaller, non-overlapping datasets.
Args:
sizes (Union[list[int], list[float]]): If a list of integers [s1, s2, …, sn] is
provided, the dataset will be split into n datasets of size s1, size s2, …, size sn
respectively. If the sum of all input sizes does not equal the original dataset size, an
error will throw.
If a list of floats [f1, f2, …, fn] is provided, all floats must be between 0 and 1
and must sum to 1, otherwise an error will throw. The dataset will be split into n
Datasets of size round(f1*K), round(f2*K), …, round(fn*K) where K is the size of the
original dataset.
If after rounding:
- Any size equals 0, an error will occur.
- The sum of split sizes < K, the difference of K - sigma(round(fi * k)) will be added to the first
split.
- The sum of split sizes > K, the difference of sigma(round(fi * K)) - K will be removed from the first
large enough split such that it will have at least 1 row after removing the difference.
randomize (bool, optional): Determines whether or not to split the data randomly. Default: ``True``.
If True, the data will be randomly split. Otherwise, each split will be created with
consecutive rows from the dataset.
Note:
1. Dataset cannot be sharded if split is going to be called.
2. It is strongly recommended to not shuffle the dataset, but use randomize=True instead.
Shuffling the dataset may not be deterministic, which means the data in each split
will be different in each epoch.
Returns:
Tuple[Dataset], a tuple of new datasets split from the original one.
Raises:
RuntimeError: If get_dataset_size returns None or is not supported for this dataset.
RuntimeError: If `sizes` is list of integers and sum of all elements in sizes does not
equal the dataset size.
RuntimeError: If `sizes` is list of float and there is a split with size 0 after calculations.
RuntimeError: If the dataset is sharded prior to calling split.
ValueError: If `sizes` is list of float and not all floats are between 0 and 1, or if the
floats don't sum to 1.
Examples:
>>> # Split the data into train part and test part.
>>> import mindspore.dataset as ds
>>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
>>> train_dataset, test_dataset = dataset.split([0.9, 0.1])
"""
if self.is_shuffled():
logger.warning("Dataset is shuffled before split.")
if self.is_sharded():
raise RuntimeError("Dataset should not be sharded before split.")
absolute_sizes = self._get_absolute_split_sizes(sizes)
splits = []
rows_to_skip = 0
for size in absolute_sizes:
ds = copy.deepcopy(self)
if randomize:
# want to shuffle the same way every epoch before split
# in alter_tree, shuffle buffer is minimum 10000, so use 10000 here
ds = ds.shuffle(10000)
ds.reshuffle_each_epoch = False
if rows_to_skip > 0:
ds = ds.skip(rows_to_skip)
ds = ds.take(size)
splits.append(ds)
rows_to_skip += size
return tuple(splits)
[docs] @check_zip_dataset
def zip(self, datasets):
"""
Zip the datasets in the sense of input tuple of datasets. Columns in the input datasets must have different
name.
Args:
datasets (Union[Dataset, tuple[Dataset]]): A tuple of datasets or a single class Dataset
to be zipped together with this dataset.
Returns:
Dataset, a new dataset with the above operation applied.
Raises:
TypeError: The parameter is not dataset object or tuple of dataset objects.
Examples:
>>> # Create a dataset which is the combination of dataset_1 and dataset_2
>>> import mindspore.dataset as ds
>>> dataset_1 = ds.GeneratorDataset([1, 2, 3], "column1")
>>> dataset_2 = ds.GeneratorDataset([1, 2, 3], "column2")
>>> dataset = dataset_1.zip(dataset_2)
"""
if isinstance(datasets, tuple):
datasets = (self, *datasets)
elif isinstance(datasets, Dataset):
datasets = (self, datasets)
else:
raise TypeError("Invalid datasets, expected Dataset object or tuple of Dataset, but got %s!" % datasets)
return ZipDataset(datasets)
[docs] @check_concat
def concat(self, datasets):
"""
Concatenate the dataset objects in the input list.
Performing "+" operation on dataset objects can achieve the same effect.
For a dataset concatenated by many other dataset objects, it returns the data in the order of
datasets passed in. If you want to change the data order(such as random selection from each dataset
instead of in sequence), apply `use_sampler` method on the concatenated dataset object.
Currently `use_sampler` supports `dataset.DistributedSampler` for sharding selection from each dataset
or `dataset.RandomSampler` for random selection from each dataset, see examples below.
Note:
The column name, and rank and type of the column data must be the same in the input datasets.
Args:
datasets (Union[list, Dataset]): A list of datasets or a single class Dataset
to be concatenated together with this dataset.
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> import mindspore.dataset as ds
>>> dataset_1 = ds.GeneratorDataset([1, 2, 3], "column1", shuffle=False)
>>> dataset_2 = ds.GeneratorDataset([4, 5, 6], "column1", shuffle=False)
>>>
>>> # Create a dataset by concatenating dataset_1 and dataset_2 with "+" operator
>>> dataset = dataset_1 + dataset_2
>>> # Create a dataset by concatenating dataset_1 and dataset_2 with concat operation
>>> dataset = dataset_1.concat(dataset_2)
>>>
>>> # Check the data order of dataset
>>> dataset_1 = ds.GeneratorDataset([1, 2, 3], "column1", shuffle=False)
>>> dataset_2 = ds.GeneratorDataset([4, 5, 6], "column1", shuffle=False)
>>> dataset = dataset_1 + dataset_2
>>> result = list(dataset)
>>> # [[Tensor(shape=[], dtype=Int64, value= 1)], [Tensor(shape=[], dtype=Int64, value= 2)],
>>> # [Tensor(shape=[], dtype=Int64, value= 3)], [Tensor(shape=[], dtype=Int64, value= 4)],
>>> # [Tensor(shape=[], dtype=Int64, value= 5)], [Tensor(shape=[], dtype=Int64, value= 6)]]
>>>
>>> # Change the data order of concatenated dataset with sharding selection
>>> dataset_1 = ds.GeneratorDataset([1, 2, 3], "column1", shuffle=False)
>>> dataset_2 = ds.GeneratorDataset([4, 5, 6], "column1", shuffle=False)
>>> dataset = dataset_1.concat(dataset_2)
>>> dataset.use_sampler(ds.DistributedSampler(num_shards=2, shard_id=1, shuffle=False))
>>> result = list(dataset)
>>> # [[Tensor(shape=[], dtype=Int64, value= 2)], [Tensor(shape=[], dtype=Int64, value= 4)],
>>> # [Tensor(shape=[], dtype=Int64, value= 6)]]
>>>
>>> # Change the data order of concatenated dataset with random selection
>>> dataset_1 = ds.GeneratorDataset([1, 2, 3], "column1", shuffle=False)
>>> dataset_2 = ds.GeneratorDataset([4, 5, 6], "column1", shuffle=False)
>>> dataset = dataset_1.concat(dataset_2)
>>> dataset.use_sampler(ds.RandomSampler())
>>> result = list(dataset)
>>> # [[Tensor(shape=[], dtype=Int64, value= 1)], [Tensor(shape=[], dtype=Int64, value= 4)],
>>> # [Tensor(shape=[], dtype=Int64, value= 2)], [Tensor(shape=[], dtype=Int64, value= 5)],
>>> # [Tensor(shape=[], dtype=Int64, value= 6)], [Tensor(shape=[], dtype=Int64, value= 3)]]
"""
if isinstance(datasets, Dataset):
datasets = [self] + [datasets]
elif isinstance(datasets, list):
datasets = [self] + datasets
else:
raise TypeError("Invalid datasets, expected Dataset object or list of Dataset, but got %s!" % datasets)
return ConcatDataset(datasets)
[docs] @check_rename
def rename(self, input_columns, output_columns):
"""
Rename the columns in input datasets.
Args:
input_columns (Union[str, list[str]]): List of names of the input columns.
output_columns (Union[str, list[str]]): List of names of the output columns.
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> import mindspore.dataset as ds
>>> input_columns = ["input_col1", "input_col2", "input_col3"]
>>> output_columns = ["output_col1", "output_col2", "output_col3"]
>>>
>>> # Create a dataset with 3 columns
>>> dataset = ds.GeneratorDataset([(1, 2, 3), (3, 4, 5), (5, 6, 7)], column_names=input_columns)
>>>
>>> # Rename "input_col1" to "output_col1", "input_col2" to "output_col2", "input_col3" to "output_col3"
>>> dataset = dataset.rename(input_columns=input_columns, output_columns=output_columns)
"""
return RenameDataset(self, input_columns, output_columns)
[docs] @check_project
def project(self, columns):
"""
The specified columns will be selected from the dataset and passed into
the pipeline with the order specified. The other columns are discarded.
Args:
columns(Union[str, list[str]]): List of names of the columns to project.
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> import mindspore.dataset as ds
>>> # Create a dataset with 3 columns
>>> input_columns = ["column1", "column2", "column3"]
>>> dataset = ds.GeneratorDataset([(1, 2, 3), (3, 4, 5), (5, 6, 7)], column_names=input_columns)
>>>
>>> columns_to_project = ["column3", "column1", "column2"]
>>> # in that order, regardless of the original order of columns.
>>> dataset = dataset.project(columns=columns_to_project)
"""
return ProjectDataset(self, columns)
[docs] def apply(self, apply_func):
"""
Apply a function in this dataset.
Args:
apply_func (function): A function that must take one `Dataset` as an argument and
return a preprocessed `Dataset` .
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> import mindspore.dataset as ds
>>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
>>>
>>> # Declare an apply_func function which returns a Dataset object
>>> def apply_func(data):
... data = data.batch(2)
... return data
>>>
>>> # Use apply to call apply_func
>>> dataset = dataset.apply(apply_func)
Raises:
TypeError: If apply_func is not a function.
TypeError: If apply_func doesn't return a Dataset.
"""
if not hasattr(apply_func, '__call__'):
raise TypeError("apply_func must be a function.")
dataset = apply_func(self)
if not isinstance(dataset, Dataset):
raise TypeError("apply_func must return a dataset.")
return dataset
@check_device_send
def device_que(self, send_epoch_end=True, create_data_info_queue=False, queue_name=""):
"""
Return a transferred Dataset that transfers data through a device.
Args:
send_epoch_end (bool, optional): Whether to send end of sequence to device or not.
Default: ``True``.
create_data_info_queue (bool, optional): Whether to create queue which stores
types and shapes of data or not. Default: ``False``.
queue_name (str, optional): Name of queue which connects dataset processing and model
computing. Default: ``""``.
Note:
If device is Ascend, features of data will be transferred one by one. The limitation
of data transmission per time is 256M.
Returns:
Dataset, a new dataset with the above operation applied.
Examples:
>>> import mindspore.dataset as ds
>>> import time
>>>
>>> data = ds.TFRecordDataset('/path/to/TF_FILES', '/path/to/TF_SCHEMA_FILE', shuffle=ds.Shuffle.FILES)
>>> data = data.device_que()
>>> data.send()
>>> time.sleep(0.1)
>>> data.stop_send()
"""
return TransferDataset(self, send_epoch_end, create_data_info_queue, queue_name)
[docs] @check_save
def save(self, file_name, num_files=1, file_type='mindrecord'):
"""
Save the dynamic data processed by the dataset pipeline in common dataset format.
Supported dataset formats: ``'mindrecord'`` only. And you can use
:class:`mindspore.dataset.MindDataset` API to read the saved file(s).
Implicit type casting exists when saving data as ``'mindrecord'`` . The transform table shows how to do
type casting.
.. list-table:: Implicit Type Casting when Saving as `mindrecord`
:widths: 25 25 50
:header-rows: 1
* - Type in `dataset`
- Type in `mindrecord`
- Details
* - bool
- int32
- transform to int32
* - int8
- int32
-
* - uint8
- int32
-
* - int16
- int32
-
* - uint16
- int32
-
* - int32
- int32
-
* - uint32
- int64
-
* - int64
- int64
-
* - uint64
- int64
- Maybe reverse
* - float16
- float32
-
* - float32
- float32
-
* - float64
- float64
-
* - string
- string
- Multi-dimensional string not supported
* - bytes
- bytes
- Multi-dimensional bytes not supported
Note:
1. To save the samples in order, set dataset's `shuffle` to ``False`` and `num_files` to ``1``.
2. Before calling the function, do not use batch operation, repeat operation or data augmentation operations
with random attribute in map operation.
3. When array dimension is variable, one-dimensional arrays or
multi-dimensional arrays with variable dimension 0 are supported.
4. MindRecord does not support multi-dimensional string or multi-dimensional bytes.
Args:
file_name (str): Path to dataset file.
num_files (int, optional): Number of dataset files. Default: ``1`` .
file_type (str, optional): Dataset format. Default: ``'mindrecord'`` .
Examples:
>>> import mindspore.dataset as ds
>>> import numpy as np
>>>
>>> def generator_1d():
... for i in range(10):
... yield (np.array([i]),)
>>>
>>> # apply dataset operations
>>> d1 = ds.GeneratorDataset(generator_1d, ["data"], shuffle=False)
>>> d1.save('/path/to/save_file')
"""
if (_get_enc_key() is not None or _get_hash_mode() is not None) and num_files > 1:
raise RuntimeError("When encode mode or hash check is enabled, " +
"the automatic sharding function is unavailable.")
ir_tree, api_tree = self.create_ir_tree()
runtime_context = cde.PythonRuntimeContext()
runtime_context.Init()
consumer = cde.PythonSaveToDisk(file_name, num_files, file_type)
consumer.Init(ir_tree)
runtime_context.AssignConsumer(consumer)
consumer.Save()
if _get_hash_mode() is not None:
append_hash_to_file(file_name)
append_hash_to_file(file_name + ".db")
if _get_enc_key() is not None:
encrypt(file_name, _get_enc_key(), _get_enc_mode())
encrypt(file_name + ".db", _get_enc_key(), _get_enc_mode())
_set_dataset_permissions(file_name, num_files)
del api_tree
[docs] @check_tuple_iterator
def create_tuple_iterator(self, columns=None, num_epochs=-1, output_numpy=False, do_copy=True):
"""
Create an iterator over the dataset that yields samples of type list, whose elements are
the data for each column.
Args:
columns (list[str], optional): Specify the output columns and the order.
Default: ``None``, keep all the output columns and their original order.
num_epochs (int, optional): The number of epochs to iterate over the entire dataset.
Default: ``-1`` , the dataset can be iterated indefinitely.
output_numpy (bool, optional): Whether to keep the output data as NumPy ndarray, or
convert it to Tensor. Default: ``False`` .
do_copy (bool, optional): Whether to copy the data when converting output to Tensor,
or reuse the buffer for better performance, only works when `output_numpy` is ``False`` .
Default: ``True`` .
Returns:
Iterator, a dataset iterator that yields samples of type list.
Examples:
>>> import mindspore.dataset as ds
>>>
>>> dataset = ds.GeneratorDataset([i for i in range(10)], "data")
>>> num_epochs = 3
>>> iterator = dataset.create_tuple_iterator(num_epochs=num_epochs)
>>> for epoch in range(num_epochs):
... for item in iterator:
... # output is of type tuple
... print(type(item))
... break
... break
<class 'list'>
"""
if output_numpy is None:
output_numpy = False
if Dataset._noop_mode():
return DummyIterator(self, 'tuple', output_numpy)
return TupleIterator(self, columns, num_epochs, output_numpy, do_copy)
[docs] @check_dict_iterator
def create_dict_iterator(self, num_epochs=-1, output_numpy=False, do_copy=True):
"""
Create an iterator over the dataset that yields samples of type dict,
while the key is the column name and the value is the data.
Args:
num_epochs (int, optional): The number of epochs to iterate over the entire dataset.
Default: ``-1`` , the dataset can be iterated indefinitely.
output_numpy (bool, optional): Whether to keep the output data as NumPy ndarray, or
convert it to Tensor. Default: ``False`` .
do_copy (bool, optional): Whether to copy the data when converting output to Tensor,
or reuse the buffer for better performance, only works when `output_numpy` is ``False`` .
Default: ``True`` .
Returns:
Iterator, a dataset iterator that yields samples of type dict.
Examples:
>>> import mindspore.dataset as ds
>>>
>>> dataset = ds.GeneratorDataset([i for i in range(10)], "data")
>>> num_epochs = 3
>>> iterator = dataset.create_dict_iterator(num_epochs=num_epochs)
>>> for epoch in range(num_epochs):
... for item in iterator:
... # output is of type dict
... print(type(item))
... break
... break
<class 'dict'>
"""
if output_numpy is None:
output_numpy = False
if Dataset._noop_mode():
return DummyIterator(self, 'dict', output_numpy)
return DictIterator(self, num_epochs, output_numpy, do_copy)
def __iter__(self):
"""Create an iterator over the dataset."""
return self.create_tuple_iterator(num_epochs=1)
@property
def input_indexs(self):
"""
Get the column index, which represents the corresponding relationship between the data column order
and the network when using the sink mode.
Returns:
int, tuple of the input index information.
Examples:
>>> import mindspore.dataset as ds
>>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
>>> # set input_indexs
>>> dataset.input_indexs = 10
>>> print(dataset.input_indexs)
10
"""
if self._input_indexs != ():
return self._input_indexs
# find input_indexes of children
children_input_index = [child.input_indexs for child in self.children]
# in case of more than one child, return the first input_indexes
for cix in children_input_index:
if cix != ():
return cix
# if all children's input_indexes are () or the node is a leaf
return self._input_indexs
@input_indexs.setter
def input_indexs(self, value):
self._input_indexs = value
def copy_batch_size(self, value):
self._batch_size = value
def _init_tree_getters(self, getter_mode=True):
"""
Get pipeline information.
Args:
getter_mode (bool, optional): Whether to build IR tree in pull mode. Default: ``True``.
"""
ir_tree, api_tree = self.create_ir_tree(getter_mode)
runtime_context = cde.PythonRuntimeContext()
runtime_context.Init()
getter = cde.TreeGetters()
getter.Init(ir_tree)
runtime_context.AssignConsumer(getter)
return getter, runtime_context, api_tree
def __init_size_getter(self):
"""
Get pipeline information.
"""
ir_tree, api_tree = self.create_ir_tree()
runtime_context = cde.PythonRuntimeContext()
runtime_context.Init()
getter = cde.DatasetSizeGetters()
getter.Init(ir_tree)
runtime_context.AssignConsumer(getter)
return getter, runtime_context, api_tree
[docs] def get_col_names(self):
"""
Return the names of the columns in dataset.
Returns:
list, list of column names in the dataset.
Examples:
>>> import mindspore.dataset as ds
>>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
>>> col_names = dataset.get_col_names()
>>> print(col_names)
['column1']
"""
if self._col_names is None:
runtime_getter = self._init_tree_getters()
self._col_names = runtime_getter[0].GetColumnNames()
return self._col_names
[docs] @check_output_shape
def output_shapes(self, estimate=False):
"""
Get the shapes of output data.
Args:
estimate (bool): If `estimate` is ``False`` , will return the shapes of first data row.
Otherwise, will iterate the whole dataset and return the estimated shapes of data row,
where dynamic shape is marked as None (used in dynamic data shapes scenario).
Default: ``False`` .
Returns:
list, list of shapes of each column.
Examples:
>>> import mindspore.dataset as ds
>>> import numpy as np
>>>
>>> def generator1():
... for i in range(1, 100):
... yield np.ones((16, 83, 83)), np.array([i])
>>>
>>> dataset = ds.GeneratorDataset(generator1, ["data1", "data2"])
>>> output_shapes = dataset.output_shapes()
>>> print(output_shapes)
[[16, 83, 83], [1]]
"""
# cache single shape
if not estimate and self.saved_output_shapes is not None:
return self.saved_output_shapes
# cache estimate shape
if estimate and self.estimated_output_shapes is not None:
return self.estimated_output_shapes
# We have a hang problem when two-level pipeline with multiprocessing, we need to extend the life cycle
# of runtime_context. We found this hang problem only occur on output_types and output_shapes.
runtime_getter = self._init_tree_getters()
self.runtime_context = runtime_getter[1]
api_tree = runtime_getter[2]
output_shapes = runtime_getter[0].GetOutputShapes(estimate)
del api_tree
# Need to terminate the runtime context to avoid the occasional hang problem for
# Python (with multiprocessing enabled) in sink mode.
self.runtime_context.Terminate()
del self.runtime_context
if estimate:
self.estimated_output_shapes = output_shapes
else:
self.saved_output_shapes = output_shapes
return output_shapes
[docs] def output_types(self):
"""
Get the types of output data.
Returns:
list, list of data types.
Examples:
>>> import mindspore.dataset as ds
>>> import numpy as np
>>>
>>> def generator1():
... for i in range(1, 100):
... yield np.ones((16, 83, 83)).astype(np.float32), np.array([i]).astype(np.int32)
>>>
>>> dataset = ds.GeneratorDataset(generator1, ["data1", "data2"])
>>> output_types = dataset.output_types()
>>> print(output_types)
[dtype('float32'), dtype('int32')]
"""
if self.saved_output_types is None:
runtime_getter = self._init_tree_getters()
# We have a hang problem when two-level pipeline with multiprocessing, we need to extend the life cycle
# of runtime_context. We found this hang problem only occur on output_types and output_shapes.
self.runtime_context = runtime_getter[1]
api_tree = runtime_getter[2]
self.saved_output_types = runtime_getter[0].GetOutputTypes()
del api_tree
# Need to terminate the runtime context to avoid the occasional hang problem for
# Python (with multiprocessing enabled) in sink mode.
self.runtime_context.Terminate()
del self.runtime_context
return self.saved_output_types
[docs] def get_dataset_size(self):
"""
Return the number of batches in an epoch.
Returns:
int, number of batches.
Examples:
>>> import mindspore.dataset as ds
>>> import numpy as np
>>>
>>> # A generator return 66 samples
>>> def generator1():
... for i in range(66):
... yield np.ones((16, 83, 83)), np.array([i])
>>>
>>> dataset = ds.GeneratorDataset(generator1, ["data1", "data2"])
>>> dataset_size = dataset.get_dataset_size()
>>> print(dataset_size)
66
"""
if self.dataset_size is None:
runtime_getter = self.__init_size_getter()
self.dataset_size = runtime_getter[0].GetDatasetSize(False)
if self.dataset_size == 0:
logger.warning("Got 0 sample from dataset pipeline, check if drop all data or load dataset fail.")
return self.dataset_size
[docs] def num_classes(self):
"""
Get the number of classes in a dataset.
Returns:
int, number of classes.
Examples:
>>> import mindspore.dataset as ds
>>> # Read image files
>>> image_folder_dataset_dir = "/path/to/image_folder_dataset_directory"
>>> dataset = ds.ImageFolderDataset(dataset_dir=image_folder_dataset_dir)
>>> # Check how many classes exist in image folder
>>> num_classes = dataset.num_classes()
"""
if self._num_classes is None:
runtime_getter = self._init_tree_getters()
self._num_classes = runtime_getter[0].GetNumClasses()
if self._num_classes == -1:
return None
return self._num_classes
def get_sync_notifiers(self):
if self.children:
return self.children[0].get_sync_notifiers()
return {}
def disable_sync(self):
if self.children:
return self.children[0].disable_sync()
return {}
def is_sync(self):
if self.children:
return self.children[0].is_sync()
return False
[docs] def sync_update(self, condition_name, num_batch=None, data=None):
"""
Release a blocking condition and trigger callback with given data.
Args:
condition_name (str): The condition name that is used to toggle sending next row.
num_batch (Union[int, None]): The number of batches (rows) that are released.
When `num_batch` is ``None``, it will default to the number specified by the
`sync_wait` operation. Default: ``None``.
data (Any): The data passed to the callback, user defined. Default: ``None``.
Examples:
>>> import numpy as np
>>> import mindspore.dataset as ds
>>>
>>> def gen():
... for i in range(100):
... yield (np.array(i),)
>>>
>>> class Augment:
... def __init__(self, loss):
... self.loss = loss
...
... def preprocess(self, input_):
... return input_
...
... def update(self, data):
... self.loss = data["loss"]
>>>
>>> batch_size = 10
>>> dataset = ds.GeneratorDataset(gen, column_names=["input"])
>>> aug = Augment(0)
>>> dataset = dataset.sync_wait(condition_name='', num_batch=1)
>>> dataset = dataset.map(input_columns=["input"], operations=[aug.preprocess])
>>> dataset = dataset.batch(batch_size)
>>>
>>> count = 0
>>> for data in dataset.create_dict_iterator(num_epochs=1, output_numpy=True):
... count += 1
... data = {"loss": count}
... dataset.sync_update(condition_name="", data=data)
"""
if (not isinstance(num_batch, int) and num_batch is not None) or \
(isinstance(num_batch, int) and num_batch <= 0):
# throwing exception, disable all sync_wait in pipeline
self.disable_sync()
raise RuntimeError("Sync_update batch size can only be positive integer, got : {}.".format(num_batch))
notifiers_dict = self.get_sync_notifiers()
if not isinstance(condition_name, str):
raise TypeError("Argument condition_name with value {} is not of type str, but got {}."
.format(condition_name, type(condition_name)))
if condition_name not in notifiers_dict:
# throwing exception, disable all sync_wait in pipeline
self.disable_sync()
raise RuntimeError("Condition name not found.")
if num_batch is not None:
num_batch *= self.get_batch_size()
notifiers_dict[condition_name](num_batch, data)
[docs] def get_batch_size(self):
"""
Return the size of batch.
Returns:
int, the batch size of data.
Examples:
>>> import mindspore.dataset as ds
>>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
>>> dataset = dataset.batch(2)
>>> batch_size = dataset.get_batch_size()
>>> print(batch_size)
2
"""
if self._batch_size is None:
runtime_getter = self._init_tree_getters()
self._batch_size = runtime_getter[0].GetBatchSize()
if self._batch_size is None:
self._batch_size = 1
return self._batch_size
[docs] def get_repeat_count(self):
"""
Get the replication times in RepeatDataset. Default: ``1`` .
Returns:
int, the count of repeat.
Examples:
>>> import mindspore.dataset as ds
>>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
>>> dataset = dataset.repeat(5)
>>> repeat_count = dataset.get_repeat_count()
>>> print(repeat_count)
5
"""
if self._repeat_count is None:
runtime_getter = self._init_tree_getters()
self._repeat_count = runtime_getter[0].GetRepeatCount()
if self._repeat_count is None:
self._repeat_count = 1
return self._repeat_count
[docs] def get_class_indexing(self):
"""
Get the mapping dictionary from category names to category indexes.
This dictionary can be used to look up which category name corresponds to a particular category index.
Returns:
Dict[str, int], the mappings from category names to category indexes.
Examples:
>>> import mindspore.dataset as ds
>>> # Read image files
>>> image_folder_dataset_dir = "/path/to/image_folder_dataset_directory"
>>> dataset = ds.ImageFolderDataset(dataset_dir=image_folder_dataset_dir)
>>> # Check how many classes exist in image folder
>>> class_indexing = dataset.get_class_indexing()
"""
if self.children:
return self.children[0].get_class_indexing()
return {}
[docs] def reset(self):
"""
Reset the dataset for next epoch.
Examples:
>>> import mindspore.dataset as ds
>>> mind_dataset_dir = ["/path/to/mind_dataset_file"]
>>> dataset = ds.MindDataset(dataset_files=mind_dataset_dir)
>>> for _ in range(5):
... num_iter = 0
... for data in dataset.create_tuple_iterator(num_epochs=1, output_numpy=True):
... num_iter += 1
... dataset.reset()
"""
def is_shuffled(self):
"""Returns True if the dataset or its children is shuffled."""
for input_dataset in self.children:
if input_dataset.is_shuffled():
return True
return False
def is_sharded(self):
"""Returns True if the dataset or its children is sharded."""
for input_dataset in self.children:
if input_dataset.is_sharded():
return True
return False
def parse(self, children=None):
raise NotImplementedError("Dataset has to implement parse method.")
def __len__(self):
"""
Get the length of dataset.
Returns:
int, the length of dataset.
"""
return self.get_dataset_size()
@staticmethod
def _update_data_shard(num_shards, shard_id):
"""
Update the shard number and shard id if necessary.
This is normally used in distributed training mode like Parameter Server training.
"""
# If this is in distributed execution mode,
# the shard number and shard id might need to be updated according to the process's rank or role.
worker_num = _get_ps_context("worker_num")
server_num = _get_ps_context("server_num")
if _is_role_pserver() and _enable_distributed_mindrt() and (worker_num != server_num):
num_shards = worker_num
shard_id = 0
return num_shards, shard_id
def pre_parse(self, getter_mode):
if getter_mode:
if hasattr(self, "python_multiprocessing"):
self.python_multiprocessing = False
if hasattr(self, "num_parallel_workers"):
self.num_parallel_workers = 1
def post_parse(self, ir_node):
if self.cache:
ir_node = ir_node.set_cache_client(self.cache.cache_client)
if self.num_parallel_workers:
ir_node = ir_node.set_num_workers(self.num_parallel_workers)
return ir_node
def set_init_step(self, init_step):
self._global_step = init_step
def get_init_step(self):
if self._global_step is not None:
return self._global_step
if len(self.children) == 1:
return self.children[0].get_init_step()
# When there are multiple children, we cannot tell from which child to get the initial step,
# so we initialize from the beginning
return 0
class VisionBaseDataset(Dataset):
"""
Abstract class to represent a vision source dataset which produces content to the data pipeline.
"""
def __init__(self, children=None, num_parallel_workers=None, cache=None):
super().__init__(children=children, num_parallel_workers=num_parallel_workers, cache=cache)
def parse(self, children=None):
raise NotImplementedError("Dataset has to implement parse method.")
class TextBaseDataset(Dataset):
"""
Abstract class to represent a text source dataset which produces content to the data pipeline.
"""
def __init__(self, children=None, num_parallel_workers=None, cache=None):
super().__init__(children=children, num_parallel_workers=num_parallel_workers, cache=cache)
def parse(self, children=None):
raise NotImplementedError("Dataset has to implement parse method.")
def build_vocab(self, columns, freq_range, top_k, special_tokens, special_first):
"""
Function to create a Vocab from source dataset.
Desired source dataset is a text type dataset.
Build a vocab from a dataset. This would collect all the unique words in a dataset and return a vocab
which contains top_k most frequent words (if top_k is specified).
Note:
mindspore.dataset.Dataset.build_vocab is deprecated from version 2.0
and will be removed in a future version. Use mindspore.dataset.text.Vocab.from_dataset instead.
Args:
columns(Union[str, list[str]]): Column names to get words from.
freq_range(tuple[int]): A tuple of integers (min_frequency, max_frequency). Words within the frequency
range will be stored.
Naturally 0 <= min_frequency <= max_frequency <= total_words. min_frequency/max_frequency
can be set to default, which corresponds to 0/total_words separately.
top_k(int): Number of words to be built into vocab. top_k most frequent words are
taken. The top_k is taken after freq_range. If not enough top_k, all words will be taken
special_tokens(list[str]): A list of strings, each one is a special token.
special_first(bool): Whether special_tokens will be prepended/appended to vocab, If special_tokens
is specified and special_first is set to default, special_tokens will be prepended.
Returns:
Vocab, vocab built from the dataset.
"""
warnings.warn("mindspore.dataset.Dataset.build_vocab is deprecated from version 2.0 "
"and will be removed in a future version. "
"Use mindspore.dataset.text.Vocab.from_dataset instead.", DeprecationWarning)
def build_sentencepiece_vocab(self, columns, vocab_size, character_coverage, model_type, params):
"""
Function to create a SentencePieceVocab from source dataset.
Desired source dataset is a text type dataset.
Note:
mindspore.dataset.Dataset.build_sentencepiece_vocab is deprecated from version 2.0
and will be removed in a future version. Use mindspore.dataset.text.SentencePieceVocab.from_dataset instead.
Args:
columns(list[str]): Column names to get words from.
vocab_size(int): Vocabulary size.
character_coverage(float): Percentage of characters covered by the model, must be between
0.98 and 1.0 Good defaults are: 0.9995 for languages with rich character sets like
Japanese or Chinese character sets, and 1.0 for other languages with small character sets
like English or Latin.
model_type(SentencePieceModel): Model type. Choose from unigram (default), bpe, char, or word.
The input sentence must be pretokenized when using word type.
params(dict): Any extra optional parameters of sentencepiece library according to your raw data
Returns:
SentencePieceVocab, vocab built from the dataset.
"""
warnings.warn("mindspore.dataset.Dataset.build_sentencepiece_vocab is deprecated from version 2.0 "
"and will be removed in a future version. "
"Use mindspore.dataset.text.SentencePieceVocab.from_dataset instead.", DeprecationWarning)
def _build_vocab(self, columns, freq_range, top_k, special_tokens, special_first):
"""
Function to create a Vocab from source dataset.
Desired source dataset is a text type dataset.
Build a vocab from a dataset. This would collect all the unique words in a dataset and return a vocab
which contains top_k most frequent words (if top_k is specified).
Args:
columns(Union[str, list[str]]): Column names to get words from.
freq_range(tuple[int]): A tuple of integers (min_frequency, max_frequency). Words within the frequency
range will be stored.
Naturally 0 <= min_frequency <= max_frequency <= total_words. min_frequency/max_frequency
can be set to default, which corresponds to 0/total_words separately.
top_k(int): Number of words to be built into vocab. top_k most frequent words are
taken. The top_k is taken after freq_range. If not enough top_k, all words will be taken
special_tokens(list[str]): A list of strings, each one is a special token.
special_first(bool): Whether special_tokens will be prepended/appended to vocab, If special_tokens
is specified and special_first is set to default, special_tokens will be prepended.
Returns:
Vocab, vocab built from the dataset.
"""
vocab = cde.Vocab()
columns = replace_none(columns, [])
if not isinstance(columns, list):
columns = [columns]
freq_range = replace_none(freq_range, (0, 9223372036854775807))
if freq_range[0] is None:
freq_range = (0, freq_range[1])
if freq_range[1] is None:
freq_range = (freq_range[0], 9223372036854775807)
special_tokens = replace_none(special_tokens, [])
top_k = replace_none(top_k, 9223372036854775807)
ir_tree, api_tree = self.create_ir_tree()
# vocab node
vocab_node = cde.BuildVocabNode(ir_tree, vocab, columns, freq_range, top_k, special_tokens, special_first)
runtime_context = cde.PythonRuntimeContext()
runtime_context.Init()
# build vocab
consumer = cde.PythonBuildVocabConsumer()
consumer.Init(vocab_node)
runtime_context.AssignConsumer(consumer)
consumer.Start()
del api_tree
return vocab
def _build_sentencepiece_vocab(self, columns, vocab_size, character_coverage, model_type, params):
"""
Function to create a SentencePieceVocab from source dataset.
Desired source dataset is a text type dataset.
Args:
columns(list[str]): Column names to get words from.
vocab_size(int): Vocabulary size.
character_coverage(float): Percentage of characters covered by the model, must be between
0.98 and 1.0 Good defaults are: 0.9995 for languages with rich character sets like
Japanese or Chinese character sets, and 1.0 for other languages with small character sets
like English or Latin.
model_type(SentencePieceModel): Model type. Choose from unigram (default), bpe, char, or word.
The input sentence must be pretokenized when using word type.
params(dict): Any extra optional parameters of sentencepiece library according to your raw data
Returns:
SentencePieceVocab, vocab built from the dataset.
"""
if not isinstance(model_type, SentencePieceModel):
raise TypeError("Argument model_type with value {0} is not of type SentencePieceModel, but got {1}." \
.format(model_type, type(model_type)))
model_type = DE_C_INTER_SENTENCEPIECE_MODE[model_type]
vocab = cde.SentencePieceVocab()
ir_tree, api_tree = self.create_ir_tree()
# vocab node
vocab_node = cde.BuildSentenceVocabNode(ir_tree, vocab, columns, vocab_size, character_coverage, model_type,
params)
runtime_context = cde.PythonRuntimeContext()
runtime_context.Init()
# build vocab
consumer = cde.PythonBuildVocabConsumer()
consumer.Init(vocab_node)
runtime_context.AssignConsumer(consumer)
consumer.Start()
del api_tree
return vocab
class AudioBaseDataset(Dataset):
"""
Abstract class to represent a audio source dataset which produces content to the data pipeline.
"""
def __init__(self, children=None, num_parallel_workers=None, cache=None):
super().__init__(children=children, num_parallel_workers=num_parallel_workers, cache=cache)
def parse(self, children=None):
raise NotImplementedError("Dataset has to implement parse method.")
class UnionBaseDataset(VisionBaseDataset, TextBaseDataset, AudioBaseDataset):
"""
Abstract class to represent a union source dataset which produces content to the data pipeline.
"""
def __init__(self, children=None, num_parallel_workers=None, cache=None):
super().__init__(children=children, num_parallel_workers=num_parallel_workers, cache=cache)
def parse(self, children=None):
raise NotImplementedError("Dataset has to implement parse method.")
class SourceDataset(Dataset):
"""
Abstract class to represent a source dataset which produces content to the data pipeline.
"""
def __init__(self, num_parallel_workers=None, num_samples=None, shuffle=True, num_shards=None, shard_id=None,
cache=None):
super().__init__(num_parallel_workers=num_parallel_workers, cache=cache)
self.num_samples = replace_none(num_samples, 0)
self.num_shards = replace_none(num_shards, 1)
self.shard_id = replace_none(shard_id, 0)
if shuffle is not None and not isinstance(shuffle, (bool, Shuffle)):
raise TypeError("shuffle must be of boolean or enum of 'Shuffle' values like 'Shuffle.GLOBAL' or "
"'Shuffle.FILES' or 'Shuffle.INFILE'.")
self.shuffle_flag = 2 # Global shuffle
if not isinstance(shuffle, Shuffle):
if shuffle is None or shuffle:
self.shuffle_flag = 2 # Global shuffle
else:
self.shuffle_flag = 0 # No shuffle
else:
if shuffle == Shuffle.GLOBAL:
self.shuffle_flag = 2 # Global shuffle
elif shuffle == Shuffle.FILES:
self.shuffle_flag = 1 # Files shuffle
elif shuffle == Shuffle.INFILE:
self.shuffle_flag = 3 # Infile shuffle
def parse(self, children=None):
raise NotImplementedError("Dataset has to implement parse method.")
@staticmethod
def _find_files(patterns):
"""
Utility function to search for files with the given glob patterns.
Args:
patterns (Union[str, list[str]]): String or list of patterns to be searched.
Returns:
list, list of files.
"""
if not isinstance(patterns, list):
patterns = [patterns]
file_list = []
unmatched_patterns = []
for pattern in patterns:
matches = [match for match in glob.glob(pattern, recursive=True) if os.path.isfile(match)]
if matches:
file_list.extend(matches)
else:
unmatched_patterns.append(pattern)
if unmatched_patterns:
raise ValueError("The following patterns did not match any files: {}.".format(unmatched_patterns))
if file_list: # not empty
return file_list
raise ValueError("The list of path names matching the patterns is empty.")
def is_shuffled(self):
return self.shuffle_flag > 0
def is_sharded(self):
if self.num_shards is not None:
return self.num_shards > 1
return False
class MappableDataset(SourceDataset):
"""
Abstract class to represent a source dataset which supports use of samplers.
"""
def parse(self, children=None):
raise NotImplementedError("Dataset has to implement parse method.")
def __init__(self, num_parallel_workers=None, sampler=None, num_samples=None, shuffle=None, num_shards=None,
shard_id=None, cache=None):
num_shards, shard_id = self._update_data_shard(num_shards, shard_id)
super().__init__(num_parallel_workers=num_parallel_workers, num_samples=num_samples, shuffle=shuffle,
num_shards=num_shards, shard_id=shard_id, cache=cache)
self.shuffle_flag = replace_none(shuffle, True)
self.sampler = samplers.select_sampler(num_samples, sampler, shuffle, num_shards, shard_id)
[docs] def add_sampler(self, new_sampler):
"""
Add a child sampler for the current dataset.
Args:
new_sampler (Sampler): The child sampler to be added.
Examples:
>>> import mindspore.dataset as ds
>>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
>>>
>>> new_sampler = ds.DistributedSampler(10, 2)
>>> dataset.add_sampler(new_sampler)
"""
# Note: By adding a sampler, the sampled IDs will flow to the new_sampler
# after first passing through the current samplers attached to this dataset.
self.dataset_size = None
new_sampler.add_child(self.sampler)
self.sampler = new_sampler
[docs] def use_sampler(self, new_sampler):
"""
Replace the last child sampler of the current dataset, remaining the parent sampler unchanged.
Args:
new_sampler (Sampler): The new sampler to replace with.
Examples:
>>> import mindspore.dataset as ds
>>> dataset = ds.GeneratorDataset([i for i in range(10)], "column1")
>>>
>>> # use a DistributedSampler instead
>>> new_sampler = ds.DistributedSampler(10, 2)
>>> dataset.use_sampler(new_sampler)
"""
if new_sampler is None:
raise TypeError("Input sampler can not be None.")
if not isinstance(new_sampler, (samplers.BuiltinSampler, samplers.Sampler)):
raise TypeError("Input sampler is not an instance of a sampler.")
self.dataset_size = None
self.sampler = self.sampler.child_sampler
self.add_sampler(new_sampler)
def is_shuffled(self):
return self.sampler.is_shuffled()
def is_sharded(self):
return self.sampler.is_sharded()
@check_split
def split(self, sizes, randomize=True):
"""
Split the dataset into smaller, non-overlapping datasets.
Args:
sizes (Union[list[int], list[float]]): If a list of integers [s1, s2, …, sn] is
provided, the dataset will be split into n datasets of size s1, size s2, …, size sn
respectively. If the sum of all sizes does not equal the original dataset size, an
error will occur.
If a list of floats [f1, f2, …, fn] is provided, all floats must be between 0 and 1
and must sum to 1, otherwise an error will occur. The dataset will be split into n
Datasets of size round(f1*K), round(f2*K), …, round(fn*K) where K is the size of the
original dataset.
If after rounding:
- Any size equals 0, an error will occur.
- The sum of split sizes < K, the difference will be added to the first split.
- The sum of split sizes > K, the difference will be removed from the first large
enough split such that it will have at least 1 row after removing the difference.
randomize (bool, optional): Determines whether or not to split the data randomly. Default: ``True``.
If ``True``, the data will be randomly split. Otherwise, each split will be created with
consecutive rows from the dataset.
Note:
1. There is an optimized split function, which will be called automatically when the dataset
that calls this function is a MappableDataset.
2. Dataset should not be sharded if split is going to be called. Instead, create a
:class:`mindspore.dataset.DistributedSampler` and specify a split to shard after splitting.
If the dataset is sharded after a split, it is strongly recommended setting the same
seed in each instance of execution, otherwise each shard may not be part of the same
split (see Examples).
3. It is strongly recommended to not shuffle the dataset, but set `randomize` to ``True`` instead.
Shuffling the dataset may not be deterministic, which means the data in each split
will be different in each epoch. Furthermore, if sharding occurs after split, each
shard may not be part of the same split.
Returns:
Tuple[Dataset], a tuple of new datasets split from the original one.
Raises:
RuntimeError: If get_dataset_size returns None or is not supported for this dataset.
RuntimeError: If `sizes` is list of integers and sum of all elements in sizes does not
equal the dataset size.
RuntimeError: If `sizes` is list of float and there is a split with size 0 after calculations.
RuntimeError: If the dataset is sharded prior to calling split.
ValueError: If `sizes` is list of float and not all floats are between 0 and 1, or if the
floats don't sum to 1.
Examples:
>>> import mindspore.dataset as ds
>>> # Since many datasets have shuffle on by default, set shuffle to False if split will be called!
>>> image_folder_dataset_dir = "/path/to/image_folder_dataset_directory"
>>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir, shuffle=False)
>>>
>>> # Set the seed, and tell split to use this seed when randomizing.
>>> # This is needed because sharding will be done later
>>> ds.config.set_seed(58)
>>> train_dataset, test_dataset = dataset.split([0.9, 0.1])
>>>
>>> # To shard the train dataset, use a DistributedSampler
>>> train_sampler = ds.DistributedSampler(10, 2)
>>> train_dataset.use_sampler(train_sampler)
"""
if self.is_shuffled():
logger.warning("Dataset is shuffled before split.")
if self.is_sharded():
raise RuntimeError("Dataset should not be sharded before split.")
absolute_sizes = self._get_absolute_split_sizes(sizes)
splits = []
current_split_start_index = 0
for size in absolute_sizes:
ds = copy.deepcopy(self)
ds.dataset_size = None
if randomize:
# want to shuffle the same way every epoch before split, we are assuming
# that the user will call set_seed
random_sampler = samplers.RandomSampler()
random_sampler.reshuffle_each_epoch = False
ds.add_sampler(random_sampler)
subset_sampler = samplers.SequentialSampler(current_split_start_index, size)
ds.add_sampler(subset_sampler)
# add sequential sampler, so that if user calls use_sampler, we will
# get rid of the sequential sampler instead of something we need
ds.add_sampler(samplers.SequentialSampler())
splits.append(ds)
current_split_start_index += size
return tuple(splits)
class BucketBatchByLengthDataset(UnionBaseDataset):
"""
The result of applying BucketBatchByLength operation to the input dataset.
"""
def __init__(self, input_dataset, column_names, bucket_boundaries, bucket_batch_sizes, element_length_function,
pad_info, pad_to_bucket_boundary, drop_remainder):
super().__init__(children=input_dataset)
self.column_names = to_list(column_names)
self.bucket_boundaries = replace_none(bucket_boundaries, [])
self.bucket_batch_sizes = replace_none(bucket_batch_sizes, [])
self.element_length_function = element_length_function
self.pad_info = replace_none(pad_info, {})
self.pad_to_bucket_boundary = replace_none(pad_to_bucket_boundary, False)
self.drop_remainder = replace_none(drop_remainder, False)
def parse(self, children=None):
return cde.BucketBatchByLengthNode(children[0], self.column_names, self.bucket_boundaries,
self.bucket_batch_sizes, self.element_length_function, self.pad_info,
self.pad_to_bucket_boundary, self.drop_remainder)
def _check_shm_usage(num_worker, queue_size, in_rowsize, out_rowsize):
"""
Check sufficient shared memory is available for shared memory queues
when training in parallel mode.
"""
threshold_ratio = 0.8
# Verify available size only when using static shared memory on Linux
if platform.system().lower() not in {"windows", "darwin"} and in_rowsize != -1 and out_rowsize != -1:
device_num = _get_device_num()
# In the cluster, _get_device_num indicates the number of the entire cluster. The maximum number of cards
# on the ascend server is 8.
if device_num > 1:
device_num = min(device_num, 8)
shm_estimate_usage = device_num * num_worker * \
(queue_size + 2) * (in_rowsize + out_rowsize) * 1024 * 1024
try:
shm_available = psutil.disk_usage('/dev/shm').free
if shm_estimate_usage >= threshold_ratio * shm_available:
raise RuntimeError(
"Insufficient shared memory available. Required: {}, Available: {}. "
"The required memory can't exceed 80% of the available shared memory, "
"it's recommended to reduce memory usage by following methods:\n"
"1. reduce value of parameter max_rowsize or num_parallel_workers.\n"
"2. reduce prefetch size by set_prefetch_size().\n"
"3. disable shared memory by set_enable_shared_mem().".format(shm_estimate_usage, shm_available))
except FileNotFoundError:
raise RuntimeError("Expected /dev/shm to exist.")
class BatchDataset(UnionBaseDataset):
"""
The result of applying Batch operation to the input dataset.
Args:
input_dataset (Dataset): Input Dataset to be batched.
batch_size (Union[int, function]): The number of rows each batch is created with. An
int or callable which takes exactly 1 parameter, BatchInfo.
drop_remainder (bool, optional): Determines whether or not to drop the last
possibly incomplete batch. Default: ``False``. If True, and if there are less
than batch_size rows available to make the last batch, then those rows will
be dropped and not propagated to the child node.
num_parallel_workers (int, optional): Number of workers to process the dataset in parallel. Default: ``None``.
per_batch_map (callable, optional): Per batch map callable. A callable which takes
(list[Tensor], list[Tensor], ..., BatchInfo) as input parameters. Each list[Tensor] represents a batch of
Tensors on a given column. The number of lists should match with number of entries in input_columns. The
last parameter of the callable must always be a BatchInfo object.
input_columns (Union[str, list[str]], optional): List of names of the input columns. The size of the list must
match with signature of per_batch_map callable.
output_columns (Union[str, list[str]], optional): List of names assigned to the columns outputted by
the last operation. This parameter is mandatory if len(input_columns) !=
len(output_columns). The size of this list must match the number of output
columns of the last operation. Default: ``None``, output columns will have the same
name as the input columns, i.e., the columns will be replaced.
max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
allocation to copy data between processes, the total occupied shared memory will increase as
``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set to -1,
shared memory will be dynamically allocated with the actual size of data. This is only used if
``python_multiprocessing`` is set to True. If it is an int value, it represents
``input_columns`` and ``output_columns`` use this value as the unit to create shared memory.
If it is a list, the first element represents the ``input_columns`` use this value as the unit to
create shared memory, and the second element represents ``output_columns`` use this value as the unit
to create shared memory. Default: 16.
"""
def __init__(self, input_dataset, batch_size, drop_remainder=False, num_parallel_workers=None, per_batch_map=None,
input_columns=None, output_columns=None, python_multiprocessing=False, max_rowsize=16):
super().__init__(children=input_dataset, num_parallel_workers=num_parallel_workers)
if BatchDataset._is_ancestor_of_repeat(input_dataset):
logger.warning("Repeat is located before batch, data from two epochs can be batched together.")
BatchDataset._update_batch_size_for_syncwait(input_dataset, batch_size)
# if batch_size is callable, set batch_size to 1 and batch_size_func to that callable function
self.batch_size = batch_size if not callable(batch_size) else 1
self.batch_size_func = None if not callable(batch_size) else batch_size
self.drop_remainder = replace_none(drop_remainder, False)
self.per_batch_map = per_batch_map
self.input_columns = to_list(input_columns)
self.output_columns = to_list(output_columns)
self.python_multiprocessing = python_multiprocessing
self.process_pool = None
if isinstance(max_rowsize, int):
self.max_rowsize = [max_rowsize * self.batch_size] * 2 if max_rowsize != -1 else [max_rowsize, max_rowsize]
else:
self.max_rowsize = [max_rowsize[0] * self.batch_size, max_rowsize[1] * self.batch_size]
def __del__(self):
if hasattr(self, "process_pool") and self.process_pool is not None:
self.process_pool.terminate()
del self.process_pool
def parse(self, children=None):
return cde.BatchNode(children[0], self.batch_size, self.drop_remainder, False, self.input_columns,
self.output_columns, self.batch_size_func, self.per_batch_map, {},
self.process_pool)
@staticmethod
def _is_ancestor_of_repeat(dataset):
"""
Utility function to find the case where repeat is used before batch.
Args:
dataset (Dataset): Dataset to be checked.
Returns:
bool, whether repeat is used before batch.
"""
if isinstance(dataset, RepeatDataset):
return True
flag = False
for input_dataset in dataset.children:
flag = flag | BatchDataset._is_ancestor_of_repeat(input_dataset)
return flag
@staticmethod
def _update_batch_size_for_syncwait(dataset, batch_size):
"""
Utility function to notify batch size to sync_wait.
Args:
dataset (Dataset): Dataset to be checked.
batch_size (int): batch size to notify.
"""
if isinstance(dataset, SyncWaitDataset):
dataset.update_sync_batch_size(batch_size)
for input_dataset in dataset.children:
BatchDataset._update_batch_size_for_syncwait(input_dataset, batch_size)
def __deepcopy__(self, memodict):
return self.__safe_deepcopy__(memodict, exclude=("per_batch_map", "batch_size_func", "__transfer_dataset__"))
# Iterator bootstrap will be called on iterator construction.
# A deep copy of Dataset object is created prior of iterator_bootstrap.
# This method will create per iterator process pool and bind pyfunc execution to the pool.
def iterator_bootstrap(self):
"""
Per iterator bootstrap callback.
"""
if self.python_multiprocessing and platform.system().lower() == 'windows':
logger.warning("Python multiprocessing is not supported on Windows platform.")
if self.python_multiprocessing and get_debug_mode():
logger.warning("Python multiprocessing is not supported in debug mode."
" Ignoring Python multiprocessing for batch operation.")
self.python_multiprocessing = False
if self.python_multiprocessing and platform.system().lower() != 'windows':
if self.per_batch_map is None:
logger.warning("per_batch_map is None so python_multiprocessing is ignored for batch.")
return
# If user didn't specify num_parallel_workers, set it to default
if self.num_parallel_workers is None:
self.num_parallel_workers = get_num_parallel_workers()
self.process_pool = _PythonMultiprocessing(str(self), self.num_parallel_workers, [self.per_batch_map],
self.max_rowsize)
# Wrap per_batch_map into _PythonCallable
self.per_batch_map = _PythonCallable(self.per_batch_map, 0, self.process_pool)
else:
if self.per_batch_map is not None:
self.per_batch_map = FuncWrapper(self.per_batch_map)
[docs]class BatchInfo(cde.CBatchInfo):
"""
This class helps to get dataset information dynamically when the input of `batch_size` or `per_batch_map`
in `batch` operation is a callable object.
"""
[docs] def get_batch_num(self):
"""
Return the batch number being processed in current epoch, start from 0.
Examples:
>>> # Create a dataset where its batch size is dynamic
>>> # Define a callable batch size function and let batch size increase 1 each time.
>>> import mindspore.dataset as ds
>>> from mindspore.dataset import BatchInfo
>>>
>>> dataset = ds.GeneratorDataset([i for i in range(3)], "column1", shuffle=False)
>>> def add_one(BatchInfo):
... return BatchInfo.get_batch_num() + 1
>>> dataset = dataset.batch(batch_size=add_one)
>>> print(list(dataset))
[[Tensor(shape=[1], dtype=Int64, value= [0])], [Tensor(shape=[2], dtype=Int64, value= [1, 2])]]
"""
return
[docs] def get_epoch_num(self):
"""
Return the epoch number, start from 0.
Examples:
>>> # Create a dataset where its batch size is dynamic
>>> # Define a callable batch size function and let batch size increase 1 each epoch.
>>> import mindspore.dataset as ds
>>> from mindspore.dataset import BatchInfo
>>>
>>> dataset = ds.GeneratorDataset([i for i in range(4)], "column1", shuffle=False)
>>> def add_one_by_epoch(BatchInfo):
... return BatchInfo.get_epoch_num() + 1
>>> dataset = dataset.batch(batch_size=add_one_by_epoch)
>>>
>>> result = []
>>> epoch = 2
>>> iterator = dataset.create_tuple_iterator(num_epochs=epoch)
>>> for i in range(epoch):
... result.extend(list(iterator))
>>> # result:
>>> # [[Tensor(shape=[1], dtype=Int64, value= [0])], [Tensor(shape=[1], dtype=Int64, value= [1])],
>>> # [Tensor(shape=[1], dtype=Int64, value= [2])], [Tensor(shape=[1], dtype=Int64, value= [3])],
>>> # [Tensor(shape=[2], dtype=Int64, value= [0, 1])], [Tensor(shape=[2], dtype=Int64, value= [2, 3])]]
"""
return
class BlockReleasePair:
"""
The blocking condition class used by SyncWaitDataset.
Args:
init_release_rows (int): Number of lines to allow through the pipeline.
callback (function): The callback function that will be called when release is called. Default: ``None``.
"""
def __init__(self, init_release_rows, callback=None):
if isinstance(init_release_rows, int) and init_release_rows <= 0:
raise ValueError("release_rows need to be greater than 0.")
self.row_count = -init_release_rows
self.cv = threading.Condition()
self.callback = callback
self.default_rows = init_release_rows
self.disable = False
def __deepcopy__(self, memodict):
return self
def reset(self):
with self.cv:
self.row_count = -self.default_rows
self.cv.notify_all()
def update_batched_size(self, batch_size):
# sanity check
if isinstance(batch_size, int) and batch_size <= 0:
raise ValueError("batch_size need to be greater than 0.")
# should only use before the pipeline creates
self.row_count *= batch_size
self.default_rows *= batch_size
def block_func(self):
"""
Function for handing blocking condition.
Returns:
bool, True.
"""
with self.cv:
# if disable is true, the always evaluate to true
not_time_out = self.cv.wait_for(lambda: (self.row_count < 0 or self.disable),
timeout=get_callback_timeout())
# time_out will be False if time out occurs
if not not_time_out:
logger.warning("Timeout happened in sync_wait, maybe dataset.sync_update(condition=...) "
"is not added after dataset.create_dict_iterator(...), now disabling lock.")
self.disable = True
self.row_count += 1
return True
def release_func(self, pass_rows=None, data=None):
with self.cv:
if pass_rows is None:
pass_rows = self.default_rows
self.row_count -= pass_rows
if self.callback is not None:
self.callback(data)
self.cv.notify_all()
def disable_lock(self):
with self.cv:
self.disable = True
self.cv.notify_all()
class PaddedBatchDataset(UnionBaseDataset):
"""
The result of applying Batch operation to the input dataset.
Args:
input_dataset (Dataset): Input Dataset to be batched.
batch_size (Union[int, function]): The number of rows each batch is created with. An
int or callable which takes exactly 1 parameter, BatchInfo.
drop_remainder (bool, optional): Determines whether or not to drop the last
possibly incomplete batch. Default: ``False``. If True, and if there are less
than batch_size rows available to make the last batch, then those rows will
be dropped and not propagated to the child node.
num_parallel_workers (int, optional): Number of workers to process the dataset in parallel. Default: ``None``.
pad_info (dict, optional): Whether to perform padding on selected columns. pad_info={"col1":([224,224],0)}
will pad column with name "col1" to a tensor of size [224,224] and fill the missing with 0.
"""
def __init__(self, input_dataset, batch_size, drop_remainder=False, num_parallel_workers=None, pad_info=None):
super().__init__(children=input_dataset, num_parallel_workers=num_parallel_workers)
if PaddedBatchDataset._is_ancestor_of_repeat(input_dataset):
logger.warning("Repeat is located before padded_batch, data from two epochs can be batched together.")
PaddedBatchDataset._update_batch_size_for_syncwait(input_dataset, batch_size)
# if batch_size is callable, set batch_size to 1 and batch_size_func to that callable function
self.batch_size = batch_size if not callable(batch_size) else 1
self.batch_size_func = None if not callable(batch_size) else batch_size
self.drop_remainder = replace_none(drop_remainder, False)
self.pad = bool(pad_info is not None)
self.pad_info = replace_none(pad_info, dict())
def parse(self, children=None):
return cde.BatchNode(children[0], self.batch_size, self.drop_remainder, self.pad, [],
[], self.batch_size_func, None, self.pad_info, None)
@staticmethod
def _is_ancestor_of_repeat(dataset):
"""
Utility function to find the case where repeat is used before batch.
Args:
dataset (Dataset): Dataset to be checked.
Returns:
bool, whether repeat is used before batch.
"""
if isinstance(dataset, RepeatDataset):
return True
flag = False
for input_dataset in dataset.children:
flag = flag | PaddedBatchDataset._is_ancestor_of_repeat(input_dataset)
return flag
@staticmethod
def _update_batch_size_for_syncwait(dataset, batch_size):
"""
Utility function to notify batch size to sync_wait.
Args:
dataset (Dataset): Dataset to be checked.
batch_size (int): batch size to notify.
"""
if isinstance(dataset, SyncWaitDataset):
dataset.update_sync_batch_size(batch_size)
for input_dataset in dataset.children:
PaddedBatchDataset._update_batch_size_for_syncwait(input_dataset, batch_size)
def __deepcopy__(self, memodict):
return self.__safe_deepcopy__(memodict, exclude=("batch_size_func", "__transfer_dataset__"))
class SyncWaitDataset(UnionBaseDataset):
"""
The result of adding a blocking condition to the input Dataset.
Args:
input_dataset (Dataset): Input dataset to apply flow control.
num_batch (int): Number of batches without blocking at the start of each epoch.
condition_name (str): Condition name that is used to toggle sending next row.
callback (function): Callback function that will be invoked when sync_update is called. Default: ``None``.
Raises:
RuntimeError: If condition name already exists.
"""
def __init__(self, input_dataset, condition_name, num_batch, callback=None):
super().__init__(children=input_dataset)
# set to the default value, waiting for the batch to update it
self._condition_name = condition_name
if isinstance(num_batch, int) and num_batch <= 0:
raise ValueError("num_batch need to be greater than 0.")
self._pair = BlockReleasePair(num_batch, callback)
if self._condition_name in self.children[0].get_sync_notifiers():
raise RuntimeError("Condition name is already in use.")
logger.info("Please remember to add dataset.sync_update(condition=%s), otherwise hanging will result. "
"If dataset.sync_update(condition=%s) has already been added, you can ignore the info.",
condition_name, condition_name)
def parse(self, children=None):
return cde.SyncWaitNode(children[0], self._condition_name, self._pair.block_func)
def get_sync_notifiers(self):
return {**self.children[0].get_sync_notifiers(), **{self._condition_name: self._pair.release_func}}
def is_sync(self):
return True
def update_sync_batch_size(self, batch_size):
if isinstance(batch_size, int) and batch_size <= 0:
raise ValueError("num_batch need to be greater than 0.")
self._pair.update_batched_size(batch_size)
def disable_sync(self):
logger.info("Disabling Sync")
self._pair.disable_lock()
@staticmethod
def _is_ancestor_of_batch(dataset):
"""
Utility function to find the case where sync_wait is used before batch.
Args:
dataset (Dataset): Dataset to be checked.
Returns:
bool, whether sync_wait is used before batch.
"""
if isinstance(dataset, (BatchDataset, PaddedBatchDataset)):
return True
flag = False
for input_dataset in dataset.children:
flag = flag | SyncWaitDataset._is_ancestor_of_batch(input_dataset)
return flag
def iterator_bootstrap(self):
self._pair.reset()
class ShuffleDataset(UnionBaseDataset):
"""
The result of applying Shuffle operation to the input Dataset.
Args:
input_dataset (Dataset): Input Dataset to be shuffled.
buffer_size (int): Size of the buffer.
Raises:
RuntimeError: If exist sync operations before shuffle.
"""
def __init__(self, input_dataset, buffer_size):
super().__init__(children=input_dataset)
self.buffer_size = buffer_size
self.reshuffle_each_epoch = True
if self.is_sync():
raise RuntimeError("No shuffle after sync operators.")
def parse(self, children=None):
return cde.ShuffleNode(children[0], self.buffer_size, self.reshuffle_each_epoch)
def is_shuffled(self):
return True
# Pyfunc collection for multiprocess pyfunc
# This global variable will only be used within subprocesses
_OP_NAME = dict()
_OP_PROCESS = dict()
# PythonCallable wrapper for multiprocess pyfunc
class _PythonCallable:
"""
Internal Python function wrapper for multiprocessing pyfunc.
"""
def __init__(self, py_callable, idx, pool=None):
# Original Python callable from user.
self.py_callable = py_callable
# Process pool created for current iterator.
self.pool = pool
# Python callable index
self.idx = idx
def __call__(self, *args):
result = None
get_data_from_worker_process = False
while get_data_from_worker_process is False:
if self.pool.is_running() and check_iterator_cleanup() is False:
try:
result = self.pool.execute(self.idx, *args)
except multiprocessing.TimeoutError:
continue
get_data_from_worker_process = True
else:
# worker process is stopped
logger.info("The worker process of map operation is stopped. "
"So return None to main thread and break the main thread.")
return None
# got value from worker process
if not isinstance(result, tuple) and get_data_from_worker_process is True:
result = (result,)
return result
def to_json(self):
return self.py_callable.to_json()
# used when python_multiprocessing=True in map
class Pipe:
"""
Class to handle communication between the master process and the worker processes.
"""
def __init__(self, warning_ctl, shared_memory=False, max_rowsize=16):
self.shared_memory = shared_memory
self.eof = multiprocessing.Event()
if self.shared_memory:
self.in_queue = _SharedQueue(1, warning_ctl, max_rowsize=max_rowsize[0])
self.res_queue = _SharedQueue(1, warning_ctl, max_rowsize=max_rowsize[1])
else:
self.in_queue = _Queue(1)
self.res_queue = _Queue(1)
self.in_queue.cancel_join_thread() # Ensure that the process does not hung when exiting
def master_send(self, func_index, data):
self.in_queue.put_nowait((func_index, *data))
def master_receive(self):
if self.eof is None:
raise RuntimeError("EOF is none when get data from worker.")
if self.eof.is_set():
return None
return self.res_queue.get(timeout=1)
def master_close(self):
self.eof.set()
self.send_finish_signal_to_worker()
self.send_finish_signal()
def send_finish_signal(self):
self.worker_send(None)
def send_finish_signal_to_worker(self):
self.master_send(0, "QUIT")
def worker_send(self, data):
self.res_queue.put_until(data, timeout=1, exit_signal=self.eof)
def worker_receive(self):
result = self.in_queue.get_until(timeout=1, exit_signal=self.eof)
if result is None:
return result
if len(result) == 1:
raise RuntimeError(f"Corrupted data. Worker received {len(result)} elements, it should be more than 1.")
func_index, *data = result
return func_index, tuple(data)
def _main_process_already_exit():
"""
Judge whether main process already exit.
"""
ppid = os.getppid()
if (platform.system().lower() != 'windows' and
not _PythonMultiprocessing.is_process_alive(ppid)):
return True
return False
def _worker_loop(operations, pipe, worker_id):
"""
Multiprocess worker process loop.
"""
# Ensure that the process does not hung when exiting
pipe.res_queue.cancel_join_thread()
def _ignore_sigint():
"""
We need to ignore sigint signal here so subprocesses can exit normally and clear.
"""
signal.signal(signal.SIGINT, signal.SIG_IGN)
# If the default random seed has not been changed, there is no need to fix the randomness.
# Otherwise, set the random seed for each child process to "base_seed + worker_id" to ensure
# that the random results of each process are different.
if get_seed() != 5489:
set_seed(get_seed() + worker_id)
while not _main_process_already_exit():
_ignore_sigint()
result = pipe.worker_receive()
if result is None:
return
(idx, input_tensors) = result
if input_tensors == "QUIT":
break
try:
output_tensors = operations[idx](*input_tensors)
pipe.worker_send(output_tensors)
except Exception:
pipe.worker_send(ExceptionHandler(where="in map(or batch) worker and execute Python function"))
# Do not return
# release the queue when stop the worker by master
del pipe.in_queue
del pipe.res_queue
def worker_target(operations, worker_id):
return lambda pipe: _worker_loop(operations, pipe, worker_id)
class _MPWorker(multiprocessing.Process):
"""
Worker process for multiprocessing.
"""
def __init__(self, operations, warning_ctl, max_rowsize=16, worker_id=0):
shared_memory = get_enable_shared_mem()
self.pipe = Pipe(warning_ctl, shared_memory=shared_memory, max_rowsize=max_rowsize)
self.check_interval = get_multiprocessing_timeout_interval()
super().__init__(target=worker_target(operations, worker_id), name="MapWorker" + str(worker_id),
args=(self.pipe,), daemon=True)
def execute(self, idx, *args):
"""Acquiring data from a worker in an infinite loop"""
self.pipe.master_send(idx, args)
time_s = time.time()
wait_count = 1
while True:
cost_time = time.time() - time_s
if cost_time / self.check_interval >= wait_count:
wait_count += 1
logger.warning("It has been waiting for " + "%.3f" % cost_time + "s because the sub-process "
"worker of the map operation is hanging. "
"Check whether the user defined data transform is too slow or the "
"output data is too large. You can also set the timeout interval by "
"ds.config.set_multiprocessing_timeout_interval to adjust the output frequency "
"of this log.")
pid = self.pid
logger.warning("Map worker subprocess ID {} is stuck.".format(pid))
install_status, _ = subprocess.getstatusoutput("py-spy --version")
if install_status == 0:
stack = subprocess.getoutput("py-spy dump -p {} -l".format(pid))
logger.warning("Map worker subprocess stack:\n{}".format(stack))
else:
logger.warning("Please `pip install py-spy` to get the stacks of the stuck process.")
try:
res = self.pipe.master_receive()
# Because there is no need to copy when creating Tensors in the C++layer, it reduces the time
# from np.ndarray to C++Tensor creation. However, when using shared memory in multiple processes,
# the address of the shared memory will always be passed to subsequent nodes in the dataset pipeline,
# and the shared memory will also be written by the current node, causing dirty data to be accessed
# by subsequent nodes in the pipeline. So make a memory copy here to solve the problem of
# shared memory being contaminated.
if get_enable_shared_mem():
res = copy.deepcopy(res)
except queue.Empty:
continue
if res is None:
# receive finish signal
return None
if isinstance(res, ExceptionHandler):
res.reraise()
return res
def close(self):
try:
if self.is_alive():
# release the eager executor which is used by current process
transforms.transforms.clean_unused_executors()
logger.info(f"Closing worker with PID: {self.pid}")
self.pipe.master_close()
# del the handle which hold by master
del self.pipe.in_queue
del self.pipe.res_queue
super().terminate()
super().join()
super().close()
except ValueError:
# Process has been closed already
return
return
def is_alive(self):
try:
return super().is_alive()
except ValueError:
return False
class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime):
"""
A wrapper to multiprocessing.pool that performs cleanup and ensure proper termination of forked processes.
"""
class _ExceptHookHandler:
"""
Internal class ExceptionHandler
"""
def __init__(self):
self.origin_hook = sys.excepthook
sys.excepthook = self.__handler_exception
@staticmethod
def mp_pool_exit_preprocess():
if check_iterator_cleanup() is False:
# Set the iterator_cleanup flag to True before exiting, and wait 3s for all apply_async
# applied to the multiprocessing task to prevent multiprocessing from hang when exiting
_set_iterator_cleanup()
time.sleep(3)
def __handler_exception(self, ex_type, value, tb):
self.origin_hook(ex_type, value, tb)
self.mp_pool_exit_preprocess()
def __init__(self, op_name, num_parallel_workers, operations, max_rowsize=16):
super(_PythonMultiprocessing, self).__init__()
self.op_name = op_name
self.num_parallel_workers = num_parallel_workers
self.operations = operations
self.max_rowsize = max_rowsize
self.workers = None
self.pids = None
self.op_id = -1
self.queues_map = {}
self.next_queue = 0
self.eot = None
self.watch_dog = None
self.ppid = os.getpid()
self.hook = None
self.warning_ctl = None
# cache thread (get_ident()) to worker_id mapping in Python layer
self.python_threads_to_workers = {}
self.eof = None
def __del__(self):
try:
self.terminate()
except TypeError:
pass
# This wait function is for cleaning zombie subprocesses
@staticmethod
def wait_pid():
"""
This function is used by the main process to release subprocess resources.
"""
try:
while True:
child_pid, _ = os.waitpid(-1, os.WNOHANG)
if child_pid == 0:
break
except OSError:
# waitpid may be failed for some reasons so we ignore this error
pass
# Dataset need watch_dog thread to monitoring fork multi-processing,
# and thread can't be a member function otherwise python won't collect and release resources.
@staticmethod
def _watch_dog(eot, workers):
"""
This thread is for monitoring subprocesses forked by GeneratorDataset/map/batch
"""
if not isinstance(workers, list):
raise TypeError("[Internal Error] The 2nd parameter of watch dog thread should be list of process, "
"but got {}.".format(type(workers)))
while not eot.is_set():
# Monitoring and count how many subprocesses already exit
clear_subprocess_timeout = _PythonMultiprocessing._monitor_subprocess_exit(workers)
# If find subprocess exit, we will wait for 30s and do some waitpid operations
if clear_subprocess_timeout > 0:
start = time.time()
while time.time() - start < clear_subprocess_timeout:
# We need to distinguishing get_dataset_size or train finished normally and hang scenario.
# If get_dataset_size or train finished normally, _stop_subprocess can be execute and
# self.need_abort can be set to True. If main process is hang in get(), self.need_abort
# will never set to True, then we wait for 30s and kill main process
if eot.is_set():
return
# Sometimes subprocess may be zombie, so in 30s we can wait and do some useful tasks(waitpid).
_PythonMultiprocessing.wait_pid()
# multiprocessing.queue may hang in .get() forever when put() process was killed.
# We have to exit main process otherwise main process will hang.
_PythonMultiprocessing._terminate_processes(workers)
logger.critical("The subprocess of dataset may exit unexpected or be killed, "
"main process will exit. If this is not an artificial operation, you can use "
"ds.config.set_enable_watchdog(False) to block this error.")
os.kill(os.getpid(), signal.SIGTERM)
# release the workers
del workers
@staticmethod
def _terminate_processes(processes):
"""Terminate subprocesses"""
for p in processes:
try:
if p.exitcode is None:
p.terminate()
except Exception: # pylint: disable=broad-except
# process has been closed already
pass
for p in processes:
if p._closed is False: # pylint: disable=W0212
# We don't use w.join because join can only used in main process or join will raise an error.
p._popen.wait() # pylint: disable=W0212
# Monitor the exit number of subprocesses
@staticmethod
def _monitor_subprocess_exit(workers):
"""
To monitor whether process is exit.
Args:
workers (list of multiprocessing.Process): multiprocessing.Process.
Returns:
int, the timeout(in seconds) when process exit.
"""
for w in workers:
try:
exit_code = w.exitcode
if exit_code is not None:
# For kill -9, we can exit quickly
if exit_code == -9:
return 1
# For kill -15, we still exit after 30s
if exit_code == -15:
return 30
# In some cases the subprocess has been killed but the exitcode is still None.
# So we use os.kill(pid, 0) to check if it is alive.
subprocess_alive = _PythonMultiprocessing.is_process_alive(w.pid)
if not subprocess_alive:
# Like kill -15, we wait 30s before exit
return 30
except ValueError:
# process has been closed already
return 0
return 0
@staticmethod
def is_process_alive(pid):
"""
Check if the process is alive or not.
Note: We hit a deadlock when we use psutil or w.exitcode to check whether a process is alive.
Instead we use os.kill(ppid, 0).
Args:
pid: pid of the process to be checked
Returns:
True if the process is alive
"""
try:
os.kill(pid, 0)
except OSError:
return False
return True
# When main process exit, subprocesses will be terminate
@staticmethod
def _clean_process(ppid, workers, quit_signal):
"""
This is the execute function of clean process, if we found main process exited, we will clean subprocesses.
Args:
ppid: The process id of main process.
workers: The list of subprocesses.
quit_signal: The flag of quit.
"""
signal.signal(signal.SIGINT, signal.SIG_IGN)
while _PythonMultiprocessing.is_process_alive(ppid):
if quit_signal.is_set():
return
time.sleep(0.1)
_PythonMultiprocessing._terminate_processes(workers)
del workers
os.kill(os.getpid(), signal.SIGTERM)
def launch(self, op_id=-1):
"""
Launch Python multiprocessing pool.
Args:
pop_id: ID for operation to have Python multiprocessing pool launched
Returns:
Python multiprocssing pool is launched.
"""
self.python_threads_to_workers = {}
self.op_id = op_id
logger.info("Launching new Python Multiprocessing pool for Op:" + str(self.op_id))
if self.is_mp_enabled():
message = "Launching a new Python multiprocessing pool while a pool already exists!" + \
" The existing pool will be terminated first."
logger.warning(message)
self.terminate()
self.reset()
self.create_pool()
def create_pool(self):
"""
Returns:
"""
if get_enable_shared_mem():
_check_shm_usage(self.num_parallel_workers, 1, self.max_rowsize[0], self.max_rowsize[1])
if self.workers is not None:
raise Exception("Pool was already created, close it first.")
# Let gc collect unreferenced memory to avoid child processes in the pool to do it
gc.collect()
# Construct python worker processes
self.workers = []
self.warning_ctl = multiprocessing.Value('i', 0)
for worker_id in range(self.num_parallel_workers):
worker = _MPWorker(self.operations, self.warning_ctl, self.max_rowsize, worker_id)
worker.start()
self.workers.append(worker)
logger.info("Op: " + str(self.op_id) + " Python multiprocessing pool workers' PIDs: " + str(self.get_pids()))
self.hook = _PythonMultiprocessing._ExceptHookHandler()
# The op (Map, Batch, etc) multiprocessing will launch a watch dog thread for monitoring sub processes
self._launch_watch_dog()
atexit.register(self.terminate)
def terminate(self):
# close watch dog first and then close all the workers
self.abort_watchdog()
self.close_all_workers()
if hasattr(self, "warning_ctl"):
del self.warning_ctl
def get_pids(self):
"""
Get list of worker's PIDs
Returns:
list of strings
"""
if not self.is_mp_enabled():
return []
if not self.pids:
self.pids = []
if self.workers:
for w in self.workers:
try:
self.pids.append(w.pid)
except ValueError:
continue
return self.pids
def add_new_workers(self, num_new_workers):
logger.info(
"Increasing num_parallel_workers of Python Multiprocessing pool for Op:" + str(self.op_id) +
", old num_workers=" + str(self.num_parallel_workers) + " new num_workers=" + str(
self.num_parallel_workers +
num_new_workers) + ".")
self.terminate()
self.num_parallel_workers += num_new_workers
self.launch(self.op_id)
def remove_workers(self, num_removed_workers):
logger.info(
"Decreasing num_parallel_workers of Python Multiprocessing pool for Op:" + str(self.op_id) +
", old num_workers=" + str(self.num_parallel_workers) + " new num_workers=" + str(
self.num_parallel_workers -
num_removed_workers) + ".")
self.terminate()
self.num_parallel_workers -= num_removed_workers
self.launch(self.op_id)
def is_mp_enabled(self):
return self.workers is not None
def execute(self, idx, *args):
"""
Execute
"""
t_id = threading.get_ident()
# get the worker_id from Python layer cache first, get from Cpp layer if not found.
worker_id = self.python_threads_to_workers.setdefault(t_id, self.get_thread_to_worker())
if worker_id >= len(self.workers):
raise RuntimeError("[Internal] worker_id value is greater than number of available workers!")
# todo check_iterator_cleanup
if self.is_running() and check_iterator_cleanup() is False:
return self.workers[worker_id].execute(idx, *args)
return None
def _launch_watch_dog(self):
"""
We will launch a watchdog thread and a clean process to cleaning subprocess when there is process was killed.
The watchdog thread will cleanup subprocesses and main process when one of the subprocesses was killed.
The cleaning subprocess will cleanup subprocesses when main process was killed.
"""
if platform.system().lower() != 'windows':
self.eof = multiprocessing.Event()
self.cleaning_process = multiprocessing.Process(target=self._clean_process,
name="MapCleanProcess",
args=(self.ppid, self.workers, self.eof),
daemon=True)
self.cleaning_process.start()
if get_enable_watchdog():
self.eot = threading.Event()
self.watch_dog = threading.Thread(target=self._watch_dog,
name="MapWatchDog",
args=(self.eot, self.workers + [self.cleaning_process]),
daemon=True)
self.watch_dog.start()
def _abort_watchdog(self):
if not self.eot.is_set():
self.eot.set()
def abort_watchdog(self):
if hasattr(self, 'watch_dog') and self.watch_dog is not None and hasattr(self, 'eot') and self.eot is not None:
self._abort_watchdog()
if hasattr(self, 'cleaning_process') and self.cleaning_process is not None:
if hasattr(self, 'eof') and self.eof is not None and not self.eof.is_set():
self.eof.set()
_PythonMultiprocessing._terminate_processes([self.cleaning_process])
del self.cleaning_process
def is_running(self):
if hasattr(self, 'workers') and self.workers is not None:
return all([w.is_alive() for w in self.workers])
return False
def close_all_workers(self):
"""Close all the subprocess workers"""
if hasattr(self, 'workers') and self.workers is not None:
for w in self.workers:
w.close()
check_interval = get_multiprocessing_timeout_interval()
for w in self.workers:
try:
subprocess_file_descriptor = w.sentinel
st = time.time()
while _PythonMultiprocessing.is_process_alive(w.pid):
time.sleep(0.01) # sleep 10ms, waiting for the subprocess exit
if time.time() - st > check_interval:
logger.warning("Waiting for the subprocess worker [{}] to exit.".format(w.pid))
st += check_interval
except ValueError as e:
if "process object is closed" in str(e):
continue
raise e
try:
if w.is_alive():
os.close(subprocess_file_descriptor)
except OSError as e:
# Maybe the file descriptor had been released, so ignore the 'Bad file descriptor'
if "Bad file descriptor" not in str(e):
raise e
# use clear to release the handle which is better than self.workers = None
self.workers.clear()
self.workers = None
self.pids = None
class MapDataset(UnionBaseDataset):
"""
The result of applying the Map operation to the input Dataset.
Args:
input_dataset (Dataset): Input Dataset to be mapped.
operations (Union[list[TensorOperation], list[functions]]): A function mapping a nested structure of tensors
to another nested structure of tensor. Default: ``None``.
input_columns (Union[str, list[str]]): List of names of the input columns.
Default: ``None``, the operations will be applied on the first columns in the dataset.
The size of the list should match the number of inputs of the first operation.
output_columns (Union[str, list[str]], optional): List of names of the output columns.
The size of the list should match the number of outputs of the last operation.
Default: ``None``, output columns will be the input columns, i.e., the columns will
be replaced.
num_parallel_workers (int, optional): Number of workers to process the dataset
in parallel. Default: ``None``.
python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker process. This
option could be beneficial if the Python operation is computational heavy. Default: ``False``.
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
Default: ``None``, which means no cache is used.
callbacks (DSCallback, list[DSCallback], optional): List of Dataset callbacks to be called. Default: ``None``.
max_rowsize(Union[int, list[int]], optional): Maximum size of row in MB that is used for shared memory
allocation to copy data between processes, the total occupied shared memory will increase as
``num_parallel_workers`` and :func:`mindspore.dataset.config.set_prefetch_size` increase. If set to -1,
shared memory will be dynamically allocated with the actual size of data. This is only used if
``python_multiprocessing`` is set to True. If it is an int value, it represents ``input_columns`` and
``output_columns`` use this value as the unit to create shared memory. If it is a list, the first element
represents the ``input_columns`` use this value as the unit to create shared memory, and the second element
represents ``output_columns`` use this value as the unit to create shared memory. Default: 16.
offload (bool, optional): Flag to indicate whether offload is used. Default: ``None``.
"""
def __init__(self, input_dataset, operations=None, input_columns=None, output_columns=None,
num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None, max_rowsize=16,
offload=None):
super().__init__(children=input_dataset, num_parallel_workers=num_parallel_workers, cache=cache)
self.operations = to_list(operations)
for op in self.operations:
# user define c_vision.HWC2CHW without parentheses is error
if type(op) == type: # pylint: disable=unidiomatic-typecheck
raise ValueError("Parameter operations's element of method map should be a dataset processing "
"operation instance, but got: {}. It may be missing parentheses for "
"instantiation.".format(op))
if not isinstance(op, (c_transforms.TensorOperation, py_transforms.PyTensorOperation)) \
and not callable(op):
raise ValueError("Parameter operations's element of method map should be a python function or "
"class method which should be callable, but got: {}. It doesn't need parentheses "
"for python function or class method.".format(op))
self.input_columns = to_list(input_columns)
self.output_columns = to_list(output_columns)
# If output_columns were not provided then use input_columns
self.output_columns = self.input_columns if not self.output_columns else self.output_columns
self.python_multiprocessing = python_multiprocessing
self.process_pool = None
self.callbacks = to_list(callbacks)
if isinstance(max_rowsize, int):
self.max_rowsize = [max_rowsize] * 2
else:
self.max_rowsize = max_rowsize
self.offload = offload
def parse(self, children=None):
operations = self.__decompose_callable_operations()
count_old_transforms, count_new_transforms, count_non_data_vision_transforms = \
self.__count_transforms(operations)
count_pyfunc = self.__count_pyfuncs(operations)
if count_new_transforms + count_pyfunc == len(operations):
prev_op = None
for op in operations:
# skip user added DebugHook to avoid changing to Py-implementation.
if self.__is_debug_hook_op(op):
if prev_op:
# manually set previous_op_name
prev_op_name = self.__parse_op_name(prev_op)
op.set_previous_op_name(prev_op_name)
continue
if op.implementation is None:
if prev_op and prev_op.implementation == Implementation.PY:
op.implementation = Implementation.PY
else:
op.implementation = Implementation.C
prev_op = op
operations = self.__insert_debug_wrapper(operations)
operations = transforms.transforms.Compose.reduce(operations)
elif count_old_transforms + count_pyfunc + count_non_data_vision_transforms == len(operations):
operations = self.__insert_debug_wrapper(operations)
operations = transforms.py_transforms.Compose.reduce(operations)
else:
raise RuntimeError("Mixing old legacy c/py_transforms and new unified transforms is not allowed.")
self.operations = self.__process_final_operations(operations)
self.prepare_multiprocessing()
callbacks = [cb.create_runtime_obj() for cb in self.callbacks]
return cde.MapNode(children[0], self.operations, self.input_columns, self.output_columns,
callbacks, OffloadToManualOffloadMode.get(self.offload), self.process_pool)
def __deepcopy__(self, memodict):
return self.__safe_deepcopy__(memodict, exclude=("operations", "callbacks", "__transfer_dataset__"))
def __del__(self):
if hasattr(self, "process_pool") and self.process_pool is not None:
self.process_pool.terminate()
del self.process_pool
@staticmethod
def __parse_op_name(op):
"""
Utility method to get operation name.
"""
op_name = ""
if isinstance(op, transforms.py_transforms_util.FuncWrapper):
try:
op_name = op.transform.__name__
except (AttributeError,):
op_name = op.transform.__class__.__name__
else:
op_name = op.__class__.__name__
return op_name
@staticmethod
def __construct_debug_hook(previous_op_name=None, is_first_op=False):
"""
Wrap debug hook into FuncWrapper.
"""
inserted_functions = []
debug_hook_list = _get_debug_hook_list()
if debug_hook_list:
for fn in debug_hook_list:
# making deep copy to allow each debug hook instance hold unique variables
new_fn = copy.deepcopy(fn)
new_fn.set_previous_op_name(previous_op_name)
new_fn.set_is_first(is_first_op)
inserted_func = transforms.py_transforms_util.FuncWrapper(new_fn)
inserted_func.implementation = Implementation.PY
inserted_functions.append(inserted_func)
return inserted_functions
@staticmethod
def __is_debug_hook_op(op):
"""
Check if the op is user added DebugHook and skip it to avoid changing transforms implementation.
"""
if isinstance(op, DebugHook):
if not get_debug_mode():
raise ValueError("It is not allowed to inject DebugHook object in non-debug mode.")
return True
return False
@staticmethod
def __count_pyfuncs(operations):
"""
Count the number of pyfuncs operations
"""
return sum([1 if isinstance(op, FuncWrapper) else 0 for op in operations])
@staticmethod
def __count_transforms(operations):
"""
Count the various flavors of transforms operations
"""
# Count the number of old legacy data and vision c_transforms and py_transforms
count_old_transforms = sum(
[1 if "c_transforms" in str(op)
or isinstance(op, (c_transforms.TensorOperation, py_transforms.PyTensorOperation))
or ("py_transforms" in str(op) and not isinstance(op, FuncWrapper))
else 0 for op in operations])
# Count the number of new unified data and vision transforms
count_new_transforms = sum([1 if hasattr(op, "implementation") and not isinstance(op, FuncWrapper)
else 0 for op in operations])
# Count the number of non-data transforms and non-vision transforms
count_non_data_vision_transforms = sum(
[1 if "text.transforms" in str(op) or "audio.transforms" in str(op) else 0 for op in operations])
return count_old_transforms, count_new_transforms, count_non_data_vision_transforms
@staticmethod
def __operation_valid_for_multiprocessing(op):
if callable(op) and str(op).find("c_transform") < 0:
return True
return False
@staticmethod
def __process_final_operations(operations):
"""
Build final list of operations
"""
operations_fin = []
for op in operations:
if hasattr(op, "implementation"):
if op.implementation == Implementation.C and not isinstance(op, (FuncWrapper, ToNumpy)):
operations_fin.append(op.parse())
elif op.implementation == Implementation.PY:
operations_fin.append(op)
elif isinstance(op, (FuncWrapper, ToNumpy)):
operations_fin.append(op)
else:
raise RuntimeError("Wrong implementation")
else:
if op and getattr(op, 'parse', None):
operations_fin.append(op.parse())
else:
operations_fin.append(op)
return operations_fin
# Iterator bootstrap will be called on iterator construction.
# A deep copy of Dataset object is created prior of iterator_bootstrap.
# This method will create per iterator process pool and bind pyfunc execution to the pool.
def prepare_multiprocessing(self):
"""
Per iterator bootstrap callback.
"""
if self.python_multiprocessing and platform.system().lower() == 'windows':
logger.warning("Python multiprocessing is not supported on Windows platform.")
return
if self.python_multiprocessing and get_debug_mode():
logger.warning("Python multiprocessing is not supported in debug mode."
" Ignoring Python multiprocessing for map operation.")
return
if self.python_multiprocessing:
iter_specific_operations = []
callable_list = []
# If user didn't specify num_parallel_workers, set it to default
if self.num_parallel_workers is None:
self.num_parallel_workers = get_num_parallel_workers()
# Pass #1, look for Python callables and build list
for op in self.operations:
# our c transforms is now callable and should not be run in Python multithreading
if MapDataset.__operation_valid_for_multiprocessing(op):
callable_list.append(op)
if callable_list:
self.process_pool = _PythonMultiprocessing(str(self), self.num_parallel_workers, callable_list,
self.max_rowsize)
# Pass #2
idx = 0
for op in self.operations:
# our c transforms is now callable and should not be run in Python multithreading
if MapDataset.__operation_valid_for_multiprocessing(op):
# Wrap Python callable into _PythonCallable
iter_specific_operations.append(_PythonCallable(op, idx, self.process_pool))
idx += 1
else:
# CPP ops remain the same
iter_specific_operations.append(op)
self.operations = iter_specific_operations
def __insert_debug_wrapper(self, operations):
"""
Insert DebuggerWrapper before and after each op if debug mode is on.
"""
if not get_debug_mode():
return operations
first_op_name = self.__parse_op_name(operations[0])
inserted_operations = self.__construct_debug_hook(first_op_name, is_first_op=True)
for op in operations:
inserted_operations.append(op)
op_name = self.__parse_op_name(op)
inserted_operations.extend(self.__construct_debug_hook(op_name))
return inserted_operations
def __decompose_callable_operations(self):
"""
Decompose operations and build list of old legacy ops which are callable
"""
decomposed_operations = transforms.transforms.Compose.decompose(self.operations)
operations = []
for op in decomposed_operations:
if callable(op) and not hasattr(op, "implementation") and str(op).find(
"c_transform") < 0 and not isinstance(op, c_transforms.TensorOperation) and \
not isinstance(op, py_transforms.PyTensorOperation):
op = transforms.py_transforms_util.FuncWrapper(op)
operations.append(op)
return operations
class FilterDataset(UnionBaseDataset):
"""
The result of applying filter predicate to the input Dataset.
Args:
input_dataset (Dataset): Input Dataset to be mapped.
predicate (callable): Python callable which returns a boolean value. If False then filter the element.
input_columns (Union[str, list[str]], optional): List of names of the input columns.
Default: ``None``, the predicate will be applied to all columns in the dataset.
num_parallel_workers (int, optional): Number of workers to process the dataset
in parallel. Default: ``None``.
"""
def __init__(self, input_dataset, predicate, input_columns=None, num_parallel_workers=None):
super().__init__(children=input_dataset, num_parallel_workers=num_parallel_workers)
self.predicate = lambda *args: bool(predicate(*args))
self.input_columns = to_list(input_columns)
def parse(self, children=None):
return cde.FilterNode(children[0], self.predicate, self.input_columns)
class RepeatDataset(UnionBaseDataset):
"""
The result of applying Repeat operation to the input Dataset.
Args:
input_dataset (Dataset): Input Dataset to be repeated.
count (int): Number of times the dataset will be repeated. Default: -1, repeat indefinitely.
"""
def __init__(self, input_dataset, count):
super().__init__(children=input_dataset)
self.count = replace_none(count, -1)
def parse(self, children=None):
return cde.RepeatNode(children[0], self.count)
class SkipDataset(UnionBaseDataset):
"""
The result of applying Skip operation to the input Dataset.
Args:
input_dataset (Dataset): Input dataset to have elements skipped.
count (int): Number of elements to be skipped in the dataset.
"""
def __init__(self, input_dataset, count):
super().__init__(input_dataset)
self.count = count
def parse(self, children=None):
return cde.SkipNode(children[0], self.count)
class TakeDataset(UnionBaseDataset):
"""
The result of applying Take operation to the input Dataset.
Args:
input_dataset (Dataset): Input Dataset to have elements taken from.
count (int): Number of elements to be taken from the dataset.
"""
def __init__(self, input_dataset, count):
super().__init__(children=input_dataset)
self.count = count
def parse(self, children=None):
return cde.TakeNode(children[0], self.count)
class ZipDataset(UnionBaseDataset):
"""
The result of applying Zip operation to the input Dataset.
Args:
datasets (tuple): A tuple of datasets to be zipped together.
Raises:
TypeError: If dataset is not an instance of Dataset.
"""
def __init__(self, datasets):
super().__init__(children=datasets)
def parse(self, children=None):
return cde.ZipNode(children)
def is_sync(self):
return any([c.is_sync() for c in self.children])
class ConcatDataset(UnionBaseDataset):
"""
The result of applying Concat operation to the input Dataset.
Args:
datasets (list): A list of datasets to be concatenated together.
Raises:
TypeError: If dataset is not an instance of Dataset.
ValueError: If there is no samples in the one of the datasets.
"""
def __init__(self, datasets):
super().__init__(children=datasets)
for dataset in datasets:
if not isinstance(dataset, Dataset):
raise TypeError("Invalid dataset, expected Dataset object, but got %s!" % type(dataset))
self.datasets = datasets
self._sampler = samplers.SequentialSampler(num_samples=None)
self.children_sizes_ = [c.get_dataset_size() for c in self.children]
child_index = 0
for item in self.children_sizes_:
if item == 0:
raise ValueError("There are no samples in the dataset number %d. Please make sure there are "
"valid samples in the dataset." % child_index)
child_index += 1
self._children_sizes = self.children_sizes_.copy()
# _children_flag_and_nums: A list of pair<int ,int>.The first element of pair is flag that characterizes
# whether the dataset is mappable. The second element of pair is length of the dataset
self._children_flag_and_nums = []
# _children_start_end_index_: A list of pair<int ,int>.The elements of pair are used to characterize
# the valid position of the dataset corresponding to the subscript when sampling
self._children_start_end_index_ = []
for index, child in enumerate(self.children):
tem_list = [-1, -1]
self._children_start_end_index_.append(tem_list)
dataset_len = self.children_sizes_[index]
from mindspore.dataset.engine.datasets_user_defined import GeneratorDataset
if isinstance(child, GeneratorDataset) and not hasattr(child.source, "__getitem__"):
dataset_len = 0
self.children_sizes_[index] = 0
if isinstance(child, MappableDataset):
self._children_flag_and_nums.append((0, dataset_len))
else:
self._children_flag_and_nums.append((1, dataset_len))
def parse(self, children=None):
return cde.ConcatNode(children, self._sampler, self._children_flag_and_nums, self._children_start_end_index_,
self._children_sizes)
def use_sampler(self, sampler):
"""
Set the distributedSampler to concat dataset
Args:
sampler (Sampler): The sampler to use for the current dataset.
Currently supported: DistributedSampler.
Raises:
TypeError: If the sampler is not an instance of DistributedSampler
ValueError: If the parameter shuffle of sampler is True
ValueError: If the parameter NumSamples of sampler is not None.
ValueError: If num_shards <=0.
"""
if not isinstance(sampler, (samplers.DistributedSampler, samplers.RandomSampler)):
raise TypeError("The parameter %s of concat must be DistributedSampler or RandomSampler!" % sampler)
if isinstance(sampler, samplers.RandomSampler):
if sampler.replacement:
raise ValueError("The parameter replacement of RandomSampler must be False!")
if sampler.get_num_samples() is not None:
raise ValueError("The parameter num_samples of RandomSampler is not support to be set!")
self._sampler = sampler
self._children_sizes = [c.get_dataset_size() for c in self.children]
# Recursive access to other child concat nodes
def set_child(node):
for c in node.children:
if isinstance(c, ConcatDataset):
c.use_sampler(sampler)
set_child(c)
set_child(self)
return
if sampler.is_shuffled():
raise ValueError("The parameter shuffle of DistributedSampler must be False!")
if sampler.num_shards <= 0:
raise ValueError("The parameter num_shards of DistributedSampler must be positive int!")
if sampler.get_num_samples() is not None:
raise ValueError("The parameter num_samples of DistributedSampler is not support to be set!")
self.dataset_size = None
self._sampler = sampler
cumulative_samples_nums = 0
for index, child in enumerate(self.children):
if hasattr(child, 'sampler') and child.sampler.get_num_samples() is not None:
raise ValueError("The parameter NumSamples of %s is not support to be set!" % child)
if isinstance(child, (BatchDataset, PaddedBatchDataset)):
raise TypeError("The parameter %s of concat must not be BatchDataset or PaddedBatchDataset!" % child)
# if child is mappable and the length is greater than 0
if not self._children_flag_and_nums[index][0] and self._children_flag_and_nums[index][1]:
tem_value = cumulative_samples_nums + self._children_flag_and_nums[index][1]
if not self._children_flag_and_nums[index][1] >= sampler.num_shards:
if tem_value < sampler.num_shards:
self._children_start_end_index_[index][0] = cumulative_samples_nums
self._children_start_end_index_[index][1] = tem_value
else:
self._children_start_end_index_[index][0] = cumulative_samples_nums
self._children_start_end_index_[index][1] = tem_value % sampler.num_shards
tem_sampler = copy.deepcopy(sampler)
tem_sampler.set_offset(cumulative_samples_nums)
child.use_sampler(tem_sampler)
cumulative_samples_nums += self.children_sizes_[index]
cumulative_samples_nums %= sampler.num_shards
class RenameDataset(UnionBaseDataset):
"""
The result of applying Rename operation to the input Dataset.
Args:
input_dataset (Dataset): Input Dataset to be Renamed.
input_columns (Union[str, list[str]]): List of names of the input columns.
output_columns (Union[str, list[str]]): List of names of the output columns.
"""
def __init__(self, input_dataset, input_columns, output_columns):
super().__init__(children=input_dataset)
self.input_column_names = to_list(input_columns)
self.output_column_names = to_list(output_columns)
def parse(self, children=None):
return cde.RenameNode(children[0], self.input_column_names, self.output_column_names)
def to_list(items):
if items is None:
return []
if isinstance(items, tuple):
return list(items)
if not isinstance(items, list):
return [items]
return items
class ProjectDataset(UnionBaseDataset):
"""
The result of applying Project operation to the input Dataset.
Args:
input_dataset (Dataset): Input Dataset to be Projected.
columns (Union[str, list[str]]): List of names of the columns to project.
"""
def __init__(self, input_dataset, columns):
super().__init__(children=input_dataset)
self.columns = to_list(columns)
def parse(self, children=None):
return cde.ProjectNode(children[0], self.columns)
class _ToDevice:
"""
Internal class to handle sending data to device.
"""
def __init__(self, dataset, num_epochs):
if get_debug_mode():
logger.error("MindData debugger cannot be used in dataset sink mode. Please manually turn off "
"sink mode and try debugger again.")
ir_tree, self.api_tree = dataset.create_ir_tree()
self._runtime_context = cde.PythonRuntimeContext()
self._runtime_context.Init()
self._to_device = cde.ToDevice(num_epochs)
if dataset.get_init_step() != 0:
init_step = dataset.get_init_step()
dataset_size = dataset.get_dataset_size()
self._to_device.Init(ir_tree, init_step, dataset_size)
else:
self._to_device.Init(ir_tree, 0, -1)
self._runtime_context.AssignConsumer(self._to_device)
ITERATORS_LIST.append(weakref.ref(self))
_unset_iterator_cleanup()
def send(self):
self._to_device.Send()
def stop_send(self):
"""
send stop send signal to pipeline, it is used when end of sequence is sent at the epoch end.
"""
self._to_device.StopSend()
def continue_send(self):
"""
send continue send signal to pipeline, it is used when end of sequence is sent at the epoch end.
"""
self._to_device.ContinueSend()
def get_data_info(self):
"""
Get type and shape of current batch.
"""
return self._to_device.GetDataInfo()
def get_mbuf_queue_size(self):
"""
Get element numbers inside mbuf.
"""
return self._to_device.GetMbufQueueSize()
def get_send_info(self):
"""
In sink mode, it returns the send information of dataset at this moment.
Send information includes number of send batches, time summary of fetching data on host
and time summary of sending data.
"""
return self._to_device.GetSendInfo()
def release(self):
"""
Manually terminate Device Queue instead of relying on out of scope destruction.
"""
if hasattr(self, '_runtime_context') and self._runtime_context:
if hasattr(self, '_to_device') and self._to_device:
self._runtime_context.Terminate()
del self._to_device
del self._runtime_context
def __deepcopy__(self, memodict):
return self
def get_offload_model(self, col_names):
"""
Get offload model containing removed offload ops from pipeline.
"""
offload_model = GetOffloadModel(self._to_device, col_names)
return offload_model
def _reset(self, step, dataset_size):
self._to_device.Reset(step, dataset_size)
class TransferDataset(Dataset):
"""
The result of applying TDT operation to the input Dataset.
Args:
input_dataset (Dataset): Input Dataset to be transferred.
send_epoch_end (bool, optional): Whether to send end of sequence to device or not. Default: ``True``.
create_data_info_queue (bool, optional): Whether to create queue which stores
types and shapes of data or not. Default: ``False``.
Raises:
TypeError: If device_type is empty.
ValueError: If device_type is not 'Ascend', 'GPU' or 'CPU'.
RuntimeError: If dataset is unknown.
"""
def __init__(self, input_dataset, send_epoch_end=True, create_data_info_queue=False, queue_name=""):
super().__init__(children=input_dataset)
if queue_name == "":
self.queue_name = str(uuid.uuid1())
logger.info(f"queue_name is newly generated. value is {self.queue_name}")
else:
self.queue_name = queue_name
logger.info(f"queue_name is read from compile cache. value is {self.queue_name}")
self.device_type = context.get_context("device_target") if context else "CPU"
self.device_id = context.get_context("device_id") if context else 0
self._send_epoch_end = replace_none(send_epoch_end, True)
self._create_data_info_queue = create_data_info_queue
self._to_device = None
self.column_name = input_dataset.get_col_names()
def parse(self, children=None):
total_batch = 0
if hasattr(self.children[0], "__total_batch__"):
total_batch = self.children[0].__total_batch__
check_total_batch(total_batch)
return cde.DataQueueNode(children[0], self.queue_name, self.device_type, self.device_id, self._send_epoch_end,
total_batch, self._create_data_info_queue)
def create_dict_iterator(self, num_epochs=-1, output_numpy=False):
raise RuntimeError("TransferDataset is not iterable.")
def create_tuple_iterator(self, columns=None, num_epochs=-1, output_numpy=False, do_copy=True):
raise RuntimeError("TransferDataset is not iterable.")
def __iter__(self):
raise RuntimeError("TransferDataset is not iterable.")
def output_shapes(self):
raise RuntimeError("TransferDataset does not support obtaining output_shapes.")
def output_types(self):
raise RuntimeError("TransferDataset does not support obtaining output_types.")
@check_to_device_send
def send(self, num_epochs=-1):
"""
Send to device
"""
if Dataset._noop_mode():
return
if self._to_device is not None:
del self._to_device
self._to_device = _ToDevice(self, num_epochs)
self._to_device.send()
def stop_send(self):
if self._to_device is not None:
self._to_device.stop_send()
def continue_send(self):
if self._to_device is not None:
self._to_device.continue_send()
def get_data_info(self):
"""
Get type and shape of current batch
"""
if self._to_device is not None:
return self._to_device.get_data_info()
raise RuntimeError("Calling get_data_info with bad state.")
def get_mbuf_queue_size(self):
"""
Get element numbers inside mbuf.
"""
if self._to_device is not None:
return self._to_device.get_mbuf_queue_size()
raise RuntimeError("Device queue is not init, call get_mbuf_queue_size failed.")
def get_send_info(self):
"""
In sink mode, it returns the send information of dataset at this moment.
Send information includes number of send batches, time summary of fetching data on host
and time summary of sending data.
"""
if self._to_device is not None:
return self._to_device.get_send_info()
raise RuntimeError("Calling get_send_info with bad state, data queue is not initialized.")
def get_offload_model(self):
if self._to_device is not None:
return self._to_device.get_offload_model(self.column_name)
raise RuntimeError("get_offload_model, _to_device is None")
def release(self):
"""
Manually terminate Device Queue instead of relying on out of scope destruction.
"""
if self._to_device is not None:
self._to_device.release()
def _reset(self, step, dataset_size):
if self._to_device is not None:
logger.info("Reset the dataset pipeline to step: " + str(step) + ", epoch: " + str(step // dataset_size))
self._to_device._reset(step, dataset_size) # pylint: disable=protected-access
[docs]class Schema:
"""
Class to represent a schema of a dataset.
Args:
schema_file (str): Path of the schema file. Default: ``None``.
Raises:
RuntimeError: If schema file failed to load.
Examples:
>>> import mindspore.dataset as ds
>>> from mindspore import dtype as mstype
>>>
>>> # Create schema; specify column name, mindspore.dtype and shape of the column
>>> schema = ds.Schema()
>>> schema.add_column(name='col1', de_type=mstype.int64, shape=[2])
"""
@check_schema
def __init__(self, schema_file=None):
self.schema_file = replace_none(schema_file, "")
self.cpp_schema = cde.SchemaObj(self.schema_file)
[docs] @check_add_column
def add_column(self, name, de_type, shape=None):
"""
Add new column to the schema.
Args:
name (str): The new name of the column.
de_type (str): Data type of the column.
shape (list[int], optional): Shape of the column.
Default: ``None``, [-1] which is an unknown shape of rank 1.
Raises:
ValueError: If column type is unknown.
Examples:
>>> import mindspore.dataset as ds
>>> from mindspore import dtype as mstype
>>>
>>> schema = ds.Schema()
>>> schema.add_column('col_1d', de_type=mstype.int64, shape=[2])
"""
if isinstance(de_type, typing.Type):
de_type = mstype_to_detype(de_type)
col_type = str(de_type)
else:
col_type = str(cde.DataType(de_type))
if shape is None:
self.cpp_schema.add_column(name, col_type)
else:
self.cpp_schema.add_column(name, col_type, shape)
[docs] def parse_columns(self, columns):
"""
Parse the columns and add it to self.
Args:
columns (Union[dict, list[dict], tuple[dict]]): Dataset attribute information, decoded from schema file.
- list[dict], `name` and `type` must be in keys, `shape` optional.
- dict, columns.keys() as name, columns.values() is dict, and `type` inside, `shape` optional.
Raises:
RuntimeError: If failed to parse columns.
RuntimeError: If column's name field is missing.
RuntimeError: If column's type field is missing.
Examples:
>>> from mindspore.dataset import Schema
>>> schema = Schema()
>>> columns1 = [{'name': 'image', 'type': 'int8', 'shape': [3, 3]},
... {'name': 'label', 'type': 'int8', 'shape': [1]}]
>>> schema.parse_columns(columns1)
>>> columns2 = {'image': {'shape': [3, 3], 'type': 'int8'}, 'label': {'shape': [1], 'type': 'int8'}}
>>> schema.parse_columns(columns2)
"""
self.cpp_schema.parse_columns(json.dumps(columns, indent=2))
[docs] def to_json(self):
"""
Get a JSON string of the schema.
Returns:
str, JSON string of the schema.
Examples:
>>> from mindspore.dataset import Schema
>>> from mindspore import dtype as mstype
>>>
>>> schema = Schema()
>>> schema.add_column('col_1d', de_type=mstype.int64, shape=[2])
>>> json = schema.to_json()
"""
return self.cpp_schema.to_json()
[docs] def from_json(self, json_obj):
"""
Get schema file from JSON object.
Args:
json_obj(dictionary): Object of JSON parsed.
Raises:
RuntimeError: if there is unknown item in the object.
RuntimeError: if dataset type is missing in the object.
RuntimeError: if columns are missing in the object.
Examples:
>>> import json
>>> from mindspore.dataset import Schema
>>>
>>> with open("/path/to/schema_file", "r") as file:
... json_obj = json.load(file)
... schema = Schema()
... schema.from_json(json_obj)
"""
self.cpp_schema.from_string(json.dumps(json_obj, indent=2))
def __str__(self):
return self.to_json()
@staticmethod
def get_num_rows(schema):
schema_obj = schema
if not isinstance(schema_obj, Schema):
schema_obj = Schema(schema_obj)
return schema_obj.cpp_schema.get_num_rows()
class DeserializedDataset(Dataset):
def __init__(self, input_obj):
super().__init__()
self.input_obj = input_obj
def parse(self, children=None):
if isinstance(self.input_obj, dict):
json_str = json.dumps(self.input_obj)
return cde.Dataset.from_json_string(json_str)
return cde.Dataset.from_json_file(self.input_obj)