# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Inversion Attack
"""
import numpy as np
from scipy.special import softmax
from mindspore.nn import Cell, MSELoss
from mindspore import Tensor
from mindspore.ops import operations as P
from mindspore import context
from mindarmour.utils.util import GradWrapWithLoss
from mindarmour.utils._check_param import check_param_type, check_param_multi_types, \
check_int_positive, check_numpy_param, check_value_positive, check_equal_shape
from mindarmour.utils.logger import LogUtil
from mindarmour.utils.util import calculate_lp_distance, compute_ssim
LOGGER = LogUtil.get_instance()
LOGGER.set_level('INFO')
TAG = 'Image inversion attack'
class InversionLoss(Cell):
"""
The loss function for inversion attack.
Args:
network (Cell): The network used to infer images' deep representations.
weights (Union[list, tuple]): Weights of three sub-loss in InversionLoss, which can be adjusted to
obtain better results.
"""
def __init__(self, network, weights):
super(InversionLoss, self).__init__()
self._network = check_param_type('network', network, Cell)
self._mse_loss = MSELoss()
self._weights = check_param_multi_types('weights', weights, [list, tuple])
self._get_shape = P.Shape()
self._zeros = P.ZerosLike()
self._device_target = context.get_context("device_target")
def construct(self, input_data, target_features):
"""
Calculate the inversion attack loss, which consists of three parts. Loss_1 is for evaluating the difference
between the target deep representations and current representations; Loss_2 is for evaluating the continuity
between adjacent pixels; Loss_3 is for regularization.
Args:
input_data (Tensor): The reconstructed image during inversion attack.
target_features (Tensor): Deep representations of the original image.
Returns:
Tensor, inversion attack loss of the current iteration.
"""
output = self._network(input_data)
loss_1 = self._mse_loss(output, target_features) / self._mse_loss(target_features, self._zeros(target_features))
data_shape = self._get_shape(input_data)
if self._device_target == 'CPU':
split_op_1 = P.Split(2, data_shape[2])
split_op_2 = P.Split(3, data_shape[3])
data_split_1 = split_op_1(input_data)
data_split_2 = split_op_2(input_data)
loss_2 = 0
for i in range(1, data_shape[2]):
loss_2 += self._mse_loss(data_split_1[i], data_split_1[i - 1])
for j in range(1, data_shape[3]):
loss_2 += self._mse_loss(data_split_2[j], data_split_2[j - 1])
else:
data_copy_1 = self._zeros(input_data)
data_copy_2 = self._zeros(input_data)
data_copy_1[:, :, :(data_shape[2] - 1), :] = input_data[:, :, 1:, :]
data_copy_2[:, :, :, :(data_shape[2] - 1)] = input_data[:, :, :, 1:]
loss_2 = self._mse_loss(input_data, data_copy_1) + self._mse_loss(input_data, data_copy_2)
loss_3 = self._mse_loss(input_data, self._zeros(input_data))
loss = loss_1*self._weights[0] + loss_2*self._weights[1] + loss_3*self._weights[2]
return loss
[docs]class ImageInversionAttack:
"""
An attack method used to reconstruct images by inverting their deep representations.
References: `Aravindh Mahendran, Andrea Vedaldi. Understanding Deep Image Representations by Inverting Them.
2014. <https://arxiv.org/pdf/1412.0035.pdf>`_
Args:
network (Cell): The network used to infer images' deep representations.
input_shape (tuple): Data shape of single network input, which should be in accordance with the given
network. The format of shape should be (channel, image_width, image_height).
input_bound (Union[tuple, list]): The pixel range of original images, which should be like [minimum_pixel,
maximum_pixel] or (minimum_pixel, maximum_pixel).
loss_weights (Union[list, tuple]): Weights of three sub-loss in InversionLoss, which can be adjusted to
obtain better results. Default: (1, 0.2, 5).
Raises:
TypeError: If the type of network is not Cell.
ValueError: If any value of input_shape is not positive int.
ValueError: If any value of loss_weights is not positive value.
Examples:
>>> import mindspore.ops.operations as P
>>> from mindspore.nn import Cell
>>> from mindarmour.privacy.evaluation.inversion_attack import ImageInversionAttack
>>> class Net(Cell):
... def __init__(self):
... super(Net, self).__init__()
... self._softmax = P.Softmax()
... self._reduce = P.ReduceSum()
... self._squeeze = P.Squeeze(1)
... def construct(self, inputs):
... out = self._softmax(inputs)
... out = self._reduce(out, 2)
... return self._squeeze(out)
>>> net = Net()
>>> original_images = np.random.random((2,1,10,10)).astype(np.float32)
>>> target_features = np.random.random((2,10)).astype(np.float32)
>>> inversion_attack = ImageInversionAttack(net,
... input_shape=(1, 10, 10),
... input_bound=(0, 1),
... loss_weights=[1, 0.2, 5])
>>> inversion_images = inversion_attack.generate(target_features, iters=10)
>>> evaluate_result = inversion_attack.evaluate(original_images, inversion_images)
"""
def __init__(self, network, input_shape, input_bound, loss_weights=(1, 0.2, 5)):
self._network = check_param_type('network', network, Cell)
for sub_loss_weight in loss_weights:
check_value_positive('sub_loss_weight', sub_loss_weight)
self._loss = InversionLoss(self._network, loss_weights)
self._input_shape = check_param_type('input_shape', input_shape, tuple)
for shape_dim in input_shape:
check_int_positive('shape_dim', shape_dim)
self._input_bound = check_param_multi_types('input_bound', input_bound, [list, tuple])
for value_bound in self._input_bound:
check_param_multi_types('value_bound', value_bound, [float, int])
if self._input_bound[0] > self._input_bound[1]:
msg = 'input_bound[0] should not be larger than input_bound[1], but got them as {} and {}'.format(
self._input_bound[0], self._input_bound[1])
raise ValueError(msg)
[docs] def generate(self, target_features, iters=100):
"""
Reconstruct images based on target_features.
Args:
target_features (numpy.ndarray): Deep representations of original images. The first dimension of
target_features should be img_num. It should be noted that the shape of target_features should be
(1, dim2, dim3, ...) if img_num equals 1.
iters (int): iteration times of inversion attack, which should be positive integers. Default: 100.
Returns:
numpy.ndarray, reconstructed images, which are expected to be similar to original images.
Raises:
TypeError: If the type of target_features is not numpy.ndarray.
ValueError: If any value of iters is not positive int.Z
"""
target_features = check_numpy_param('target_features', target_features)
iters = check_int_positive('iters', iters)
# shape checking
img_num = target_features.shape[0]
test_input = np.random.random((img_num,) + self._input_shape).astype(np.float32)
test_out = self._network(Tensor(test_input)).asnumpy()
if test_out.shape != target_features.shape:
msg = "The shape of target_features ({}) is not in accordance with the shape" \
" of network output({})".format(target_features.shape, test_out.shape)
raise ValueError(msg)
loss_net = self._loss
loss_grad = GradWrapWithLoss(loss_net)
inversion_images = []
for i in range(img_num):
target_feature_n = target_features[i]
inversion_image_n = np.random.random((1,) + self._input_shape).astype(np.float32)*0.05
for s in range(iters):
x_grad = loss_grad(Tensor(inversion_image_n), Tensor(target_feature_n)).asnumpy()
x_grad_sign = np.sign(x_grad)
inversion_image_n -= x_grad_sign*0.01
inversion_image_n = np.clip(inversion_image_n, self._input_bound[0], self._input_bound[1])
current_loss = loss_net(Tensor(inversion_image_n), Tensor(target_feature_n))
LOGGER.info(TAG, 'iteration step: {}, loss is {}'.format(s, current_loss))
inversion_images.append(inversion_image_n)
return np.concatenate(np.array(inversion_images))
[docs] def evaluate(self, original_images, inversion_images, labels=None, new_network=None):
"""
Evaluate the quality of inverted images by three index: the average L2 distance and SSIM value between
original images and inversion images, and the average of inverted images' confidence on true labels of inverted
inferred by a new trained network.
Args:
original_images (numpy.ndarray): Original images, whose shape should be (img_num, channels, img_width,
img_height).
inversion_images (numpy.ndarray): Inversion images, whose shape should be (img_num, channels, img_width,
img_height).
labels (numpy.ndarray): Ground truth labels of original images. Default: None.
new_network (Cell): A network whose structure contains all parts of self._network, but loaded with different
checkpoint file. Default: None.
Returns:
- float, l2 distance.
- float, average ssim value.
- Union[float, None], average confidence. It would be None if labels or new_network is None.
"""
check_numpy_param('original_images', original_images)
check_numpy_param('inversion_images', inversion_images)
if labels is not None:
check_numpy_param('labels', labels)
true_labels = np.squeeze(labels)
if len(true_labels.shape) > 1:
msg = 'Shape of true_labels should be (1, n) or (n,), but got {}'.format(true_labels.shape)
raise ValueError(msg)
if true_labels.size != original_images.shape[0]:
msg = 'The size of true_labels should equal the number of images, but got {} and {}'.format(
true_labels.size, original_images.shape[0])
raise ValueError(msg)
if new_network is not None:
check_param_type('new_network', new_network, Cell)
LOGGER.info(TAG, 'Please make sure that the network you pass is loaded with different checkpoint files '
'compared with that of self._network.')
img_1, img_2 = check_equal_shape('original_images', original_images, 'inversion_images', inversion_images)
if (len(img_1.shape) != 4) or (img_1.shape[1] != 1 and img_1.shape[1] != 3):
msg = 'The shape format of img_1 and img_2 should be (img_num, channels, img_width, img_height),' \
' but got {} and {}'.format(img_1.shape, img_2.shape)
raise ValueError(msg)
total_l2_distance = 0
total_ssim = 0
img_1 = img_1.transpose(0, 2, 3, 1)
img_2 = img_2.transpose(0, 2, 3, 1)
for i in range(img_1.shape[0]):
_, l2_dis, _ = calculate_lp_distance(img_1[i], img_2[i])
total_l2_distance += l2_dis
total_ssim += compute_ssim(img_1[i], img_2[i])
avg_l2_dis = total_l2_distance / img_1.shape[0]
avg_ssim = total_ssim / img_1.shape[0]
avg_confi = None
if (new_network is not None) and (labels is not None):
pred_logits = new_network(Tensor(inversion_images.astype(np.float32))).asnumpy()
logits_softmax = softmax(pred_logits, axis=1)
avg_confi = np.mean(logits_softmax[np.arange(img_1.shape[0]), true_labels])
return avg_l2_dis, avg_ssim, avg_confi