import fnmatch import math import os import sys import time from operator import itemgetter import gc import numpy as np import torch import torch.optim as optim import torch.nn as nn import torch.nn.functional as F from PIL import Image from torch.utils.data import Dataset from torchvision import transforms # from darknet import Darknet from median_pool import MedianPool2d # print('starting test read') # im = Image.open('data/horse.jpg').convert('RGB') # print('img read!') class MaxProbExtractor(nn.Module): """MaxProbExtractor: extracts max class probability for class from YOLO output. Module providing the functionality necessary to extract the max class probability for one class from YOLO output. """ def __init__(self, cls_id, num_cls, config): super(MaxProbExtractor, self).__init__() self.cls_id = cls_id self.num_cls = num_cls self.config = config self.anchor_num = 3 def forward(self, YOLOoutput): # get values neccesary for transformation if YOLOoutput.dim() == 3: YOLOoutput = YOLOoutput.unsqueeze(0) batch = YOLOoutput.size(0) assert (YOLOoutput.size(1) == (5 + self.num_cls) * self.anchor_num) h = YOLOoutput.size(2) w = YOLOoutput.size(3) # transform the output tensor from [batch, 425, 19, 19] to [batch, 80, 1805] output = YOLOoutput.view(batch, self.anchor_num, 5 + self.num_cls, h * w) # [batch, 5, 85, 361] output = output.transpose(1, 2).contiguous() # [batch, 85, 5, 361] output = output.view(batch, 5 + self.num_cls, self.anchor_num * h * w) # [batch, 85, 1805] output_objectness = torch.sigmoid(output[:, 4, :]) # [batch, 1805] # 是否有物体 output = output[:, 5:5 + self.num_cls, :] # [batch, 80, 1805] # perform softmax to normalize probabilities for object classes to [0,1] normal_confs = torch.nn.Softmax(dim=1)(output) # 物体类别 # we only care for probabilities of the class of interest (person) confs_for_class = normal_confs[:, self.cls_id, :] # 类别 序号对应的为人 confs_if_object = output_objectness # confs_for_class * output_objectness confs_if_object = confs_for_class * output_objectness confs_if_object = self.config.loss_target(output_objectness, confs_for_class) # find the max probability for person max_conf, max_conf_idx = torch.max(confs_if_object, dim=1) return max_conf class NPSCalculator(nn.Module): """NMSCalculator: calculates the non-printability score of a patch. Module providing the functionality necessary to calculate the non-printability score (NMS) of an adversarial patch. """ def __init__(self, printability_file, patch_side): super(NPSCalculator, self).__init__() self.printability_array = nn.Parameter(self.get_printability_array(printability_file, patch_side), requires_grad=False) def forward(self, adv_patch): # calculate euclidian distance between colors in patch and colors in printability_array # square root of sum of squared difference color_dist = (adv_patch - self.printability_array + 0.000001) color_dist = color_dist ** 2 color_dist = torch.sum(color_dist, 1) + 0.000001 color_dist = torch.sqrt(color_dist) # only work with the min distance color_dist_prod = torch.min(color_dist, 0)[0] # test: change prod for min (find distance to closest color) # calculate the nps by summing over all pixels nps_score = torch.sum(color_dist_prod, 0) nps_score = torch.sum(nps_score, 0) return nps_score / torch.numel(adv_patch) def get_printability_array(self, printability_file, side): printability_list = [] # read in printability triplets and put them in a list with open(printability_file) as f: for line in f: printability_list.append(line.split(",")) printability_array = [] for printability_triplet in printability_list: printability_imgs = [] red, green, blue = printability_triplet printability_imgs.append(np.full((side, side), red)) printability_imgs.append(np.full((side, side), green)) printability_imgs.append(np.full((side, side), blue)) printability_array.append(printability_imgs) printability_array = np.asarray(printability_array) printability_array = np.float32(printability_array) pa = torch.from_numpy(printability_array) return pa class TotalVariation(nn.Module): """TotalVariation: calculates the total variation of a patch. Module providing the functionality necessary to calculate the total Variation (TV) of an adversarial patch. TotalVariation：计算补丁的总变化。 该模块提供了计算对抗性补丁的总变化 (TV) 所需的功能。 """ def __init__(self): super(TotalVariation, self).__init__() def forward(self, adv_patch): # bereken de total variation van de adv_patch tvcomp1 = torch.sum(torch.abs(adv_patch[:, :, 1:] - adv_patch[:, :, :-1] + 0.000001), 0) tvcomp1 = torch.sum(torch.sum(tvcomp1, 0), 0) tvcomp2 = torch.sum(torch.abs(adv_patch[:, 1:, :] - adv_patch[:, :-1, :] + 0.000001), 0) tvcomp2 = torch.sum(torch.sum(tvcomp2, 0), 0) tv = tvcomp1 + tvcomp2 return tv / torch.numel(adv_patch) class PatchTransformer(nn.Module): """PatchTransformer: transforms batch of patches Module providing the functionality necessary to transform a batch of patches, randomly adjusting brightness and contrast, adding random amount of noise, and rotating randomly. Resizes-patches according to as size based on the batch of labels, and pads them to the dimension of an image. 变换一批补丁，随机调整亮度和对比度，添加随机数量的噪声，随机旋转。 根据标签批次的大小调整补丁大小，并将它们填充到图像的尺寸中。 """ def __init__(self): super(PatchTransformer, self).__init__() self.min_contrast = 0.8 self.max_contrast = 1.2 self.min_brightness = -0.1 self.max_brightness = 0.1 self.noise_factor = 0.10 self.minangle = -20 / 180 * math.pi self.maxangle = 20 / 180 * math.pi self.medianpooler = MedianPool2d(7, same=True) # 中值池化 ''' kernel = torch.cuda.FloatTensor([[0.003765, 0.015019, 0.023792, 0.015019, 0.003765], [0.015019, 0.059912, 0.094907, 0.059912, 0.015019], [0.023792, 0.094907, 0.150342, 0.094907, 0.023792], [0.015019, 0.059912, 0.094907, 0.059912, 0.015019], [0.003765, 0.015019, 0.023792, 0.015019, 0.003765]]) self.kernel = kernel.unsqueeze(0).unsqueeze(0).expand(3,3,-1,-1) ''' def forward(self, adv_patch, lab_batch, img_size, do_rotate=True, rand_loc=True): # adv_patch = F.conv2d(adv_patch.unsqueeze(0),self.kernel,padding=(2,2)) adv_patch = self.medianpooler(adv_patch.unsqueeze(0)) # Determine size of padding pad = (img_size - adv_patch.size(-1)) / 2 # Make a batch of patches adv_patch = adv_patch.unsqueeze(0) # .unsqueeze(0) # 这里又扩大一维，变成5维 1, 1, 3, 300, 300 adv_batch = adv_patch.expand(lab_batch.size(0), lab_batch.size(1), -1, -1, -1) # adv_batch ！！ 不是adv_patch！！ 8, 14, 3, 300, 300 batch_size = torch.Size((lab_batch.size(0), lab_batch.size(1))) # 8, 14 # Contrast, brightness and noise transforms # Create random contrast tensor contrast = torch.cuda.FloatTensor(batch_size).uniform_(self.min_contrast, self.max_contrast) contrast = contrast.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1) contrast = contrast.expand(-1, -1, adv_batch.size(-3), adv_batch.size(-2), adv_batch.size(-1)) contrast = contrast.cuda() # Create random brightness tensor brightness = torch.cuda.FloatTensor(batch_size).uniform_(self.min_brightness, self.max_brightness) brightness = brightness.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1) brightness = brightness.expand(-1, -1, adv_batch.size(-3), adv_batch.size(-2), adv_batch.size(-1)) brightness = brightness.cuda() # Create random noise tensor noise = torch.cuda.FloatTensor(adv_batch.size()).uniform_(-1, 1) * self.noise_factor # Apply contrast/brightness/noise, clamp adv_batch = adv_batch * contrast + brightness + noise adv_batch = torch.clamp(adv_batch, 0.000001, 0.99999) # 限制到0到1之间 # Where the label class_id is 1 we don't want a patch (padding) --> fill mask with zero's cls_ids = torch.narrow(lab_batch, 2, 0, 1) # torch.narrow(input,dim,start,length) 从dim开始，返回共享内存的数据start到start+length-1 cls_mask = cls_ids.expand(-1, -1, 3) # 接上，这里取出 lab_batch的代表id那列，相当于现在的lab_batch[..., 0] cls_mask = cls_mask.unsqueeze(-1) cls_mask = cls_mask.expand(-1, -1, -1, adv_batch.size(3)) cls_mask = cls_mask.unsqueeze(-1) cls_mask = cls_mask.expand(-1, -1, -1, -1, adv_batch.size(4)) # cls_mask 的大小是 8, 14, 3, 300, 300 数据是类别 msk_batch = torch.cuda.FloatTensor(cls_mask.size()).fill_(1) - cls_mask # 这里取出有人所对应的msk # Pad patch and mask to image dimensions mypad = nn.ConstantPad2d((int(pad + 0.5), int(pad), int(pad + 0.5), int(pad)), 0) # (padding_left、padding_right、padding_top、padding_bottom) 填充0 adv_batch = mypad(adv_batch) # 用0填充到416 msk_batch = mypad(msk_batch) # Rotation and rescaling transforms anglesize = (lab_batch.size(0) * lab_batch.size(1)) # 这里是旋转的数量 if do_rotate: angle = torch.cuda.FloatTensor(anglesize).uniform_(self.minangle, self.maxangle) else: angle = torch.cuda.FloatTensor(anglesize).fill_(0) # Resizes and rotates current_patch_size = adv_patch.size(-1) lab_batch_scaled = torch.cuda.FloatTensor(lab_batch.size()).fill_(0) # lab_batch_scaled是在原图上的尺寸？ lab_batch_scaled[:, :, 1] = lab_batch[:, :, 1] * img_size lab_batch_scaled[:, :, 2] = lab_batch[:, :, 2] * img_size lab_batch_scaled[:, :, 3] = lab_batch[:, :, 3] * img_size lab_batch_scaled[:, :, 4] = lab_batch[:, :, 4] * img_size target_size = torch.sqrt( ((lab_batch_scaled[:, :, 3].mul(0.2)) ** 2) + ((lab_batch_scaled[:, :, 4].mul(0.2)) ** 2)) target_x = lab_batch[:, :, 1].view(np.prod(batch_size)) target_y = lab_batch[:, :, 2].view(np.prod(batch_size)) targetoff_x = lab_batch[:, :, 3].view(np.prod(batch_size)) targetoff_y = lab_batch[:, :, 4].view(np.prod(batch_size)) if (rand_loc): off_x = targetoff_x * (torch.cuda.FloatTensor(targetoff_x.size()).uniform_(-0.4, 0.4)) target_x = target_x + off_x off_y = targetoff_y * (torch.cuda.FloatTensor(targetoff_y.size()).uniform_(-0.4, 0.4)) target_y = target_y + off_y target_y = target_y - 0.05 scale = target_size / current_patch_size # 原图相对于补丁大小的缩放因子？ scale = scale.view(anglesize) s = adv_batch.size() adv_batch = adv_batch.view(s[0] * s[1], s[2], s[3], s[4]) msk_batch = msk_batch.view(s[0] * s[1], s[2], s[3], s[4]) tx = (-target_x + 0.5) * 2 ty = (-target_y + 0.5) * 2 sin = torch.sin(angle) cos = torch.cos(angle) # Theta = rotation,rescale matrix theta = torch.cuda.FloatTensor(anglesize, 2, 3).fill_(0) theta[:, 0, 0] = cos / scale theta[:, 0, 1] = sin / scale theta[:, 0, 2] = tx * cos / scale + ty * sin / scale theta[:, 1, 0] = -sin / scale theta[:, 1, 1] = cos / scale theta[:, 1, 2] = -tx * sin / scale + ty * cos / scale b_sh = adv_batch.shape grid = F.affine_grid(theta, adv_batch.shape) adv_batch_t = F.grid_sample(adv_batch, grid) msk_batch_t = F.grid_sample(msk_batch, grid) ''' # Theta2 = translation matrix theta2 = torch.cuda.FloatTensor(anglesize, 2, 3).fill_(0) theta2[:, 0, 0] = 1 theta2[:, 0, 1] = 0 theta2[:, 0, 2] = (-target_x + 0.5) * 2 theta2[:, 1, 0] = 0 theta2[:, 1, 1] = 1 theta2[:, 1, 2] = (-target_y + 0.5) * 2 grid2 = F.affine_grid(theta2, adv_batch.shape) adv_batch_t = F.grid_sample(adv_batch_t, grid2) msk_batch_t = F.grid_sample(msk_batch_t, grid2) ''' adv_batch_t = adv_batch_t.view(s[0], s[1], s[2], s[3], s[4]) msk_batch_t = msk_batch_t.view(s[0], s[1], s[2], s[3], s[4]) adv_batch_t = torch.clamp(adv_batch_t, 0.000001, 0.999999) # img = msk_batch_t[0, 0, :, :, :].detach().cpu() # img = transforms.ToPILImage()(img) # img.show() # exit() return adv_batch_t * msk_batch_t class PatchApplier(nn.Module): """PatchApplier: applies adversarial patches to images. Module providing the functionality necessary to apply a patch to all detections in all images in the batch. PatchApplier：对图像应用对抗补丁。 """ def __init__(self): super(PatchApplier, self).__init__() def forward(self, img_batch, adv_batch): advs = torch.unbind(adv_batch, 1) # 沿1维解开 for adv in advs: img_batch = torch.where((adv == 0), img_batch, adv) # 对图像相应的坐标位置替换其像素？好像还没到图像的环节 return img_batch ''' class PatchGenerator(nn.Module): """PatchGenerator: network module that generates adversarial patches. Module representing the neural network that will generate adversarial patches. """ def __init__(self, cfgfile, weightfile, img_dir, lab_dir): super(PatchGenerator, self).__init__() self.yolo = Darknet(cfgfile).load_weights(weightfile) self.dataloader = torch.utils.data.DataLoader(InriaDataset(img_dir, lab_dir, shuffle=True), batch_size=5, shuffle=True) self.patchapplier = PatchApplier() self.nmscalculator = NMSCalculator() self.totalvariation = TotalVariation() def forward(self, *input): pass ''' class InriaDataset(Dataset): """InriaDataset: representation of the INRIA person dataset. Internal representation of the commonly used INRIA person dataset. Available at: http://pascal.inrialpes.fr/data/human/ Attributes: len: An integer number of elements in the img_dir: Directory containing the images of the INRIA dataset. lab_dir: Directory containing the labels of the INRIA dataset. img_names: List of all image file names in img_dir. shuffle: Whether or not to shuffle the dataset. """ def __init__(self, img_dir, lab_dir, max_lab, imgsize, shuffle=True): n_png_images = len(fnmatch.filter(os.listdir(img_dir), '*.png')) # 614 fnmatch.filter返回一个list n_jpg_images = len(fnmatch.filter(os.listdir(img_dir), '*.jpg')) # 0 n_images = n_png_images + n_jpg_images # 图像的总数 n_labels = len(fnmatch.filter(os.listdir(lab_dir), '*.txt')) assert n_images == n_labels, "Number of images and number of labels don't match" self.len = n_images self.img_dir = img_dir self.lab_dir = lab_dir self.imgsize = imgsize self.img_names = fnmatch.filter(os.listdir(img_dir), '*.png') + fnmatch.filter(os.listdir(img_dir), '*.jpg') self.shuffle = shuffle self.img_paths = [] for img_name in self.img_names: self.img_paths.append(os.path.join(self.img_dir, img_name)) self.lab_paths = [] for img_name in self.img_names: lab_path = os.path.join(self.lab_dir, img_name).replace('.jpg', '.txt').replace('.png', '.txt') self.lab_paths.append(lab_path) self.max_n_labels = max_lab # label的长度 def __len__(self): return self.len def __getitem__(self, idx): assert idx <= len(self), 'index range error' img_path = os.path.join(self.img_dir, self.img_names[idx]) lab_path = os.path.join(self.lab_dir, self.img_names[idx]).replace('.jpg', '.txt').replace('.png', '.txt') image = Image.open(img_path).convert('RGB') if os.path.getsize(lab_path): # check to see if label file contains data. label = np.loadtxt(lab_path) else: label = np.ones([5]) label = torch.from_numpy(label).float() if label.dim() == 1: label = label.unsqueeze(0) image, label = self.pad_and_scale(image, label) transform = transforms.ToTensor() image = transform(image) label = self.pad_lab(label) # print("image size :", image.shape) # print("label size :", label.shape) return image, label def pad_and_scale(self, img, lab): """ Args: img: Returns: """ w, h = img.size if w == h: padded_img = img else: dim_to_pad = 1 if w < h else 2 if dim_to_pad == 1: padding = (h - w) / 2 padded_img = Image.new('RGB', (h, h), color=(127, 127, 127)) padded_img.paste(img, (int(padding), 0)) lab[:, [1]] = (lab[:, [1]] * w + padding) / h lab[:, [3]] = (lab[:, [3]] * w / h) else: padding = (w - h) / 2 padded_img = Image.new('RGB', (w, w), color=(127, 127, 127)) padded_img.paste(img, (0, int(padding))) lab[:, [2]] = (lab[:, [2]] * h + padding) / w lab[:, [4]] = (lab[:, [4]] * h / w) resize = transforms.Resize((self.imgsize, self.imgsize)) padded_img = resize(padded_img) # choose here return padded_img, lab def pad_lab(self, lab): pad_size = self.max_n_labels - lab.shape[0] if (pad_size > 0): padded_lab = F.pad(lab, (0, 0, 0, pad_size), value=1) # (左边填充数， 右边填充数， 上边填充数， 下边填充数) else: padded_lab = lab return padded_lab if __name__ == '__main__': if len(sys.argv) == 3: img_dir = sys.argv[1] lab_dir = sys.argv[2] else: print('Usage: ') print(' python load_data.py img_dir lab_dir') sys.exit() test_loader = torch.utils.data.DataLoader(InriaDataset(img_dir, lab_dir, shuffle=True), batch_size=3, shuffle=True) cfgfile = "cfg/yolov2.cfg" weightfile = "weights/yolov2.weights" printfile = "non_printability/30values.txt" patch_size = 400 darknet_model = Darknet(cfgfile) darknet_model.load_weights(weightfile) darknet_model = darknet_model.cuda() patch_applier = PatchApplier().cuda() patch_transformer = PatchTransformer().cuda() prob_extractor = MaxProbExtractor(0, 80).cuda() nms_calculator = NMSCalculator(printfile, patch_size) total_variation = TotalVariation() ''' img = Image.open('data/horse.jpg').convert('RGB') img = img.resize((darknet_model.width, darknet_model.height)) width = img.width height = img.height img = torch.ByteTensor(torch.ByteStorage.from_buffer(img.tobytes())) img = img.view(height, width, 3).transpose(0, 1).transpose(0, 2).contiguous() img = img.view(1, 3, height, width) img = img.float().div(255.0) img = torch.autograd.Variable(img) output = darknet_model(img) ''' optimizer = torch.optim.Adam(model.parameters(), lr=0.0001) tl0 = time.time() tl1 = time.time() for i_batch, (img_batch, lab_batch) in enumerate(test_loader): tl1 = time.time() print('time to fetch items: ', tl1 - tl0) img_batch = img_batch.cuda() lab_batch = lab_batch.cuda() adv_patch = Image.open('data/horse.jpg').convert('RGB') adv_patch = adv_patch.resize((patch_size, patch_size)) transform = transforms.ToTensor() adv_patch = transform(adv_patch).cuda() img_size = img_batch.size(-1) print('transforming patches') t0 = time.time() adv_batch_t = patch_transformer.forward(adv_patch, lab_batch, img_size) print('applying patches') t1 = time.time() img_batch = patch_applier.forward(img_batch, adv_batch_t) img_batch = torch.autograd.Variable(img_batch) img_batch = F.interpolate(img_batch, (darknet_model.height, darknet_model.width)) print('running patched images through model') t2 = time.time() for obj in gc.get_objects(): try: if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)): try: print(type(obj), obj.size()) except: pass except: pass print(torch.cuda.memory_allocated()) output = darknet_model(img_batch) print('extracting max probs') t3 = time.time() max_prob = prob_extractor(output) t4 = time.time() nms = nms_calculator.forward(adv_patch) tv = total_variation(adv_patch) print('---------------------------------') print(' patch transformation : %f' % (t1 - t0)) print(' patch application : %f' % (t2 - t1)) print(' darknet forward : %f' % (t3 - t2)) print(' probability extraction : %f' % (t4 - t3)) print('---------------------------------') print(' total forward pass : %f' % (t4 - t0)) del img_batch, lab_batch, adv_patch, adv_batch_t, output, max_prob torch.cuda.empty_cache() tl0 = time.time()