532 lines
22 KiB
Python
532 lines
22 KiB
Python
|
import fnmatch
|
|||
|
|
import math
|
|||
|
|
import os
|
|||
|
|
import sys
|
|||
|
|
import time
|
|||
|
|
from operator import itemgetter
|
|||
|
|
|
|||
|
|
import gc
|
|||
|
|
import numpy as np
|
|||
|
|
import torch
|
|||
|
|
import torch.optim as optim
|
|||
|
|
import torch.nn as nn
|
|||
|
|
import torch.nn.functional as F
|
|||
|
|
from PIL import Image
|
|||
|
|
from torch.utils.data import Dataset
|
|||
|
|
from torchvision import transforms
|
|||
|
|
|
|||
|
|
# from darknet import Darknet
|
|||
|
|
|
|||
|
|
from median_pool import MedianPool2d
|
|||
|
|
|
|||
|
|
# print('starting test read')
|
|||
|
|
# im = Image.open('data/horse.jpg').convert('RGB')
|
|||
|
|
# print('img read!')
|
|||
|
|
|
|||
|
|
|
|||
|
|
class MaxProbExtractor(nn.Module):
|
|||
|
|
"""MaxProbExtractor: extracts max class probability for class from YOLO output.
|
|||
|
|
|
|||
|
|
Module providing the functionality necessary to extract the max class probability for one class from YOLO output.
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self, cls_id, num_cls, config):
|
|||
|
|
super(MaxProbExtractor, self).__init__()
|
|||
|
|
self.cls_id = cls_id
|
|||
|
|
self.num_cls = num_cls
|
|||
|
|
self.config = config
|
|||
|
|
self.anchor_num = 3
|
|||
|
|
|
|||
|
|
def forward(self, YOLOoutput):
|
|||
|
|
# get values neccesary for transformation
|
|||
|
|
if YOLOoutput.dim() == 3:
|
|||
|
|
YOLOoutput = YOLOoutput.unsqueeze(0)
|
|||
|
|
batch = YOLOoutput.size(0)
|
|||
|
|
assert (YOLOoutput.size(1) == (5 + self.num_cls) * self.anchor_num)
|
|||
|
|
h = YOLOoutput.size(2)
|
|||
|
|
w = YOLOoutput.size(3)
|
|||
|
|
# transform the output tensor from [batch, 425, 19, 19] to [batch, 80, 1805]
|
|||
|
|
output = YOLOoutput.view(batch, self.anchor_num, 5 + self.num_cls, h * w) # [batch, 5, 85, 361]
|
|||
|
|
output = output.transpose(1, 2).contiguous() # [batch, 85, 5, 361]
|
|||
|
|
output = output.view(batch, 5 + self.num_cls, self.anchor_num * h * w) # [batch, 85, 1805]
|
|||
|
|
output_objectness = torch.sigmoid(output[:, 4, :]) # [batch, 1805] # 是否有物体
|
|||
|
|
output = output[:, 5:5 + self.num_cls, :] # [batch, 80, 1805]
|
|||
|
|
# perform softmax to normalize probabilities for object classes to [0,1]
|
|||
|
|
normal_confs = torch.nn.Softmax(dim=1)(output) # 物体类别
|
|||
|
|
# we only care for probabilities of the class of interest (person)
|
|||
|
|
confs_for_class = normal_confs[:, self.cls_id, :] # 类别 序号对应的为人
|
|||
|
|
confs_if_object = output_objectness # confs_for_class * output_objectness
|
|||
|
|
confs_if_object = confs_for_class * output_objectness
|
|||
|
|
confs_if_object = self.config.loss_target(output_objectness, confs_for_class)
|
|||
|
|
# find the max probability for person
|
|||
|
|
max_conf, max_conf_idx = torch.max(confs_if_object, dim=1)
|
|||
|
|
|
|||
|
|
return max_conf
|
|||
|
|
|
|||
|
|
|
|||
|
|
class NPSCalculator(nn.Module):
|
|||
|
|
"""NMSCalculator: calculates the non-printability score of a patch.
|
|||
|
|
|
|||
|
|
Module providing the functionality necessary to calculate the non-printability score (NMS) of an adversarial patch.
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self, printability_file, patch_side):
|
|||
|
|
super(NPSCalculator, self).__init__()
|
|||
|
|
self.printability_array = nn.Parameter(self.get_printability_array(printability_file, patch_side),
|
|||
|
|
requires_grad=False)
|
|||
|
|
|
|||
|
|
def forward(self, adv_patch):
|
|||
|
|
# calculate euclidian distance between colors in patch and colors in printability_array
|
|||
|
|
# square root of sum of squared difference
|
|||
|
|
color_dist = (adv_patch - self.printability_array + 0.000001)
|
|||
|
|
color_dist = color_dist ** 2
|
|||
|
|
color_dist = torch.sum(color_dist, 1) + 0.000001
|
|||
|
|
color_dist = torch.sqrt(color_dist)
|
|||
|
|
# only work with the min distance
|
|||
|
|
color_dist_prod = torch.min(color_dist, 0)[0] # test: change prod for min (find distance to closest color)
|
|||
|
|
# calculate the nps by summing over all pixels
|
|||
|
|
nps_score = torch.sum(color_dist_prod, 0)
|
|||
|
|
nps_score = torch.sum(nps_score, 0)
|
|||
|
|
return nps_score / torch.numel(adv_patch)
|
|||
|
|
|
|||
|
|
def get_printability_array(self, printability_file, side):
|
|||
|
|
printability_list = []
|
|||
|
|
|
|||
|
|
# read in printability triplets and put them in a list
|
|||
|
|
with open(printability_file) as f:
|
|||
|
|
for line in f:
|
|||
|
|
printability_list.append(line.split(","))
|
|||
|
|
|
|||
|
|
printability_array = []
|
|||
|
|
for printability_triplet in printability_list:
|
|||
|
|
printability_imgs = []
|
|||
|
|
red, green, blue = printability_triplet
|
|||
|
|
printability_imgs.append(np.full((side, side), red))
|
|||
|
|
printability_imgs.append(np.full((side, side), green))
|
|||
|
|
printability_imgs.append(np.full((side, side), blue))
|
|||
|
|
printability_array.append(printability_imgs)
|
|||
|
|
|
|||
|
|
printability_array = np.asarray(printability_array)
|
|||
|
|
printability_array = np.float32(printability_array)
|
|||
|
|
pa = torch.from_numpy(printability_array)
|
|||
|
|
return pa
|
|||
|
|
|
|||
|
|
|
|||
|
|
class TotalVariation(nn.Module):
|
|||
|
|
"""TotalVariation: calculates the total variation of a patch.
|
|||
|
|
|
|||
|
|
Module providing the functionality necessary to calculate the total Variation (TV) of an adversarial patch.
|
|||
|
|
|
|||
|
|
TotalVariation：计算补丁的总变化。
|
|||
|
|
该模块提供了计算对抗性补丁的总变化 (TV) 所需的功能。
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self):
|
|||
|
|
super(TotalVariation, self).__init__()
|
|||
|
|
|
|||
|
|
def forward(self, adv_patch):
|
|||
|
|
# bereken de total variation van de adv_patch
|
|||
|
|
tvcomp1 = torch.sum(torch.abs(adv_patch[:, :, 1:] - adv_patch[:, :, :-1] + 0.000001), 0)
|
|||
|
|
tvcomp1 = torch.sum(torch.sum(tvcomp1, 0), 0)
|
|||
|
|
tvcomp2 = torch.sum(torch.abs(adv_patch[:, 1:, :] - adv_patch[:, :-1, :] + 0.000001), 0)
|
|||
|
|
tvcomp2 = torch.sum(torch.sum(tvcomp2, 0), 0)
|
|||
|
|
tv = tvcomp1 + tvcomp2
|
|||
|
|
return tv / torch.numel(adv_patch)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class PatchTransformer(nn.Module):
|
|||
|
|
"""PatchTransformer: transforms batch of patches
|
|||
|
|
|
|||
|
|
Module providing the functionality necessary to transform a batch of patches, randomly adjusting brightness and
|
|||
|
|
contrast, adding random amount of noise, and rotating randomly. Resizes-patches according to as size based on the
|
|||
|
|
batch of labels, and pads them to the dimension of an image.
|
|||
|
|
|
|||
|
|
变换一批补丁，随机调整亮度和对比度，添加随机数量的噪声，随机旋转。 根据标签批次的大小调整补丁大小，并将它们填充到图像的尺寸中。
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self):
|
|||
|
|
super(PatchTransformer, self).__init__()
|
|||
|
|
self.min_contrast = 0.8
|
|||
|
|
self.max_contrast = 1.2
|
|||
|
|
self.min_brightness = -0.1
|
|||
|
|
self.max_brightness = 0.1
|
|||
|
|
self.noise_factor = 0.10
|
|||
|
|
self.minangle = -20 / 180 * math.pi
|
|||
|
|
self.maxangle = 20 / 180 * math.pi
|
|||
|
|
self.medianpooler = MedianPool2d(7, same=True) # 中值池化
|
|||
|
|
'''
|
|||
|
|
kernel = torch.cuda.FloatTensor([[0.003765, 0.015019, 0.023792, 0.015019, 0.003765],
|
|||
|
|
[0.015019, 0.059912, 0.094907, 0.059912, 0.015019],
|
|||
|
|
[0.023792, 0.094907, 0.150342, 0.094907, 0.023792],
|
|||
|
|
[0.015019, 0.059912, 0.094907, 0.059912, 0.015019],
|
|||
|
|
[0.003765, 0.015019, 0.023792, 0.015019, 0.003765]])
|
|||
|
|
self.kernel = kernel.unsqueeze(0).unsqueeze(0).expand(3,3,-1,-1)
|
|||
|
|
'''
|
|||
|
|
|
|||
|
|
def forward(self, adv_patch, lab_batch, img_size, do_rotate=True, rand_loc=True):
|
|||
|
|
# adv_patch = F.conv2d(adv_patch.unsqueeze(0),self.kernel,padding=(2,2))
|
|||
|
|
adv_patch = self.medianpooler(adv_patch.unsqueeze(0))
|
|||
|
|
# Determine size of padding
|
|||
|
|
pad = (img_size - adv_patch.size(-1)) / 2
|
|||
|
|
# Make a batch of patches
|
|||
|
|
adv_patch = adv_patch.unsqueeze(0) # .unsqueeze(0) # 这里又扩大一维，变成5维 1, 1, 3, 300, 300
|
|||
|
|
adv_batch = adv_patch.expand(lab_batch.size(0), lab_batch.size(1), -1, -1, -1) # adv_batch ！！ 不是adv_patch！！ 8, 14, 3, 300, 300
|
|||
|
|
batch_size = torch.Size((lab_batch.size(0), lab_batch.size(1))) # 8, 14
|
|||
|
|
|
|||
|
|
# Contrast, brightness and noise transforms
|
|||
|
|
|
|||
|
|
# Create random contrast tensor
|
|||
|
|
contrast = torch.cuda.FloatTensor(batch_size).uniform_(self.min_contrast, self.max_contrast)
|
|||
|
|
contrast = contrast.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1)
|
|||
|
|
contrast = contrast.expand(-1, -1, adv_batch.size(-3), adv_batch.size(-2), adv_batch.size(-1))
|
|||
|
|
contrast = contrast.cuda()
|
|||
|
|
|
|||
|
|
# Create random brightness tensor
|
|||
|
|
brightness = torch.cuda.FloatTensor(batch_size).uniform_(self.min_brightness, self.max_brightness)
|
|||
|
|
brightness = brightness.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1)
|
|||
|
|
brightness = brightness.expand(-1, -1, adv_batch.size(-3), adv_batch.size(-2), adv_batch.size(-1))
|
|||
|
|
brightness = brightness.cuda()
|
|||
|
|
|
|||
|
|
# Create random noise tensor
|
|||
|
|
noise = torch.cuda.FloatTensor(adv_batch.size()).uniform_(-1, 1) * self.noise_factor
|
|||
|
|
|
|||
|
|
# Apply contrast/brightness/noise, clamp
|
|||
|
|
adv_batch = adv_batch * contrast + brightness + noise
|
|||
|
|
|
|||
|
|
adv_batch = torch.clamp(adv_batch, 0.000001, 0.99999) # 限制到0到1之间
|
|||
|
|
|
|||
|
|
# Where the label class_id is 1 we don't want a patch (padding) --> fill mask with zero's
|
|||
|
|
cls_ids = torch.narrow(lab_batch, 2, 0, 1) # torch.narrow(input,dim,start,length) 从dim开始，返回共享内存的数据start到start+length-1
|
|||
|
|
cls_mask = cls_ids.expand(-1, -1, 3) # 接上，这里取出 lab_batch的代表id那列，相当于现在的lab_batch[..., 0]
|
|||
|
|
cls_mask = cls_mask.unsqueeze(-1)
|
|||
|
|
cls_mask = cls_mask.expand(-1, -1, -1, adv_batch.size(3))
|
|||
|
|
cls_mask = cls_mask.unsqueeze(-1)
|
|||
|
|
cls_mask = cls_mask.expand(-1, -1, -1, -1, adv_batch.size(4)) # cls_mask 的大小是 8, 14, 3, 300, 300 数据是类别
|
|||
|
|
msk_batch = torch.cuda.FloatTensor(cls_mask.size()).fill_(1) - cls_mask # 这里取出有人所对应的msk
|
|||
|
|
|
|||
|
|
# Pad patch and mask to image dimensions
|
|||
|
|
mypad = nn.ConstantPad2d((int(pad + 0.5), int(pad), int(pad + 0.5), int(pad)), 0) # (padding_left、padding_right、padding_top、padding_bottom) 填充0
|
|||
|
|
adv_batch = mypad(adv_batch) # 用0填充到416
|
|||
|
|
msk_batch = mypad(msk_batch)
|
|||
|
|
|
|||
|
|
# Rotation and rescaling transforms
|
|||
|
|
anglesize = (lab_batch.size(0) * lab_batch.size(1)) # 这里是旋转的数量
|
|||
|
|
if do_rotate:
|
|||
|
|
angle = torch.cuda.FloatTensor(anglesize).uniform_(self.minangle, self.maxangle)
|
|||
|
|
else:
|
|||
|
|
angle = torch.cuda.FloatTensor(anglesize).fill_(0)
|
|||
|
|
|
|||
|
|
# Resizes and rotates
|
|||
|
|
current_patch_size = adv_patch.size(-1)
|
|||
|
|
lab_batch_scaled = torch.cuda.FloatTensor(lab_batch.size()).fill_(0) # lab_batch_scaled是在原图上的尺寸？
|
|||
|
|
lab_batch_scaled[:, :, 1] = lab_batch[:, :, 1] * img_size
|
|||
|
|
lab_batch_scaled[:, :, 2] = lab_batch[:, :, 2] * img_size
|
|||
|
|
lab_batch_scaled[:, :, 3] = lab_batch[:, :, 3] * img_size
|
|||
|
|
lab_batch_scaled[:, :, 4] = lab_batch[:, :, 4] * img_size
|
|||
|
|
target_size = torch.sqrt(
|
|||
|
|
((lab_batch_scaled[:, :, 3].mul(0.2)) ** 2) + ((lab_batch_scaled[:, :, 4].mul(0.2)) ** 2))
|
|||
|
|
target_x = lab_batch[:, :, 1].view(np.prod(batch_size))
|
|||
|
|
target_y = lab_batch[:, :, 2].view(np.prod(batch_size))
|
|||
|
|
targetoff_x = lab_batch[:, :, 3].view(np.prod(batch_size))
|
|||
|
|
targetoff_y = lab_batch[:, :, 4].view(np.prod(batch_size))
|
|||
|
|
if (rand_loc):
|
|||
|
|
off_x = targetoff_x * (torch.cuda.FloatTensor(targetoff_x.size()).uniform_(-0.4, 0.4))
|
|||
|
|
target_x = target_x + off_x
|
|||
|
|
off_y = targetoff_y * (torch.cuda.FloatTensor(targetoff_y.size()).uniform_(-0.4, 0.4))
|
|||
|
|
target_y = target_y + off_y
|
|||
|
|
target_y = target_y - 0.05
|
|||
|
|
scale = target_size / current_patch_size # 原图相对于补丁大小的缩放因子？
|
|||
|
|
scale = scale.view(anglesize)
|
|||
|
|
|
|||
|
|
s = adv_batch.size()
|
|||
|
|
adv_batch = adv_batch.view(s[0] * s[1], s[2], s[3], s[4])
|
|||
|
|
msk_batch = msk_batch.view(s[0] * s[1], s[2], s[3], s[4])
|
|||
|
|
|
|||
|
|
tx = (-target_x + 0.5) * 2
|
|||
|
|
ty = (-target_y + 0.5) * 2
|
|||
|
|
sin = torch.sin(angle)
|
|||
|
|
cos = torch.cos(angle)
|
|||
|
|
|
|||
|
|
# Theta = rotation,rescale matrix
|
|||
|
|
theta = torch.cuda.FloatTensor(anglesize, 2, 3).fill_(0)
|
|||
|
|
theta[:, 0, 0] = cos / scale
|
|||
|
|
theta[:, 0, 1] = sin / scale
|
|||
|
|
theta[:, 0, 2] = tx * cos / scale + ty * sin / scale
|
|||
|
|
theta[:, 1, 0] = -sin / scale
|
|||
|
|
theta[:, 1, 1] = cos / scale
|
|||
|
|
theta[:, 1, 2] = -tx * sin / scale + ty * cos / scale
|
|||
|
|
|
|||
|
|
b_sh = adv_batch.shape
|
|||
|
|
grid = F.affine_grid(theta, adv_batch.shape)
|
|||
|
|
|
|||
|
|
adv_batch_t = F.grid_sample(adv_batch, grid)
|
|||
|
|
msk_batch_t = F.grid_sample(msk_batch, grid)
|
|||
|
|
|
|||
|
|
'''
|
|||
|
|
# Theta2 = translation matrix
|
|||
|
|
theta2 = torch.cuda.FloatTensor(anglesize, 2, 3).fill_(0)
|
|||
|
|
theta2[:, 0, 0] = 1
|
|||
|
|
theta2[:, 0, 1] = 0
|
|||
|
|
theta2[:, 0, 2] = (-target_x + 0.5) * 2
|
|||
|
|
theta2[:, 1, 0] = 0
|
|||
|
|
theta2[:, 1, 1] = 1
|
|||
|
|
theta2[:, 1, 2] = (-target_y + 0.5) * 2
|
|||
|
|
|
|||
|
|
grid2 = F.affine_grid(theta2, adv_batch.shape)
|
|||
|
|
adv_batch_t = F.grid_sample(adv_batch_t, grid2)
|
|||
|
|
msk_batch_t = F.grid_sample(msk_batch_t, grid2)
|
|||
|
|
|
|||
|
|
'''
|
|||
|
|
adv_batch_t = adv_batch_t.view(s[0], s[1], s[2], s[3], s[4])
|
|||
|
|
msk_batch_t = msk_batch_t.view(s[0], s[1], s[2], s[3], s[4])
|
|||
|
|
|
|||
|
|
adv_batch_t = torch.clamp(adv_batch_t, 0.000001, 0.999999)
|
|||
|
|
# img = msk_batch_t[0, 0, :, :, :].detach().cpu()
|
|||
|
|
# img = transforms.ToPILImage()(img)
|
|||
|
|
# img.show()
|
|||
|
|
# exit()
|
|||
|
|
|
|||
|
|
return adv_batch_t * msk_batch_t
|
|||
|
|
|
|||
|
|
|
|||
|
|
class PatchApplier(nn.Module):
|
|||
|
|
"""PatchApplier: applies adversarial patches to images.
|
|||
|
|
|
|||
|
|
Module providing the functionality necessary to apply a patch to all detections in all images in the batch.
|
|||
|
|
|
|||
|
|
PatchApplier：对图像应用对抗补丁。
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self):
|
|||
|
|
super(PatchApplier, self).__init__()
|
|||
|
|
|
|||
|
|
def forward(self, img_batch, adv_batch):
|
|||
|
|
advs = torch.unbind(adv_batch, 1) # 沿1维解开
|
|||
|
|
for adv in advs:
|
|||
|
|
img_batch = torch.where((adv == 0), img_batch, adv) # 对图像相应的坐标位置替换其像素？好像还没到图像的环节
|
|||
|
|
return img_batch
|
|||
|
|
|
|||
|
|
|
|||
|
|
'''
|
|||
|
|
class PatchGenerator(nn.Module):
|
|||
|
|
"""PatchGenerator: network module that generates adversarial patches.
|
|||
|
|
|
|||
|
|
Module representing the neural network that will generate adversarial patches.
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self, cfgfile, weightfile, img_dir, lab_dir):
|
|||
|
|
super(PatchGenerator, self).__init__()
|
|||
|
|
self.yolo = Darknet(cfgfile).load_weights(weightfile)
|
|||
|
|
self.dataloader = torch.utils.data.DataLoader(InriaDataset(img_dir, lab_dir, shuffle=True),
|
|||
|
|
batch_size=5,
|
|||
|
|
shuffle=True)
|
|||
|
|
self.patchapplier = PatchApplier()
|
|||
|
|
self.nmscalculator = NMSCalculator()
|
|||
|
|
self.totalvariation = TotalVariation()
|
|||
|
|
|
|||
|
|
def forward(self, *input):
|
|||
|
|
pass
|
|||
|
|
'''
|
|||
|
|
|
|||
|
|
|
|||
|
|
class InriaDataset(Dataset):
|
|||
|
|
"""InriaDataset: representation of the INRIA person dataset.
|
|||
|
|
|
|||
|
|
Internal representation of the commonly used INRIA person dataset.
|
|||
|
|
Available at: http://pascal.inrialpes.fr/data/human/
|
|||
|
|
|
|||
|
|
Attributes:
|
|||
|
|
len: An integer number of elements in the
|
|||
|
|
img_dir: Directory containing the images of the INRIA dataset.
|
|||
|
|
lab_dir: Directory containing the labels of the INRIA dataset.
|
|||
|
|
img_names: List of all image file names in img_dir.
|
|||
|
|
shuffle: Whether or not to shuffle the dataset.
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self, img_dir, lab_dir, max_lab, imgsize, shuffle=True):
|
|||
|
|
n_png_images = len(fnmatch.filter(os.listdir(img_dir), '*.png')) # 614 fnmatch.filter返回一个list
|
|||
|
|
n_jpg_images = len(fnmatch.filter(os.listdir(img_dir), '*.jpg')) # 0
|
|||
|
|
n_images = n_png_images + n_jpg_images # 图像的总数
|
|||
|
|
n_labels = len(fnmatch.filter(os.listdir(lab_dir), '*.txt'))
|
|||
|
|
assert n_images == n_labels, "Number of images and number of labels don't match"
|
|||
|
|
self.len = n_images
|
|||
|
|
self.img_dir = img_dir
|
|||
|
|
self.lab_dir = lab_dir
|
|||
|
|
self.imgsize = imgsize
|
|||
|
|
self.img_names = fnmatch.filter(os.listdir(img_dir), '*.png') + fnmatch.filter(os.listdir(img_dir), '*.jpg')
|
|||
|
|
self.shuffle = shuffle
|
|||
|
|
self.img_paths = []
|
|||
|
|
for img_name in self.img_names:
|
|||
|
|
self.img_paths.append(os.path.join(self.img_dir, img_name))
|
|||
|
|
self.lab_paths = []
|
|||
|
|
for img_name in self.img_names:
|
|||
|
|
lab_path = os.path.join(self.lab_dir, img_name).replace('.jpg', '.txt').replace('.png', '.txt')
|
|||
|
|
self.lab_paths.append(lab_path)
|
|||
|
|
self.max_n_labels = max_lab # label的长度
|
|||
|
|
|
|||
|
|
def __len__(self):
|
|||
|
|
return self.len
|
|||
|
|
|
|||
|
|
def __getitem__(self, idx):
|
|||
|
|
assert idx <= len(self), 'index range error'
|
|||
|
|
img_path = os.path.join(self.img_dir, self.img_names[idx])
|
|||
|
|
lab_path = os.path.join(self.lab_dir, self.img_names[idx]).replace('.jpg', '.txt').replace('.png', '.txt')
|
|||
|
|
image = Image.open(img_path).convert('RGB')
|
|||
|
|
if os.path.getsize(lab_path): # check to see if label file contains data.
|
|||
|
|
label = np.loadtxt(lab_path)
|
|||
|
|
else:
|
|||
|
|
label = np.ones([5])
|
|||
|
|
|
|||
|
|
label = torch.from_numpy(label).float()
|
|||
|
|
if label.dim() == 1:
|
|||
|
|
label = label.unsqueeze(0)
|
|||
|
|
|
|||
|
|
image, label = self.pad_and_scale(image, label)
|
|||
|
|
transform = transforms.ToTensor()
|
|||
|
|
image = transform(image)
|
|||
|
|
label = self.pad_lab(label)
|
|||
|
|
# print("image size :", image.shape)
|
|||
|
|
# print("label size :", label.shape)
|
|||
|
|
return image, label
|
|||
|
|
|
|||
|
|
def pad_and_scale(self, img, lab):
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
img:
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
w, h = img.size
|
|||
|
|
if w == h:
|
|||
|
|
padded_img = img
|
|||
|
|
else:
|
|||
|
|
dim_to_pad = 1 if w < h else 2
|
|||
|
|
if dim_to_pad == 1:
|
|||
|
|
padding = (h - w) / 2
|
|||
|
|
padded_img = Image.new('RGB', (h, h), color=(127, 127, 127))
|
|||
|
|
padded_img.paste(img, (int(padding), 0))
|
|||
|
|
lab[:, [1]] = (lab[:, [1]] * w + padding) / h
|
|||
|
|
lab[:, [3]] = (lab[:, [3]] * w / h)
|
|||
|
|
else:
|
|||
|
|
padding = (w - h) / 2
|
|||
|
|
padded_img = Image.new('RGB', (w, w), color=(127, 127, 127))
|
|||
|
|
padded_img.paste(img, (0, int(padding)))
|
|||
|
|
lab[:, [2]] = (lab[:, [2]] * h + padding) / w
|
|||
|
|
lab[:, [4]] = (lab[:, [4]] * h / w)
|
|||
|
|
resize = transforms.Resize((self.imgsize, self.imgsize))
|
|||
|
|
padded_img = resize(padded_img) # choose here
|
|||
|
|
return padded_img, lab
|
|||
|
|
|
|||
|
|
def pad_lab(self, lab):
|
|||
|
|
pad_size = self.max_n_labels - lab.shape[0]
|
|||
|
|
if (pad_size > 0):
|
|||
|
|
padded_lab = F.pad(lab, (0, 0, 0, pad_size), value=1) # (左边填充数， 右边填充数， 上边填充数， 下边填充数)
|
|||
|
|
else:
|
|||
|
|
padded_lab = lab
|
|||
|
|
return padded_lab
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
if len(sys.argv) == 3:
|
|||
|
|
img_dir = sys.argv[1]
|
|||
|
|
lab_dir = sys.argv[2]
|
|||
|
|
|
|||
|
|
else:
|
|||
|
|
print('Usage: ')
|
|||
|
|
print(' python load_data.py img_dir lab_dir')
|
|||
|
|
sys.exit()
|
|||
|
|
|
|||
|
|
test_loader = torch.utils.data.DataLoader(InriaDataset(img_dir, lab_dir, shuffle=True),
|
|||
|
|
batch_size=3, shuffle=True)
|
|||
|
|
|
|||
|
|
cfgfile = "cfg/yolov2.cfg"
|
|||
|
|
weightfile = "weights/yolov2.weights"
|
|||
|
|
printfile = "non_printability/30values.txt"
|
|||
|
|
|
|||
|
|
patch_size = 400
|
|||
|
|
|
|||
|
|
darknet_model = Darknet(cfgfile)
|
|||
|
|
darknet_model.load_weights(weightfile)
|
|||
|
|
darknet_model = darknet_model.cuda()
|
|||
|
|
patch_applier = PatchApplier().cuda()
|
|||
|
|
patch_transformer = PatchTransformer().cuda()
|
|||
|
|
prob_extractor = MaxProbExtractor(0, 80).cuda()
|
|||
|
|
nms_calculator = NMSCalculator(printfile, patch_size)
|
|||
|
|
total_variation = TotalVariation()
|
|||
|
|
'''
|
|||
|
|
img = Image.open('data/horse.jpg').convert('RGB')
|
|||
|
|
img = img.resize((darknet_model.width, darknet_model.height))
|
|||
|
|
width = img.width
|
|||
|
|
height = img.height
|
|||
|
|
img = torch.ByteTensor(torch.ByteStorage.from_buffer(img.tobytes()))
|
|||
|
|
img = img.view(height, width, 3).transpose(0, 1).transpose(0, 2).contiguous()
|
|||
|
|
img = img.view(1, 3, height, width)
|
|||
|
|
img = img.float().div(255.0)
|
|||
|
|
img = torch.autograd.Variable(img)
|
|||
|
|
|
|||
|
|
output = darknet_model(img)
|
|||
|
|
'''
|
|||
|
|
optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
|
|||
|
|
|
|||
|
|
tl0 = time.time()
|
|||
|
|
tl1 = time.time()
|
|||
|
|
for i_batch, (img_batch, lab_batch) in enumerate(test_loader):
|
|||
|
|
tl1 = time.time()
|
|||
|
|
print('time to fetch items: ', tl1 - tl0)
|
|||
|
|
img_batch = img_batch.cuda()
|
|||
|
|
lab_batch = lab_batch.cuda()
|
|||
|
|
adv_patch = Image.open('data/horse.jpg').convert('RGB')
|
|||
|
|
adv_patch = adv_patch.resize((patch_size, patch_size))
|
|||
|
|
transform = transforms.ToTensor()
|
|||
|
|
adv_patch = transform(adv_patch).cuda()
|
|||
|
|
img_size = img_batch.size(-1)
|
|||
|
|
print('transforming patches')
|
|||
|
|
t0 = time.time()
|
|||
|
|
adv_batch_t = patch_transformer.forward(adv_patch, lab_batch, img_size)
|
|||
|
|
print('applying patches')
|
|||
|
|
t1 = time.time()
|
|||
|
|
img_batch = patch_applier.forward(img_batch, adv_batch_t)
|
|||
|
|
img_batch = torch.autograd.Variable(img_batch)
|
|||
|
|
img_batch = F.interpolate(img_batch, (darknet_model.height, darknet_model.width))
|
|||
|
|
print('running patched images through model')
|
|||
|
|
t2 = time.time()
|
|||
|
|
|
|||
|
|
for obj in gc.get_objects():
|
|||
|
|
try:
|
|||
|
|
if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)):
|
|||
|
|
try:
|
|||
|
|
print(type(obj), obj.size())
|
|||
|
|
except:
|
|||
|
|
pass
|
|||
|
|
except:
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
print(torch.cuda.memory_allocated())
|
|||
|
|
|
|||
|
|
output = darknet_model(img_batch)
|
|||
|
|
print('extracting max probs')
|
|||
|
|
t3 = time.time()
|
|||
|
|
max_prob = prob_extractor(output)
|
|||
|
|
t4 = time.time()
|
|||
|
|
nms = nms_calculator.forward(adv_patch)
|
|||
|
|
tv = total_variation(adv_patch)
|
|||
|
|
print('---------------------------------')
|
|||
|
|
print(' patch transformation : %f' % (t1 - t0))
|
|||
|
|
print(' patch application : %f' % (t2 - t1))
|
|||
|
|
print(' darknet forward : %f' % (t3 - t2))
|
|||
|
|
print(' probability extraction : %f' % (t4 - t3))
|
|||
|
|
print('---------------------------------')
|
|||
|
|
print(' total forward pass : %f' % (t4 - t0))
|
|||
|
|
del img_batch, lab_batch, adv_patch, adv_batch_t, output, max_prob
|
|||
|
|
torch.cuda.empty_cache()
|
|||
|
|
tl0 = time.time()
|