first
commit
4a2986bde5
|
|
@ -0,0 +1,140 @@
|
|||
# ignore map, miou, datasets
|
||||
map_out/
|
||||
miou_out/
|
||||
VOCdevkit/
|
||||
datasets/
|
||||
Medical_Datasets/
|
||||
lfw/
|
||||
logs/
|
||||
model_data/
|
||||
.temp_map_out/
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
|
@ -0,0 +1,161 @@
|
|||
import os
|
||||
import random
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import numpy as np
|
||||
|
||||
from utils.utils import get_classes
|
||||
|
||||
# --------------------------------------------------------------------------------------------------------------------------------#
|
||||
# annotation_mode用于指定该文件运行时计算的内容
|
||||
# annotation_mode为0代表整个标签处理过程，包括获得VOCdevkit/VOC2007/ImageSets里面的txt以及训练用的2007_train.txt、2007_val.txt
|
||||
# annotation_mode为1代表获得VOCdevkit/VOC2007/ImageSets里面的txt
|
||||
# annotation_mode为2代表获得训练用的2007_train.txt、2007_val.txt
|
||||
# --------------------------------------------------------------------------------------------------------------------------------#
|
||||
annotation_mode = 0
|
||||
# -------------------------------------------------------------------#
|
||||
# 必须要修改，用于生成2007_train.txt、2007_val.txt的目标信息
|
||||
# 与训练和预测所用的classes_path一致即可
|
||||
# 如果生成的2007_train.txt里面没有目标信息
|
||||
# 那么就是因为classes没有设定正确
|
||||
# 仅在annotation_mode为0和2的时候有效
|
||||
# -------------------------------------------------------------------#
|
||||
classes_path = 'model_data/voc_classes.txt'
|
||||
# --------------------------------------------------------------------------------------------------------------------------------#
|
||||
# trainval_percent用于指定(训练集+验证集)与测试集的比例，默认情况下 (训练集+验证集):测试集 = 9:1
|
||||
# train_percent用于指定(训练集+验证集)中训练集与验证集的比例，默认情况下 训练集:验证集 = 9:1
|
||||
# 仅在annotation_mode为0和1的时候有效
|
||||
# --------------------------------------------------------------------------------------------------------------------------------#
|
||||
trainval_percent = 0.9
|
||||
train_percent = 0.9
|
||||
# -------------------------------------------------------#
|
||||
# 指向VOC数据集所在的文件夹
|
||||
# 默认指向根目录下的VOC数据集
|
||||
# -------------------------------------------------------#
|
||||
VOCdevkit_path = 'VOCdevkit'
|
||||
|
||||
VOCdevkit_sets = [('2007', 'train'), ('2007', 'val')]
|
||||
classes, _ = get_classes(classes_path)
|
||||
|
||||
# -------------------------------------------------------#
|
||||
# 统计目标数量
|
||||
# -------------------------------------------------------#
|
||||
photo_nums = np.zeros(len(VOCdevkit_sets)) # 生成train的数目，val的数目
|
||||
nums = np.zeros(len(classes))
|
||||
|
||||
|
||||
def convert_annotation(year, image_id, list_file):
|
||||
in_file = open(os.path.join(VOCdevkit_path, 'VOC%s/Annotations/%s.xml' % (year, image_id)), encoding='utf-8')
|
||||
tree = ET.parse(in_file)
|
||||
root = tree.getroot()
|
||||
|
||||
for obj in root.iter('object'):
|
||||
difficult = 0
|
||||
if obj.find('difficult') != None:
|
||||
difficult = obj.find('difficult').text
|
||||
cls = obj.find('name').text
|
||||
if cls not in classes or int(difficult) == 1:
|
||||
continue
|
||||
cls_id = classes.index(cls)
|
||||
xmlbox = obj.find('bndbox')
|
||||
b = (int(float(xmlbox.find('xmin').text)), int(float(xmlbox.find('ymin').text)),
|
||||
int(float(xmlbox.find('xmax').text)), int(float(xmlbox.find('ymax').text)))
|
||||
list_file.write(" " + ",".join([str(a) for a in b]) + ',' + str(cls_id))
|
||||
|
||||
nums[classes.index(cls)] = nums[classes.index(cls)] + 1 # 统计各个类别的个数
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
random.seed(0)
|
||||
if " " in os.path.abspath(VOCdevkit_path):
|
||||
raise ValueError("数据集存放的文件夹路径与图片名称中不可以存在空格，否则会影响正常的模型训练，请注意修改。")
|
||||
|
||||
if annotation_mode == 0 or annotation_mode == 1:
|
||||
print("Generate txt in ImageSets.")
|
||||
xmlfilepath = os.path.join(VOCdevkit_path, 'VOC2007/Annotations')
|
||||
saveBasePath = os.path.join(VOCdevkit_path, 'VOC2007/ImageSets')
|
||||
temp_xml = os.listdir(xmlfilepath)
|
||||
total_xml = []
|
||||
for xml in temp_xml:
|
||||
if xml.endswith(".xml"):
|
||||
total_xml.append(xml)
|
||||
|
||||
num = len(total_xml)
|
||||
list = range(num)
|
||||
tv = int(num * trainval_percent) # 训练、验证集 总数
|
||||
tr = int(tv * train_percent) # 训练、验证集中 训练集的总数
|
||||
trainval = random.sample(list, tv) # 在总数里采样
|
||||
train = random.sample(trainval, tr) # 在tv中采样tr
|
||||
|
||||
print("train and val size", tv)
|
||||
print("train size", tr)
|
||||
ftrainval = open(os.path.join(saveBasePath, 'trainval.txt'), 'w')
|
||||
ftest = open(os.path.join(saveBasePath, 'test.txt'), 'w')
|
||||
ftrain = open(os.path.join(saveBasePath, 'train.txt'), 'w')
|
||||
fval = open(os.path.join(saveBasePath, 'val.txt'), 'w')
|
||||
|
||||
for i in list:
|
||||
name = total_xml[i][:-4] + '\n'
|
||||
if i in trainval:
|
||||
ftrainval.write(name)
|
||||
if i in train:
|
||||
ftrain.write(name)
|
||||
else:
|
||||
fval.write(name)
|
||||
else:
|
||||
ftest.write(name)
|
||||
|
||||
ftrainval.close()
|
||||
ftrain.close()
|
||||
fval.close()
|
||||
ftest.close()
|
||||
print("Generate txt in ImageSets done.")
|
||||
|
||||
if annotation_mode == 0 or annotation_mode == 2:
|
||||
print("Generate 2007_train.txt and 2007_val.txt for train.")
|
||||
type_index = 0
|
||||
for year, image_set in VOCdevkit_sets:
|
||||
image_ids = open(os.path.join(VOCdevkit_path, 'VOC%s/ImageSets/Main/%s.txt' % (year, image_set)),
|
||||
encoding='utf-8').read().strip().split()
|
||||
list_file = open('%s_%s.txt' % (year, image_set), 'w', encoding='utf-8')
|
||||
for image_id in image_ids:
|
||||
list_file.write(
|
||||
'%s/VOC%s/JPEGImages/%s.jpg' % (os.path.abspath(VOCdevkit_path), year, image_id)) # 文件名字是拼出来的
|
||||
|
||||
convert_annotation(year, image_id, list_file)
|
||||
list_file.write('\n')
|
||||
photo_nums[type_index] = len(image_ids)
|
||||
type_index += 1
|
||||
list_file.close()
|
||||
print("Generate 2007_train.txt and 2007_val.txt for train done.")
|
||||
|
||||
|
||||
def printTable(List1, List2):
|
||||
for i in range(len(List1[0])):
|
||||
print("|", end=' ')
|
||||
for j in range(len(List1)):
|
||||
print(List1[j][i].rjust(int(List2[j])), end=' ')
|
||||
print("|", end=' ')
|
||||
print()
|
||||
|
||||
|
||||
str_nums = [str(int(x)) for x in nums]
|
||||
tableData = [
|
||||
classes, str_nums
|
||||
]
|
||||
colWidths = [0] * len(tableData)
|
||||
len1 = 0
|
||||
for i in range(len(tableData)):
|
||||
for j in range(len(tableData[i])):
|
||||
if len(tableData[i][j]) > colWidths[i]:
|
||||
colWidths[i] = len(tableData[i][j])
|
||||
printTable(tableData, colWidths)
|
||||
|
||||
if photo_nums[0] <= 500:
|
||||
print("训练集数量小于500，属于较小的数据量，请注意设置较大的训练世代（Epoch）以满足足够的梯度下降次数（Step）。")
|
||||
|
||||
if np.sum(nums) == 0:
|
||||
print("在数据集中并未获得任何目标，请注意修改classes_path对应自己的数据集，并且保证标签名字正确，否则训练将会没有任何效果！")
|
||||
print("在数据集中并未获得任何目标，请注意修改classes_path对应自己的数据集，并且保证标签名字正确，否则训练将会没有任何效果！")
|
||||
print("在数据集中并未获得任何目标，请注意修改classes_path对应自己的数据集，并且保证标签名字正确，否则训练将会没有任何效果！")
|
||||
print("（重要的事情说三遍）。")
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2020 JiaQi Xu
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
from torch import optim
|
||||
|
||||
|
||||
class BaseConfig(object):
|
||||
"""
|
||||
Default parameters for all config files.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Set the defaults.
|
||||
"""
|
||||
self.img_dir = "inria/Train/pos"
|
||||
self.lab_dir = "inria/Train/pos/yolo-labels"
|
||||
self.cfgfile = "cfg/yolo.cfg"
|
||||
self.weightfile = "weights/yolo.weights"
|
||||
self.printfile = "non_printability/30values.txt"
|
||||
self.patch_size = 300
|
||||
|
||||
self.start_learning_rate = 0.03
|
||||
|
||||
self.patch_name = 'base'
|
||||
|
||||
self.scheduler_factory = lambda x: optim.lr_scheduler.ReduceLROnPlateau(x, 'min', patience=50)
|
||||
self.max_tv = 0
|
||||
|
||||
self.batch_size = 20
|
||||
|
||||
self.loss_target = lambda obj, cls: obj * cls
|
||||
|
||||
|
||||
class ReproducePaperObj(BaseConfig):
|
||||
"""
|
||||
Reproduce the results from the paper: Generate a patch that minimises object score.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.batch_size = 8
|
||||
self.patch_size = 300
|
||||
|
||||
self.patch_name = 'ObjectOnlyPaper'
|
||||
self.max_tv = 0.165
|
||||
|
||||
self.loss_target = lambda obj, cls: obj
|
||||
|
|
@ -0,0 +1,138 @@
|
|||
import os
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from PIL import Image
|
||||
from tqdm import tqdm
|
||||
|
||||
from utils.utils import get_classes
|
||||
from utils.utils_map import get_coco_map, get_map
|
||||
from yolo import YOLO
|
||||
|
||||
if __name__ == "__main__":
|
||||
'''
|
||||
Recall和Precision不像AP是一个面积的概念，因此在门限值（Confidence）不同时，网络的Recall和Precision值是不同的。
|
||||
默认情况下，本代码计算的Recall和Precision代表的是当门限值（Confidence）为0.5时，所对应的Recall和Precision值。
|
||||
|
||||
受到mAP计算原理的限制，网络在计算mAP时需要获得近乎所有的预测框，这样才可以计算不同门限条件下的Recall和Precision值
|
||||
因此，本代码获得的map_out/detection-results/里面的txt的框的数量一般会比直接predict多一些，目的是列出所有可能的预测框，
|
||||
'''
|
||||
# ------------------------------------------------------------------------------------------------------------------#
|
||||
# map_mode用于指定该文件运行时计算的内容
|
||||
# map_mode为0代表整个map计算流程，包括获得预测结果、获得真实框、计算VOC_map。
|
||||
# map_mode为1代表仅仅获得预测结果。
|
||||
# map_mode为2代表仅仅获得真实框。
|
||||
# map_mode为3代表仅仅计算VOC_map。
|
||||
# map_mode为4代表利用COCO工具箱计算当前数据集的0.50:0.95map。需要获得预测结果、获得真实框后并安装pycocotools才行
|
||||
# -------------------------------------------------------------------------------------------------------------------#
|
||||
map_mode = 0
|
||||
# --------------------------------------------------------------------------------------#
|
||||
# 此处的classes_path用于指定需要测量VOC_map的类别
|
||||
# 一般情况下与训练和预测所用的classes_path一致即可
|
||||
# --------------------------------------------------------------------------------------#
|
||||
classes_path = 'model_data/voc_classes.txt'
|
||||
# --------------------------------------------------------------------------------------#
|
||||
# MINOVERLAP用于指定想要获得的mAP0.x，mAP0.x的意义是什么请同学们百度一下。
|
||||
# 比如计算mAP0.75，可以设定MINOVERLAP = 0.75。
|
||||
#
|
||||
# 当某一预测框与真实框重合度大于MINOVERLAP时，该预测框被认为是正样本，否则为负样本。
|
||||
# 因此MINOVERLAP的值越大，预测框要预测的越准确才能被认为是正样本，此时算出来的mAP值越低，
|
||||
# --------------------------------------------------------------------------------------#
|
||||
MINOVERLAP = 0.5
|
||||
# --------------------------------------------------------------------------------------#
|
||||
# 受到mAP计算原理的限制，网络在计算mAP时需要获得近乎所有的预测框，这样才可以计算mAP
|
||||
# 因此，confidence的值应当设置的尽量小进而获得全部可能的预测框。
|
||||
#
|
||||
# 该值一般不调整。因为计算mAP需要获得近乎所有的预测框，此处的confidence不能随便更改。
|
||||
# 想要获得不同门限值下的Recall和Precision值，请修改下方的score_threhold。
|
||||
# --------------------------------------------------------------------------------------#
|
||||
confidence = 0.001
|
||||
# --------------------------------------------------------------------------------------#
|
||||
# 预测时使用到的非极大抑制值的大小，越大表示非极大抑制越不严格。
|
||||
#
|
||||
# 该值一般不调整。
|
||||
# --------------------------------------------------------------------------------------#
|
||||
nms_iou = 0.5
|
||||
# ---------------------------------------------------------------------------------------------------------------#
|
||||
# Recall和Precision不像AP是一个面积的概念，因此在门限值不同时，网络的Recall和Precision值是不同的。
|
||||
#
|
||||
# 默认情况下，本代码计算的Recall和Precision代表的是当门限值为0.5（此处定义为score_threhold）时所对应的Recall和Precision值。
|
||||
# 因为计算mAP需要获得近乎所有的预测框，上面定义的confidence不能随便更改。
|
||||
# 这里专门定义一个score_threhold用于代表门限值，进而在计算mAP时找到门限值对应的Recall和Precision值。
|
||||
# ---------------------------------------------------------------------------------------------------------------#
|
||||
score_threhold = 0.5
|
||||
# -------------------------------------------------------#
|
||||
# map_vis用于指定是否开启VOC_map计算的可视化
|
||||
# -------------------------------------------------------#
|
||||
map_vis = False
|
||||
# -------------------------------------------------------#
|
||||
# 指向VOC数据集所在的文件夹
|
||||
# 默认指向根目录下的VOC数据集
|
||||
# -------------------------------------------------------#
|
||||
VOCdevkit_path = 'VOCdevkit'
|
||||
# -------------------------------------------------------#
|
||||
# 结果输出的文件夹，默认为map_out
|
||||
# -------------------------------------------------------#
|
||||
map_out_path = 'map_out'
|
||||
|
||||
image_ids = open(os.path.join(VOCdevkit_path, "VOC2007/ImageSets/Main/test.txt")).read().strip().split()
|
||||
|
||||
if not os.path.exists(map_out_path):
|
||||
os.makedirs(map_out_path)
|
||||
if not os.path.exists(os.path.join(map_out_path, 'ground-truth')):
|
||||
os.makedirs(os.path.join(map_out_path, 'ground-truth'))
|
||||
if not os.path.exists(os.path.join(map_out_path, 'detection-results')):
|
||||
os.makedirs(os.path.join(map_out_path, 'detection-results'))
|
||||
if not os.path.exists(os.path.join(map_out_path, 'images-optional')):
|
||||
os.makedirs(os.path.join(map_out_path, 'images-optional'))
|
||||
|
||||
class_names, _ = get_classes(classes_path)
|
||||
|
||||
if map_mode == 0 or map_mode == 1:
|
||||
print("Load model.")
|
||||
yolo = YOLO(confidence=confidence, nms_iou=nms_iou)
|
||||
print("Load model done.")
|
||||
|
||||
print("Get predict result.")
|
||||
for image_id in tqdm(image_ids):
|
||||
image_path = os.path.join(VOCdevkit_path, "VOC2007/JPEGImages/" + image_id + ".jpg")
|
||||
image = Image.open(image_path)
|
||||
if map_vis:
|
||||
image.save(os.path.join(map_out_path, "images-optional/" + image_id + ".jpg"))
|
||||
yolo.get_map_txt(image_id, image, class_names, map_out_path)
|
||||
print("Get predict result done.")
|
||||
|
||||
if map_mode == 0 or map_mode == 2:
|
||||
print("Get ground truth result.")
|
||||
for image_id in tqdm(image_ids):
|
||||
with open(os.path.join(map_out_path, "ground-truth/" + image_id + ".txt"), "w") as new_f:
|
||||
root = ET.parse(os.path.join(VOCdevkit_path, "VOC2007/Annotations/" + image_id + ".xml")).getroot()
|
||||
for obj in root.findall('object'):
|
||||
difficult_flag = False
|
||||
if obj.find('difficult') != None:
|
||||
difficult = obj.find('difficult').text
|
||||
if int(difficult) == 1:
|
||||
difficult_flag = True
|
||||
obj_name = obj.find('name').text
|
||||
if obj_name not in class_names:
|
||||
continue
|
||||
bndbox = obj.find('bndbox')
|
||||
left = bndbox.find('xmin').text
|
||||
top = bndbox.find('ymin').text
|
||||
right = bndbox.find('xmax').text
|
||||
bottom = bndbox.find('ymax').text
|
||||
|
||||
if difficult_flag:
|
||||
new_f.write("%s %s %s %s %s difficult\n" % (obj_name, left, top, right, bottom))
|
||||
else:
|
||||
new_f.write("%s %s %s %s %s\n" % (obj_name, left, top, right, bottom))
|
||||
print("Get ground truth result done.")
|
||||
|
||||
if map_mode == 0 or map_mode == 3:
|
||||
print("Get map.")
|
||||
get_map(MINOVERLAP, True, score_threhold=score_threhold, path=map_out_path)
|
||||
print("Get map done.")
|
||||
|
||||
if map_mode == 4:
|
||||
print("Get map.")
|
||||
get_coco_map(class_names=class_names, path=map_out_path)
|
||||
print("Get map done.")
|
||||
|
|
@ -0,0 +1,167 @@
|
|||
# -------------------------------------------------------------------------------------------------------#
|
||||
# kmeans虽然会对数据集中的框进行聚类，但是很多数据集由于框的大小相近，聚类出来的9个框相差不大，
|
||||
# 这样的框反而不利于模型的训练。因为不同的特征层适合不同大小的先验框，shape越小的特征层适合越大的先验框
|
||||
# 原始网络的先验框已经按大中小比例分配好了，不进行聚类也会有非常好的效果。
|
||||
# -------------------------------------------------------------------------------------------------------#
|
||||
import glob
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
def cas_iou(box, cluster):
|
||||
x = np.minimum(cluster[:, 0], box[0])
|
||||
y = np.minimum(cluster[:, 1], box[1])
|
||||
|
||||
intersection = x * y
|
||||
area1 = box[0] * box[1]
|
||||
|
||||
area2 = cluster[:, 0] * cluster[:, 1]
|
||||
iou = intersection / (area1 + area2 - intersection)
|
||||
|
||||
return iou
|
||||
|
||||
|
||||
def avg_iou(box, cluster):
|
||||
return np.mean([np.max(cas_iou(box[i], cluster)) for i in range(box.shape[0])])
|
||||
|
||||
|
||||
def kmeans(box, k):
|
||||
# -------------------------------------------------------------#
|
||||
# 取出一共有多少框
|
||||
# -------------------------------------------------------------#
|
||||
row = box.shape[0]
|
||||
|
||||
# -------------------------------------------------------------#
|
||||
# 每个框各个点的位置
|
||||
# -------------------------------------------------------------#
|
||||
distance = np.empty((row, k))
|
||||
|
||||
# -------------------------------------------------------------#
|
||||
# 最后的聚类位置
|
||||
# -------------------------------------------------------------#
|
||||
last_clu = np.zeros((row,))
|
||||
|
||||
np.random.seed()
|
||||
|
||||
# -------------------------------------------------------------#
|
||||
# 随机选5个当聚类中心
|
||||
# -------------------------------------------------------------#
|
||||
cluster = box[np.random.choice(row, k, replace=False)]
|
||||
|
||||
iter = 0
|
||||
while True:
|
||||
# -------------------------------------------------------------#
|
||||
# 计算当前框和先验框的宽高比例
|
||||
# -------------------------------------------------------------#
|
||||
for i in range(row):
|
||||
distance[i] = 1 - cas_iou(box[i], cluster)
|
||||
|
||||
# -------------------------------------------------------------#
|
||||
# 取出最小点
|
||||
# -------------------------------------------------------------#
|
||||
near = np.argmin(distance, axis=1)
|
||||
|
||||
if (last_clu == near).all():
|
||||
break
|
||||
|
||||
# -------------------------------------------------------------#
|
||||
# 求每一个类的中位点
|
||||
# -------------------------------------------------------------#
|
||||
for j in range(k):
|
||||
cluster[j] = np.median(
|
||||
box[near == j], axis=0)
|
||||
|
||||
last_clu = near
|
||||
if iter % 5 == 0:
|
||||
print('iter: {:d}. avg_iou:{:.2f}'.format(iter, avg_iou(box, cluster)))
|
||||
iter += 1
|
||||
|
||||
return cluster, near
|
||||
|
||||
|
||||
def load_data(path):
|
||||
data = []
|
||||
# -------------------------------------------------------------#
|
||||
# 对于每一个xml都寻找box
|
||||
# -------------------------------------------------------------#
|
||||
for xml_file in tqdm(glob.glob('{}/*xml'.format(path))):
|
||||
tree = ET.parse(xml_file)
|
||||
height = int(tree.findtext('./size/height'))
|
||||
width = int(tree.findtext('./size/width'))
|
||||
if height <= 0 or width <= 0:
|
||||
continue
|
||||
|
||||
# -------------------------------------------------------------#
|
||||
# 对于每一个目标都获得它的宽高
|
||||
# -------------------------------------------------------------#
|
||||
for obj in tree.iter('object'):
|
||||
xmin = int(float(obj.findtext('bndbox/xmin'))) / width
|
||||
ymin = int(float(obj.findtext('bndbox/ymin'))) / height
|
||||
xmax = int(float(obj.findtext('bndbox/xmax'))) / width
|
||||
ymax = int(float(obj.findtext('bndbox/ymax'))) / height
|
||||
|
||||
xmin = np.float64(xmin)
|
||||
ymin = np.float64(ymin)
|
||||
xmax = np.float64(xmax)
|
||||
ymax = np.float64(ymax)
|
||||
# 得到宽高
|
||||
data.append([xmax - xmin, ymax - ymin])
|
||||
return np.array(data)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
np.random.seed(0)
|
||||
# -------------------------------------------------------------#
|
||||
# 运行该程序会计算'./VOCdevkit/VOC2007/Annotations'的xml
|
||||
# 会生成yolo_anchors.txt
|
||||
# -------------------------------------------------------------#
|
||||
input_shape = [416, 416]
|
||||
anchors_num = 9
|
||||
# -------------------------------------------------------------#
|
||||
# 载入数据集，可以使用VOC的xml
|
||||
# -------------------------------------------------------------#
|
||||
path = 'VOCdevkit/VOC2007/Annotations'
|
||||
|
||||
# -------------------------------------------------------------#
|
||||
# 载入所有的xml
|
||||
# 存储格式为转化为比例后的width,height
|
||||
# -------------------------------------------------------------#
|
||||
print('Load xmls.')
|
||||
data = load_data(path)
|
||||
print('Load xmls done.')
|
||||
|
||||
# -------------------------------------------------------------#
|
||||
# 使用k聚类算法
|
||||
# -------------------------------------------------------------#
|
||||
print('K-means boxes.')
|
||||
cluster, near = kmeans(data, anchors_num)
|
||||
print('K-means boxes done.')
|
||||
data = data * np.array([input_shape[1], input_shape[0]])
|
||||
cluster = cluster * np.array([input_shape[1], input_shape[0]])
|
||||
|
||||
# -------------------------------------------------------------#
|
||||
# 绘图
|
||||
# -------------------------------------------------------------#
|
||||
for j in range(anchors_num):
|
||||
plt.scatter(data[near == j][:, 0], data[near == j][:, 1])
|
||||
plt.scatter(cluster[j][0], cluster[j][1], marker='x', c='black')
|
||||
plt.savefig("kmeans_for_anchors.jpg")
|
||||
plt.show()
|
||||
print('Save kmeans_for_anchors.jpg in root dir.')
|
||||
|
||||
cluster = cluster[np.argsort(cluster[:, 0] * cluster[:, 1])]
|
||||
print('avg_ratio:{:.2f}'.format(avg_iou(data, cluster)))
|
||||
print(cluster)
|
||||
|
||||
f = open("yolo_anchors.txt", 'w')
|
||||
row = np.shape(cluster)[0]
|
||||
for i in range(row):
|
||||
if i == 0:
|
||||
x_y = "%d,%d" % (cluster[i][0], cluster[i][1])
|
||||
else:
|
||||
x_y = ", %d,%d" % (cluster[i][0], cluster[i][1])
|
||||
f.write(x_y)
|
||||
f.close()
|
||||
|
|
@ -0,0 +1,531 @@
|
|||
import fnmatch
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from operator import itemgetter
|
||||
|
||||
import gc
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.optim as optim
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from PIL import Image
|
||||
from torch.utils.data import Dataset
|
||||
from torchvision import transforms
|
||||
|
||||
# from darknet import Darknet
|
||||
|
||||
from median_pool import MedianPool2d
|
||||
|
||||
# print('starting test read')
|
||||
# im = Image.open('data/horse.jpg').convert('RGB')
|
||||
# print('img read!')
|
||||
|
||||
|
||||
class MaxProbExtractor(nn.Module):
|
||||
"""MaxProbExtractor: extracts max class probability for class from YOLO output.
|
||||
|
||||
Module providing the functionality necessary to extract the max class probability for one class from YOLO output.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, cls_id, num_cls, config):
|
||||
super(MaxProbExtractor, self).__init__()
|
||||
self.cls_id = cls_id
|
||||
self.num_cls = num_cls
|
||||
self.config = config
|
||||
self.anchor_num = 3
|
||||
|
||||
def forward(self, YOLOoutput):
|
||||
# get values neccesary for transformation
|
||||
if YOLOoutput.dim() == 3:
|
||||
YOLOoutput = YOLOoutput.unsqueeze(0)
|
||||
batch = YOLOoutput.size(0)
|
||||
assert (YOLOoutput.size(1) == (5 + self.num_cls) * self.anchor_num)
|
||||
h = YOLOoutput.size(2)
|
||||
w = YOLOoutput.size(3)
|
||||
# transform the output tensor from [batch, 425, 19, 19] to [batch, 80, 1805]
|
||||
output = YOLOoutput.view(batch, self.anchor_num, 5 + self.num_cls, h * w) # [batch, 5, 85, 361]
|
||||
output = output.transpose(1, 2).contiguous() # [batch, 85, 5, 361]
|
||||
output = output.view(batch, 5 + self.num_cls, self.anchor_num * h * w) # [batch, 85, 1805]
|
||||
output_objectness = torch.sigmoid(output[:, 4, :]) # [batch, 1805] # 是否有物体
|
||||
output = output[:, 5:5 + self.num_cls, :] # [batch, 80, 1805]
|
||||
# perform softmax to normalize probabilities for object classes to [0,1]
|
||||
normal_confs = torch.nn.Softmax(dim=1)(output) # 物体类别
|
||||
# we only care for probabilities of the class of interest (person)
|
||||
confs_for_class = normal_confs[:, self.cls_id, :] # 类别 序号对应的为人
|
||||
confs_if_object = output_objectness # confs_for_class * output_objectness
|
||||
confs_if_object = confs_for_class * output_objectness
|
||||
confs_if_object = self.config.loss_target(output_objectness, confs_for_class)
|
||||
# find the max probability for person
|
||||
max_conf, max_conf_idx = torch.max(confs_if_object, dim=1)
|
||||
|
||||
return max_conf
|
||||
|
||||
|
||||
class NPSCalculator(nn.Module):
|
||||
"""NMSCalculator: calculates the non-printability score of a patch.
|
||||
|
||||
Module providing the functionality necessary to calculate the non-printability score (NMS) of an adversarial patch.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, printability_file, patch_side):
|
||||
super(NPSCalculator, self).__init__()
|
||||
self.printability_array = nn.Parameter(self.get_printability_array(printability_file, patch_side),
|
||||
requires_grad=False)
|
||||
|
||||
def forward(self, adv_patch):
|
||||
# calculate euclidian distance between colors in patch and colors in printability_array
|
||||
# square root of sum of squared difference
|
||||
color_dist = (adv_patch - self.printability_array + 0.000001)
|
||||
color_dist = color_dist ** 2
|
||||
color_dist = torch.sum(color_dist, 1) + 0.000001
|
||||
color_dist = torch.sqrt(color_dist)
|
||||
# only work with the min distance
|
||||
color_dist_prod = torch.min(color_dist, 0)[0] # test: change prod for min (find distance to closest color)
|
||||
# calculate the nps by summing over all pixels
|
||||
nps_score = torch.sum(color_dist_prod, 0)
|
||||
nps_score = torch.sum(nps_score, 0)
|
||||
return nps_score / torch.numel(adv_patch)
|
||||
|
||||
def get_printability_array(self, printability_file, side):
|
||||
printability_list = []
|
||||
|
||||
# read in printability triplets and put them in a list
|
||||
with open(printability_file) as f:
|
||||
for line in f:
|
||||
printability_list.append(line.split(","))
|
||||
|
||||
printability_array = []
|
||||
for printability_triplet in printability_list:
|
||||
printability_imgs = []
|
||||
red, green, blue = printability_triplet
|
||||
printability_imgs.append(np.full((side, side), red))
|
||||
printability_imgs.append(np.full((side, side), green))
|
||||
printability_imgs.append(np.full((side, side), blue))
|
||||
printability_array.append(printability_imgs)
|
||||
|
||||
printability_array = np.asarray(printability_array)
|
||||
printability_array = np.float32(printability_array)
|
||||
pa = torch.from_numpy(printability_array)
|
||||
return pa
|
||||
|
||||
|
||||
class TotalVariation(nn.Module):
|
||||
"""TotalVariation: calculates the total variation of a patch.
|
||||
|
||||
Module providing the functionality necessary to calculate the total Variation (TV) of an adversarial patch.
|
||||
|
||||
TotalVariation：计算补丁的总变化。
|
||||
该模块提供了计算对抗性补丁的总变化 (TV) 所需的功能。
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super(TotalVariation, self).__init__()
|
||||
|
||||
def forward(self, adv_patch):
|
||||
# bereken de total variation van de adv_patch
|
||||
tvcomp1 = torch.sum(torch.abs(adv_patch[:, :, 1:] - adv_patch[:, :, :-1] + 0.000001), 0)
|
||||
tvcomp1 = torch.sum(torch.sum(tvcomp1, 0), 0)
|
||||
tvcomp2 = torch.sum(torch.abs(adv_patch[:, 1:, :] - adv_patch[:, :-1, :] + 0.000001), 0)
|
||||
tvcomp2 = torch.sum(torch.sum(tvcomp2, 0), 0)
|
||||
tv = tvcomp1 + tvcomp2
|
||||
return tv / torch.numel(adv_patch)
|
||||
|
||||
|
||||
class PatchTransformer(nn.Module):
|
||||
"""PatchTransformer: transforms batch of patches
|
||||
|
||||
Module providing the functionality necessary to transform a batch of patches, randomly adjusting brightness and
|
||||
contrast, adding random amount of noise, and rotating randomly. Resizes-patches according to as size based on the
|
||||
batch of labels, and pads them to the dimension of an image.
|
||||
|
||||
变换一批补丁，随机调整亮度和对比度，添加随机数量的噪声，随机旋转。 根据标签批次的大小调整补丁大小，并将它们填充到图像的尺寸中。
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super(PatchTransformer, self).__init__()
|
||||
self.min_contrast = 0.8
|
||||
self.max_contrast = 1.2
|
||||
self.min_brightness = -0.1
|
||||
self.max_brightness = 0.1
|
||||
self.noise_factor = 0.10
|
||||
self.minangle = -20 / 180 * math.pi
|
||||
self.maxangle = 20 / 180 * math.pi
|
||||
self.medianpooler = MedianPool2d(7, same=True) # 中值池化
|
||||
'''
|
||||
kernel = torch.cuda.FloatTensor([[0.003765, 0.015019, 0.023792, 0.015019, 0.003765],
|
||||
[0.015019, 0.059912, 0.094907, 0.059912, 0.015019],
|
||||
[0.023792, 0.094907, 0.150342, 0.094907, 0.023792],
|
||||
[0.015019, 0.059912, 0.094907, 0.059912, 0.015019],
|
||||
[0.003765, 0.015019, 0.023792, 0.015019, 0.003765]])
|
||||
self.kernel = kernel.unsqueeze(0).unsqueeze(0).expand(3,3,-1,-1)
|
||||
'''
|
||||
|
||||
def forward(self, adv_patch, lab_batch, img_size, do_rotate=True, rand_loc=True):
|
||||
# adv_patch = F.conv2d(adv_patch.unsqueeze(0),self.kernel,padding=(2,2))
|
||||
adv_patch = self.medianpooler(adv_patch.unsqueeze(0))
|
||||
# Determine size of padding
|
||||
pad = (img_size - adv_patch.size(-1)) / 2
|
||||
# Make a batch of patches
|
||||
adv_patch = adv_patch.unsqueeze(0) # .unsqueeze(0) # 这里又扩大一维，变成5维 1, 1, 3, 300, 300
|
||||
adv_batch = adv_patch.expand(lab_batch.size(0), lab_batch.size(1), -1, -1, -1) # adv_batch ！！ 不是adv_patch！！ 8, 14, 3, 300, 300
|
||||
batch_size = torch.Size((lab_batch.size(0), lab_batch.size(1))) # 8, 14
|
||||
|
||||
# Contrast, brightness and noise transforms
|
||||
|
||||
# Create random contrast tensor
|
||||
contrast = torch.cuda.FloatTensor(batch_size).uniform_(self.min_contrast, self.max_contrast)
|
||||
contrast = contrast.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1)
|
||||
contrast = contrast.expand(-1, -1, adv_batch.size(-3), adv_batch.size(-2), adv_batch.size(-1))
|
||||
contrast = contrast.cuda()
|
||||
|
||||
# Create random brightness tensor
|
||||
brightness = torch.cuda.FloatTensor(batch_size).uniform_(self.min_brightness, self.max_brightness)
|
||||
brightness = brightness.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1)
|
||||
brightness = brightness.expand(-1, -1, adv_batch.size(-3), adv_batch.size(-2), adv_batch.size(-1))
|
||||
brightness = brightness.cuda()
|
||||
|
||||
# Create random noise tensor
|
||||
noise = torch.cuda.FloatTensor(adv_batch.size()).uniform_(-1, 1) * self.noise_factor
|
||||
|
||||
# Apply contrast/brightness/noise, clamp
|
||||
adv_batch = adv_batch * contrast + brightness + noise
|
||||
|
||||
adv_batch = torch.clamp(adv_batch, 0.000001, 0.99999) # 限制到0到1之间
|
||||
|
||||
# Where the label class_id is 1 we don't want a patch (padding) --> fill mask with zero's
|
||||
cls_ids = torch.narrow(lab_batch, 2, 0, 1) # torch.narrow(input,dim,start,length) 从dim开始，返回共享内存的数据start到start+length-1
|
||||
cls_mask = cls_ids.expand(-1, -1, 3) # 接上，这里取出 lab_batch的代表id那列，相当于现在的lab_batch[..., 0]
|
||||
cls_mask = cls_mask.unsqueeze(-1)
|
||||
cls_mask = cls_mask.expand(-1, -1, -1, adv_batch.size(3))
|
||||
cls_mask = cls_mask.unsqueeze(-1)
|
||||
cls_mask = cls_mask.expand(-1, -1, -1, -1, adv_batch.size(4)) # cls_mask 的大小是 8, 14, 3, 300, 300 数据是类别
|
||||
msk_batch = torch.cuda.FloatTensor(cls_mask.size()).fill_(1) - cls_mask # 这里取出有人所对应的msk
|
||||
|
||||
# Pad patch and mask to image dimensions
|
||||
mypad = nn.ConstantPad2d((int(pad + 0.5), int(pad), int(pad + 0.5), int(pad)), 0) # (padding_left、padding_right、padding_top、padding_bottom) 填充0
|
||||
adv_batch = mypad(adv_batch) # 用0填充到416
|
||||
msk_batch = mypad(msk_batch)
|
||||
|
||||
# Rotation and rescaling transforms
|
||||
anglesize = (lab_batch.size(0) * lab_batch.size(1)) # 这里是旋转的数量
|
||||
if do_rotate:
|
||||
angle = torch.cuda.FloatTensor(anglesize).uniform_(self.minangle, self.maxangle)
|
||||
else:
|
||||
angle = torch.cuda.FloatTensor(anglesize).fill_(0)
|
||||
|
||||
# Resizes and rotates
|
||||
current_patch_size = adv_patch.size(-1)
|
||||
lab_batch_scaled = torch.cuda.FloatTensor(lab_batch.size()).fill_(0) # lab_batch_scaled是在原图上的尺寸？
|
||||
lab_batch_scaled[:, :, 1] = lab_batch[:, :, 1] * img_size
|
||||
lab_batch_scaled[:, :, 2] = lab_batch[:, :, 2] * img_size
|
||||
lab_batch_scaled[:, :, 3] = lab_batch[:, :, 3] * img_size
|
||||
lab_batch_scaled[:, :, 4] = lab_batch[:, :, 4] * img_size
|
||||
target_size = torch.sqrt(
|
||||
((lab_batch_scaled[:, :, 3].mul(0.2)) ** 2) + ((lab_batch_scaled[:, :, 4].mul(0.2)) ** 2))
|
||||
target_x = lab_batch[:, :, 1].view(np.prod(batch_size))
|
||||
target_y = lab_batch[:, :, 2].view(np.prod(batch_size))
|
||||
targetoff_x = lab_batch[:, :, 3].view(np.prod(batch_size))
|
||||
targetoff_y = lab_batch[:, :, 4].view(np.prod(batch_size))
|
||||
if (rand_loc):
|
||||
off_x = targetoff_x * (torch.cuda.FloatTensor(targetoff_x.size()).uniform_(-0.4, 0.4))
|
||||
target_x = target_x + off_x
|
||||
off_y = targetoff_y * (torch.cuda.FloatTensor(targetoff_y.size()).uniform_(-0.4, 0.4))
|
||||
target_y = target_y + off_y
|
||||
target_y = target_y - 0.05
|
||||
scale = target_size / current_patch_size # 原图相对于补丁大小的缩放因子？
|
||||
scale = scale.view(anglesize)
|
||||
|
||||
s = adv_batch.size()
|
||||
adv_batch = adv_batch.view(s[0] * s[1], s[2], s[3], s[4])
|
||||
msk_batch = msk_batch.view(s[0] * s[1], s[2], s[3], s[4])
|
||||
|
||||
tx = (-target_x + 0.5) * 2
|
||||
ty = (-target_y + 0.5) * 2
|
||||
sin = torch.sin(angle)
|
||||
cos = torch.cos(angle)
|
||||
|
||||
# Theta = rotation,rescale matrix
|
||||
theta = torch.cuda.FloatTensor(anglesize, 2, 3).fill_(0)
|
||||
theta[:, 0, 0] = cos / scale
|
||||
theta[:, 0, 1] = sin / scale
|
||||
theta[:, 0, 2] = tx * cos / scale + ty * sin / scale
|
||||
theta[:, 1, 0] = -sin / scale
|
||||
theta[:, 1, 1] = cos / scale
|
||||
theta[:, 1, 2] = -tx * sin / scale + ty * cos / scale
|
||||
|
||||
b_sh = adv_batch.shape
|
||||
grid = F.affine_grid(theta, adv_batch.shape)
|
||||
|
||||
adv_batch_t = F.grid_sample(adv_batch, grid)
|
||||
msk_batch_t = F.grid_sample(msk_batch, grid)
|
||||
|
||||
'''
|
||||
# Theta2 = translation matrix
|
||||
theta2 = torch.cuda.FloatTensor(anglesize, 2, 3).fill_(0)
|
||||
theta2[:, 0, 0] = 1
|
||||
theta2[:, 0, 1] = 0
|
||||
theta2[:, 0, 2] = (-target_x + 0.5) * 2
|
||||
theta2[:, 1, 0] = 0
|
||||
theta2[:, 1, 1] = 1
|
||||
theta2[:, 1, 2] = (-target_y + 0.5) * 2
|
||||
|
||||
grid2 = F.affine_grid(theta2, adv_batch.shape)
|
||||
adv_batch_t = F.grid_sample(adv_batch_t, grid2)
|
||||
msk_batch_t = F.grid_sample(msk_batch_t, grid2)
|
||||
|
||||
'''
|
||||
adv_batch_t = adv_batch_t.view(s[0], s[1], s[2], s[3], s[4])
|
||||
msk_batch_t = msk_batch_t.view(s[0], s[1], s[2], s[3], s[4])
|
||||
|
||||
adv_batch_t = torch.clamp(adv_batch_t, 0.000001, 0.999999)
|
||||
# img = msk_batch_t[0, 0, :, :, :].detach().cpu()
|
||||
# img = transforms.ToPILImage()(img)
|
||||
# img.show()
|
||||
# exit()
|
||||
|
||||
return adv_batch_t * msk_batch_t
|
||||
|
||||
|
||||
class PatchApplier(nn.Module):
|
||||
"""PatchApplier: applies adversarial patches to images.
|
||||
|
||||
Module providing the functionality necessary to apply a patch to all detections in all images in the batch.
|
||||
|
||||
PatchApplier：对图像应用对抗补丁。
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super(PatchApplier, self).__init__()
|
||||
|
||||
def forward(self, img_batch, adv_batch):
|
||||
advs = torch.unbind(adv_batch, 1) # 沿1维解开
|
||||
for adv in advs:
|
||||
img_batch = torch.where((adv == 0), img_batch, adv) # 对图像相应的坐标位置替换其像素？好像还没到图像的环节
|
||||
return img_batch
|
||||
|
||||
|
||||
'''
|
||||
class PatchGenerator(nn.Module):
|
||||
"""PatchGenerator: network module that generates adversarial patches.
|
||||
|
||||
Module representing the neural network that will generate adversarial patches.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, cfgfile, weightfile, img_dir, lab_dir):
|
||||
super(PatchGenerator, self).__init__()
|
||||
self.yolo = Darknet(cfgfile).load_weights(weightfile)
|
||||
self.dataloader = torch.utils.data.DataLoader(InriaDataset(img_dir, lab_dir, shuffle=True),
|
||||
batch_size=5,
|
||||
shuffle=True)
|
||||
self.patchapplier = PatchApplier()
|
||||
self.nmscalculator = NMSCalculator()
|
||||
self.totalvariation = TotalVariation()
|
||||
|
||||
def forward(self, *input):
|
||||
pass
|
||||
'''
|
||||
|
||||
|
||||
class InriaDataset(Dataset):
|
||||
"""InriaDataset: representation of the INRIA person dataset.
|
||||
|
||||
Internal representation of the commonly used INRIA person dataset.
|
||||
Available at: http://pascal.inrialpes.fr/data/human/
|
||||
|
||||
Attributes:
|
||||
len: An integer number of elements in the
|
||||
img_dir: Directory containing the images of the INRIA dataset.
|
||||
lab_dir: Directory containing the labels of the INRIA dataset.
|
||||
img_names: List of all image file names in img_dir.
|
||||
shuffle: Whether or not to shuffle the dataset.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, img_dir, lab_dir, max_lab, imgsize, shuffle=True):
|
||||
n_png_images = len(fnmatch.filter(os.listdir(img_dir), '*.png')) # 614 fnmatch.filter返回一个list
|
||||
n_jpg_images = len(fnmatch.filter(os.listdir(img_dir), '*.jpg')) # 0
|
||||
n_images = n_png_images + n_jpg_images # 图像的总数
|
||||
n_labels = len(fnmatch.filter(os.listdir(lab_dir), '*.txt'))
|
||||
assert n_images == n_labels, "Number of images and number of labels don't match"
|
||||
self.len = n_images
|
||||
self.img_dir = img_dir
|
||||
self.lab_dir = lab_dir
|
||||
self.imgsize = imgsize
|
||||
self.img_names = fnmatch.filter(os.listdir(img_dir), '*.png') + fnmatch.filter(os.listdir(img_dir), '*.jpg')
|
||||
self.shuffle = shuffle
|
||||
self.img_paths = []
|
||||
for img_name in self.img_names:
|
||||
self.img_paths.append(os.path.join(self.img_dir, img_name))
|
||||
self.lab_paths = []
|
||||
for img_name in self.img_names:
|
||||
lab_path = os.path.join(self.lab_dir, img_name).replace('.jpg', '.txt').replace('.png', '.txt')
|
||||
self.lab_paths.append(lab_path)
|
||||
self.max_n_labels = max_lab # label的长度
|
||||
|
||||
def __len__(self):
|
||||
return self.len
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx <= len(self), 'index range error'
|
||||
img_path = os.path.join(self.img_dir, self.img_names[idx])
|
||||
lab_path = os.path.join(self.lab_dir, self.img_names[idx]).replace('.jpg', '.txt').replace('.png', '.txt')
|
||||
image = Image.open(img_path).convert('RGB')
|
||||
if os.path.getsize(lab_path): # check to see if label file contains data.
|
||||
label = np.loadtxt(lab_path)
|
||||
else:
|
||||
label = np.ones([5])
|
||||
|
||||
label = torch.from_numpy(label).float()
|
||||
if label.dim() == 1:
|
||||
label = label.unsqueeze(0)
|
||||
|
||||
image, label = self.pad_and_scale(image, label)
|
||||
transform = transforms.ToTensor()
|
||||
image = transform(image)
|
||||
label = self.pad_lab(label)
|
||||
# print("image size :", image.shape)
|
||||
# print("label size :", label.shape)
|
||||
return image, label
|
||||
|
||||
def pad_and_scale(self, img, lab):
|
||||
"""
|
||||
|
||||
Args:
|
||||
img:
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
w, h = img.size
|
||||
if w == h:
|
||||
padded_img = img
|
||||
else:
|
||||
dim_to_pad = 1 if w < h else 2
|
||||
if dim_to_pad == 1:
|
||||
padding = (h - w) / 2
|
||||
padded_img = Image.new('RGB', (h, h), color=(127, 127, 127))
|
||||
padded_img.paste(img, (int(padding), 0))
|
||||
lab[:, [1]] = (lab[:, [1]] * w + padding) / h
|
||||
lab[:, [3]] = (lab[:, [3]] * w / h)
|
||||
else:
|
||||
padding = (w - h) / 2
|
||||
padded_img = Image.new('RGB', (w, w), color=(127, 127, 127))
|
||||
padded_img.paste(img, (0, int(padding)))
|
||||
lab[:, [2]] = (lab[:, [2]] * h + padding) / w
|
||||
lab[:, [4]] = (lab[:, [4]] * h / w)
|
||||
resize = transforms.Resize((self.imgsize, self.imgsize))
|
||||
padded_img = resize(padded_img) # choose here
|
||||
return padded_img, lab
|
||||
|
||||
def pad_lab(self, lab):
|
||||
pad_size = self.max_n_labels - lab.shape[0]
|
||||
if (pad_size > 0):
|
||||
padded_lab = F.pad(lab, (0, 0, 0, pad_size), value=1) # (左边填充数， 右边填充数， 上边填充数， 下边填充数)
|
||||
else:
|
||||
padded_lab = lab
|
||||
return padded_lab
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) == 3:
|
||||
img_dir = sys.argv[1]
|
||||
lab_dir = sys.argv[2]
|
||||
|
||||
else:
|
||||
print('Usage: ')
|
||||
print(' python load_data.py img_dir lab_dir')
|
||||
sys.exit()
|
||||
|
||||
test_loader = torch.utils.data.DataLoader(InriaDataset(img_dir, lab_dir, shuffle=True),
|
||||
batch_size=3, shuffle=True)
|
||||
|
||||
cfgfile = "cfg/yolov2.cfg"
|
||||
weightfile = "weights/yolov2.weights"
|
||||
printfile = "non_printability/30values.txt"
|
||||
|
||||
patch_size = 400
|
||||
|
||||
darknet_model = Darknet(cfgfile)
|
||||
darknet_model.load_weights(weightfile)
|
||||
darknet_model = darknet_model.cuda()
|
||||
patch_applier = PatchApplier().cuda()
|
||||
patch_transformer = PatchTransformer().cuda()
|
||||
prob_extractor = MaxProbExtractor(0, 80).cuda()
|
||||
nms_calculator = NMSCalculator(printfile, patch_size)
|
||||
total_variation = TotalVariation()
|
||||
'''
|
||||
img = Image.open('data/horse.jpg').convert('RGB')
|
||||
img = img.resize((darknet_model.width, darknet_model.height))
|
||||
width = img.width
|
||||
height = img.height
|
||||
img = torch.ByteTensor(torch.ByteStorage.from_buffer(img.tobytes()))
|
||||
img = img.view(height, width, 3).transpose(0, 1).transpose(0, 2).contiguous()
|
||||
img = img.view(1, 3, height, width)
|
||||
img = img.float().div(255.0)
|
||||
img = torch.autograd.Variable(img)
|
||||
|
||||
output = darknet_model(img)
|
||||
'''
|
||||
optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
|
||||
|
||||
tl0 = time.time()
|
||||
tl1 = time.time()
|
||||
for i_batch, (img_batch, lab_batch) in enumerate(test_loader):
|
||||
tl1 = time.time()
|
||||
print('time to fetch items: ', tl1 - tl0)
|
||||
img_batch = img_batch.cuda()
|
||||
lab_batch = lab_batch.cuda()
|
||||
adv_patch = Image.open('data/horse.jpg').convert('RGB')
|
||||
adv_patch = adv_patch.resize((patch_size, patch_size))
|
||||
transform = transforms.ToTensor()
|
||||
adv_patch = transform(adv_patch).cuda()
|
||||
img_size = img_batch.size(-1)
|
||||
print('transforming patches')
|
||||
t0 = time.time()
|
||||
adv_batch_t = patch_transformer.forward(adv_patch, lab_batch, img_size)
|
||||
print('applying patches')
|
||||
t1 = time.time()
|
||||
img_batch = patch_applier.forward(img_batch, adv_batch_t)
|
||||
img_batch = torch.autograd.Variable(img_batch)
|
||||
img_batch = F.interpolate(img_batch, (darknet_model.height, darknet_model.width))
|
||||
print('running patched images through model')
|
||||
t2 = time.time()
|
||||
|
||||
for obj in gc.get_objects():
|
||||
try:
|
||||
if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)):
|
||||
try:
|
||||
print(type(obj), obj.size())
|
||||
except:
|
||||
pass
|
||||
except:
|
||||
pass
|
||||
|
||||
print(torch.cuda.memory_allocated())
|
||||
|
||||
output = darknet_model(img_batch)
|
||||
print('extracting max probs')
|
||||
t3 = time.time()
|
||||
max_prob = prob_extractor(output)
|
||||
t4 = time.time()
|
||||
nms = nms_calculator.forward(adv_patch)
|
||||
tv = total_variation(adv_patch)
|
||||
print('---------------------------------')
|
||||
print(' patch transformation : %f' % (t1 - t0))
|
||||
print(' patch application : %f' % (t2 - t1))
|
||||
print(' darknet forward : %f' % (t3 - t2))
|
||||
print(' probability extraction : %f' % (t4 - t3))
|
||||
print('---------------------------------')
|
||||
print(' total forward pass : %f' % (t4 - t0))
|
||||
del img_batch, lab_batch, adv_patch, adv_batch_t, output, max_prob
|
||||
torch.cuda.empty_cache()
|
||||
tl0 = time.time()
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
import math
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.nn.modules.utils import _pair, _quadruple
|
||||
|
||||
|
||||
class MedianPool2d(nn.Module):
|
||||
""" Median pool (usable as median filter when stride=1) module.
|
||||
|
||||
Args:
|
||||
kernel_size: size of pooling kernel, int or 2-tuple
|
||||
stride: pool stride, int or 2-tuple
|
||||
padding: pool padding, int or 4-tuple (l, r, t, b) as in pytorch F.pad
|
||||
same: override padding and enforce same padding, boolean
|
||||
"""
|
||||
def __init__(self, kernel_size=3, stride=1, padding=0, same=False):
|
||||
super(MedianPool2d, self).__init__()
|
||||
self.k = _pair(kernel_size)
|
||||
self.stride = _pair(stride)
|
||||
self.padding = _quadruple(padding) # convert to l, r, t, b
|
||||
self.same = same
|
||||
|
||||
def _padding(self, x):
|
||||
if self.same:
|
||||
ih, iw = x.size()[2:]
|
||||
if ih % self.stride[0] == 0:
|
||||
ph = max(self.k[0] - self.stride[0], 0)
|
||||
else:
|
||||
ph = max(self.k[0] - (ih % self.stride[0]), 0)
|
||||
if iw % self.stride[1] == 0:
|
||||
pw = max(self.k[1] - self.stride[1], 0)
|
||||
else:
|
||||
pw = max(self.k[1] - (iw % self.stride[1]), 0)
|
||||
pl = pw // 2
|
||||
pr = pw - pl
|
||||
pt = ph // 2
|
||||
pb = ph - pt
|
||||
padding = (pl, pr, pt, pb)
|
||||
else:
|
||||
padding = self.padding
|
||||
return padding
|
||||
|
||||
def forward(self, x):
|
||||
# using existing pytorch functions and tensor ops so that we get autograd,
|
||||
# would likely be more efficient to implement from scratch at C/Cuda level
|
||||
x = F.pad(x, self._padding(x), mode='reflect')
|
||||
x = x.unfold(2, self.k[0], self.stride[0]).unfold(3, self.k[1], self.stride[1])
|
||||
x = x.contiguous().view(x.size()[:4] + (-1,)).median(dim=-1)[0]
|
||||
return x
|
||||
|
|
@ -0,0 +1 @@
|
|||
#
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
import math
|
||||
from collections import OrderedDict
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------#
|
||||
# 残差结构
|
||||
# 利用一个1x1卷积下降通道数，然后利用一个3x3卷积提取特征并且上升通道数
|
||||
# 最后接上一个残差边
|
||||
# ---------------------------------------------------------------------#
|
||||
class BasicBlock(nn.Module):
|
||||
def __init__(self, inplanes, planes):
|
||||
super(BasicBlock, self).__init__()
|
||||
self.conv1 = nn.Conv2d(inplanes, planes[0], kernel_size=1, stride=1, padding=0, bias=False) # 从大通道转化小通道。又从小通道转为大通道。
|
||||
self.bn1 = nn.BatchNorm2d(planes[0])
|
||||
self.relu1 = nn.LeakyReLU(0.1)
|
||||
|
||||
self.conv2 = nn.Conv2d(planes[0], planes[1], kernel_size=3, stride=1, padding=1, bias=False)
|
||||
self.bn2 = nn.BatchNorm2d(planes[1])
|
||||
self.relu2 = nn.LeakyReLU(0.1)
|
||||
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
|
||||
out = self.conv1(x)
|
||||
out = self.bn1(out)
|
||||
out = self.relu1(out)
|
||||
|
||||
out = self.conv2(out)
|
||||
out = self.bn2(out)
|
||||
out = self.relu2(out)
|
||||
|
||||
out += residual
|
||||
return out
|
||||
|
||||
|
||||
class DarkNet(nn.Module):
|
||||
def __init__(self, layers):
|
||||
super(DarkNet, self).__init__()
|
||||
self.inplanes = 32 # 第一次卷积，输出通道为32
|
||||
# 416,416,3 -> 416,416,32
|
||||
self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False)
|
||||
self.bn1 = nn.BatchNorm2d(self.inplanes)
|
||||
self.relu1 = nn.LeakyReLU(0.1)
|
||||
|
||||
# 416,416,32 -> 208,208,64
|
||||
self.layer1 = self._make_layer([32, 64], layers[0]) # layers 中保存的是程序块重复的次数
|
||||
# 208,208,64 -> 104,104,128
|
||||
self.layer2 = self._make_layer([64, 128], layers[1])
|
||||
# 104,104,128 -> 52,52,256
|
||||
self.layer3 = self._make_layer([128, 256], layers[2])
|
||||
# 52,52,256 -> 26,26,512
|
||||
self.layer4 = self._make_layer([256, 512], layers[3])
|
||||
# 26,26,512 -> 13,13,1024
|
||||
self.layer5 = self._make_layer([512, 1024], layers[4])
|
||||
|
||||
self.layers_out_filters = [64, 128, 256, 512, 1024]
|
||||
|
||||
# 进行权值初始化
|
||||
for m in self.modules():
|
||||
if isinstance(m, nn.Conv2d):
|
||||
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
|
||||
m.weight.data.normal_(0, math.sqrt(2. / n))
|
||||
elif isinstance(m, nn.BatchNorm2d):
|
||||
m.weight.data.fill_(1)
|
||||
m.bias.data.zero_()
|
||||
|
||||
# ---------------------------------------------------------------------#
|
||||
# 在每一个layer里面，首先利用一个步长为2的3x3卷积进行下采样
|
||||
# 然后进行残差结构的堆叠
|
||||
# ---------------------------------------------------------------------#
|
||||
def _make_layer(self, planes, blocks):
|
||||
layers = []
|
||||
# 下采样，步长为2，卷积核大小为3 # 进入_make_layer先创建一层网络，用于降采样，然后再是多个重复的block
|
||||
layers.append(("ds_conv", nn.Conv2d(self.inplanes, planes[1], kernel_size=3, stride=2, padding=1, bias=False)))
|
||||
layers.append(("ds_bn", nn.BatchNorm2d(planes[1])))
|
||||
layers.append(("ds_relu", nn.LeakyReLU(0.1)))
|
||||
# 加入残差结构
|
||||
self.inplanes = planes[1] # 保存这一层的输出通道，也是下一层的输入通道
|
||||
for i in range(0, blocks):
|
||||
layers.append(("residual_{}".format(i), BasicBlock(self.inplanes, planes)))
|
||||
return nn.Sequential(OrderedDict(layers))
|
||||
|
||||
def forward(self, x):
|
||||
x = self.conv1(x)
|
||||
x = self.bn1(x)
|
||||
x = self.relu1(x)
|
||||
|
||||
x = self.layer1(x)
|
||||
x = self.layer2(x)
|
||||
out3 = self.layer3(x)
|
||||
out4 = self.layer4(out3)
|
||||
out5 = self.layer5(out4)
|
||||
|
||||
return out3, out4, out5
|
||||
|
||||
|
||||
def darknet53():
|
||||
model = DarkNet([1, 2, 8, 8, 4])
|
||||
return model
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
from collections import OrderedDict
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from nets.darknet import darknet53
|
||||
|
||||
|
||||
def conv2d(filter_in, filter_out, kernel_size):
|
||||
pad = (kernel_size - 1) // 2 if kernel_size else 0
|
||||
return nn.Sequential(OrderedDict([
|
||||
("conv", nn.Conv2d(filter_in, filter_out, kernel_size=kernel_size, stride=1, padding=pad, bias=False)),
|
||||
("bn", nn.BatchNorm2d(filter_out)),
|
||||
("relu", nn.LeakyReLU(0.1)),
|
||||
]))
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------#
|
||||
# make_last_layers里面一共有七个卷积，前五个用于提取特征。
|
||||
# 后两个用于获得yolo网络的预测结果
|
||||
# ------------------------------------------------------------------------#
|
||||
def make_last_layers(filters_list, in_filters, out_filter):
|
||||
m = nn.Sequential(
|
||||
conv2d(in_filters, filters_list[0], 1), # 多次使用 1*1 的卷积调整通道，并进行通道方向的信息融合
|
||||
conv2d(filters_list[0], filters_list[1], 3),
|
||||
conv2d(filters_list[1], filters_list[0], 1),
|
||||
conv2d(filters_list[0], filters_list[1], 3),
|
||||
conv2d(filters_list[1], filters_list[0], 1),
|
||||
conv2d(filters_list[0], filters_list[1], 3),
|
||||
nn.Conv2d(filters_list[1], out_filter, kernel_size=1, stride=1, padding=0, bias=True)
|
||||
)
|
||||
return m
|
||||
|
||||
|
||||
class YoloBody(nn.Module):
|
||||
def __init__(self, anchors_mask, num_classes, pretrained=False):
|
||||
super(YoloBody, self).__init__()
|
||||
self.width = 416 # 临时加
|
||||
self.height = 416 # 临时加
|
||||
# ---------------------------------------------------#
|
||||
# 生成darknet53的主干模型
|
||||
# 获得三个有效特征层，他们的shape分别是：
|
||||
# 52,52,256
|
||||
# 26,26,512
|
||||
# 13,13,1024
|
||||
# ---------------------------------------------------#
|
||||
self.backbone = darknet53()
|
||||
if pretrained: # 载入预训练的权重，darknet53是一个分类网络
|
||||
self.backbone.load_state_dict(torch.load("model_data/darknet53_backbone_weights.pth"))
|
||||
|
||||
# ---------------------------------------------------#
|
||||
# out_filters : [64, 128, 256, 512, 1024]
|
||||
# ---------------------------------------------------#
|
||||
out_filters = self.backbone.layers_out_filters
|
||||
|
||||
# ------------------------------------------------------------------------#
|
||||
# 计算yolo_head的输出通道数，对于voc数据集而言
|
||||
# final_out_filter0 = final_out_filter1 = final_out_filter2 = 75
|
||||
# ------------------------------------------------------------------------# len(anchors_mask[0]) 为 3
|
||||
self.last_layer0 = make_last_layers([512, 1024], out_filters[-1], len(anchors_mask[0]) * (num_classes + 5))
|
||||
|
||||
self.last_layer1_conv = conv2d(512, 256, 1)
|
||||
self.last_layer1_upsample = nn.Upsample(scale_factor=2, mode='nearest')
|
||||
self.last_layer1 = make_last_layers([256, 512], out_filters[-2] + 256, len(anchors_mask[1]) * (num_classes + 5))
|
||||
|
||||
self.last_layer2_conv = conv2d(256, 128, 1)
|
||||
self.last_layer2_upsample = nn.Upsample(scale_factor=2, mode='nearest')
|
||||
self.last_layer2 = make_last_layers([128, 256], out_filters[-3] + 128, len(anchors_mask[2]) * (num_classes + 5))
|
||||
|
||||
def forward(self, x):
|
||||
# ---------------------------------------------------#
|
||||
# 获得三个有效特征层，他们的shape分别是：
|
||||
# 52,52,256；26,26,512；13,13,1024
|
||||
# ---------------------------------------------------#
|
||||
x2, x1, x0 = self.backbone(x)
|
||||
|
||||
# ---------------------------------------------------#
|
||||
# 第一个特征层
|
||||
# out0 = (batch_size,255,13,13)
|
||||
# ---------------------------------------------------#
|
||||
# 13,13,1024 -> 13,13,512 -> 13,13,1024 -> 13,13,512 -> 13,13,1024 -> 13,13,512
|
||||
out0_branch = self.last_layer0[:5](x0)
|
||||
out0 = self.last_layer0[5:](out0_branch) # 8, 75, 13, 13 刚开始的2是测试用的，不是正式数据
|
||||
|
||||
# 13,13,512 -> 13,13,256 -> 26,26,256
|
||||
x1_in = self.last_layer1_conv(out0_branch) # 融合分支
|
||||
x1_in = self.last_layer1_upsample(x1_in)
|
||||
|
||||
# 26,26,256 + 26,26,512 -> 26,26,768
|
||||
x1_in = torch.cat([x1_in, x1], 1)
|
||||
# ---------------------------------------------------#
|
||||
# 第二个特征层
|
||||
# out1 = (batch_size,255,26,26)
|
||||
# ---------------------------------------------------#
|
||||
# 26,26,768 -> 26,26,256 -> 26,26,512 -> 26,26,256 -> 26,26,512 -> 26,26,256
|
||||
out1_branch = self.last_layer1[:5](x1_in)
|
||||
out1 = self.last_layer1[5:](out1_branch)
|
||||
|
||||
# 26,26,256 -> 26,26,128 -> 52,52,128
|
||||
x2_in = self.last_layer2_conv(out1_branch) # 融合
|
||||
x2_in = self.last_layer2_upsample(x2_in)
|
||||
|
||||
# 52,52,128 + 52,52,256 -> 52,52,384
|
||||
x2_in = torch.cat([x2_in, x2], 1)
|
||||
# ---------------------------------------------------#
|
||||
# 第三个特征层
|
||||
# out3 = (batch_size,255,52,52)
|
||||
# ---------------------------------------------------#
|
||||
# 52,52,384 -> 52,52,128 -> 52,52,256 -> 52,52,128 -> 52,52,256 -> 52,52,128
|
||||
out2 = self.last_layer2(x2_in)
|
||||
return out0, out1, out2
|
||||
|
|
@ -0,0 +1,488 @@
|
|||
import math
|
||||
from functools import partial
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
class YOLOLoss(nn.Module):
|
||||
def __init__(self, anchors, num_classes, input_shape, cuda, anchors_mask=[[6, 7, 8], [3, 4, 5], [0, 1, 2]]):
|
||||
super(YOLOLoss, self).__init__()
|
||||
# -----------------------------------------------------------#
|
||||
# 13x13的特征层对应的anchor是[116,90],[156,198],[373,326]
|
||||
# 26x26的特征层对应的anchor是[30,61],[62,45],[59,119]
|
||||
# 52x52的特征层对应的anchor是[10,13],[16,30],[33,23]
|
||||
# -----------------------------------------------------------#
|
||||
self.anchors = anchors
|
||||
self.num_classes = num_classes
|
||||
self.bbox_attrs = 5 + num_classes
|
||||
self.input_shape = input_shape
|
||||
self.anchors_mask = anchors_mask
|
||||
|
||||
self.giou = True
|
||||
self.balance = [0.4, 1.0, 4]
|
||||
self.box_ratio = 0.05
|
||||
self.obj_ratio = 5 * (input_shape[0] * input_shape[1]) / (416 ** 2)
|
||||
self.cls_ratio = 1 * (num_classes / 80)
|
||||
|
||||
self.ignore_threshold = 0.5
|
||||
self.cuda = cuda
|
||||
|
||||
def clip_by_tensor(self, t, t_min, t_max):
|
||||
t = t.float()
|
||||
result = (t >= t_min).float() * t + (t < t_min).float() * t_min # 要么是t，要么是t_min
|
||||
result = (result <= t_max).float() * result + (result > t_max).float() * t_max
|
||||
return result
|
||||
|
||||
def MSELoss(self, pred, target):
|
||||
return torch.pow(pred - target, 2)
|
||||
|
||||
def BCELoss(self, pred, target):
|
||||
epsilon = 1e-7
|
||||
pred = self.clip_by_tensor(pred, epsilon, 1.0 - epsilon) # 保证tensor在 epsilon和1.0 - epsilon之间
|
||||
output = - target * torch.log(pred) - (1.0 - target) * torch.log(1.0 - pred)
|
||||
return output
|
||||
|
||||
def box_giou(self, b1, b2):
|
||||
"""
|
||||
输入为：
|
||||
----------
|
||||
b1: tensor, shape=(batch, feat_w, feat_h, anchor_num, 4), xywh
|
||||
b2: tensor, shape=(batch, feat_w, feat_h, anchor_num, 4), xywh
|
||||
|
||||
返回为：
|
||||
-------
|
||||
giou: tensor, shape=(batch, feat_w, feat_h, anchor_num, 1)
|
||||
"""
|
||||
# ----------------------------------------------------#
|
||||
# 求出预测框左上角右下角
|
||||
# ----------------------------------------------------#
|
||||
b1_xy = b1[..., :2]
|
||||
b1_wh = b1[..., 2:4]
|
||||
b1_wh_half = b1_wh / 2.
|
||||
b1_mins = b1_xy - b1_wh_half
|
||||
b1_maxes = b1_xy + b1_wh_half
|
||||
# ----------------------------------------------------#
|
||||
# 求出真实框左上角右下角
|
||||
# ----------------------------------------------------#
|
||||
b2_xy = b2[..., :2]
|
||||
b2_wh = b2[..., 2:4]
|
||||
b2_wh_half = b2_wh / 2.
|
||||
b2_mins = b2_xy - b2_wh_half
|
||||
b2_maxes = b2_xy + b2_wh_half
|
||||
|
||||
# ----------------------------------------------------#
|
||||
# 求真实框和预测框所有的iou
|
||||
# ----------------------------------------------------#
|
||||
intersect_mins = torch.max(b1_mins, b2_mins)
|
||||
intersect_maxes = torch.min(b1_maxes, b2_maxes)
|
||||
intersect_wh = torch.max(intersect_maxes - intersect_mins, torch.zeros_like(intersect_maxes))
|
||||
intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1]
|
||||
b1_area = b1_wh[..., 0] * b1_wh[..., 1]
|
||||
b2_area = b2_wh[..., 0] * b2_wh[..., 1]
|
||||
union_area = b1_area + b2_area - intersect_area
|
||||
iou = intersect_area / union_area
|
||||
|
||||
# ----------------------------------------------------#
|
||||
# 找到包裹两个框的最小框的左上角和右下角
|
||||
# ----------------------------------------------------#
|
||||
enclose_mins = torch.min(b1_mins, b2_mins)
|
||||
enclose_maxes = torch.max(b1_maxes, b2_maxes)
|
||||
enclose_wh = torch.max(enclose_maxes - enclose_mins, torch.zeros_like(intersect_maxes))
|
||||
# ----------------------------------------------------#
|
||||
# 计算对角线距离
|
||||
# ----------------------------------------------------#
|
||||
enclose_area = enclose_wh[..., 0] * enclose_wh[..., 1]
|
||||
giou = iou - (enclose_area - union_area) / enclose_area
|
||||
|
||||
return giou
|
||||
|
||||
def forward(self, l, input, targets=None):
|
||||
# ----------------------------------------------------#
|
||||
# l代表的是，当前输入进来的有效特征层，是第几个有效特征层
|
||||
# input的shape为 bs, 3*(5+num_classes), 13, 13
|
||||
# bs, 3*(5+num_classes), 26, 26
|
||||
# bs, 3*(5+num_classes), 52, 52
|
||||
# targets代表的是真实框。
|
||||
# ----------------------------------------------------#
|
||||
# --------------------------------#
|
||||
# 获得图片数量，特征层的高和宽
|
||||
# 13和13
|
||||
# --------------------------------#
|
||||
bs = input.size(0)
|
||||
in_h = input.size(2)
|
||||
in_w = input.size(3)
|
||||
# -----------------------------------------------------------------------#
|
||||
# 计算步长
|
||||
# 每一个特征点对应原来的图片上多少个像素点
|
||||
# 如果特征层为13x13的话，一个特征点就对应原来的图片上的32个像素点
|
||||
# 如果特征层为26x26的话，一个特征点就对应原来的图片上的16个像素点
|
||||
# 如果特征层为52x52的话，一个特征点就对应原来的图片上的8个像素点
|
||||
# stride_h = stride_w = 32、16、8
|
||||
# stride_h和stride_w都是32。
|
||||
# -----------------------------------------------------------------------#
|
||||
stride_h = self.input_shape[0] / in_h
|
||||
stride_w = self.input_shape[1] / in_w
|
||||
# -------------------------------------------------#
|
||||
# 把anchor转换到此时获得的scaled_anchors大小是相对于特征层的
|
||||
# -------------------------------------------------#
|
||||
scaled_anchors = [(a_w / stride_w, a_h / stride_h) for a_w, a_h in self.anchors] # 把anchor也缩放到与输出特征图相同尺度
|
||||
# -----------------------------------------------#
|
||||
# 输入的input一共有三个，他们的shape分别是
|
||||
# bs, 3*(5+num_classes), 13, 13 => batch_size, 3, 13, 13, 5 + num_classes
|
||||
# batch_size, 3, 26, 26, 5 + num_classes
|
||||
# batch_size, 3, 52, 52, 5 + num_classes
|
||||
# -----------------------------------------------#
|
||||
prediction = input.view(bs, len(self.anchors_mask[l]), self.bbox_attrs, in_h, in_w).permute(
|
||||
0, 1, 3, 4, 2).contiguous() # batch_size, 3种anchor, h, w, 单个anchor对应的25个输出值
|
||||
|
||||
# -----------------------------------------------#
|
||||
# 先验框的中心位置的调整参数
|
||||
# -----------------------------------------------#
|
||||
x = torch.sigmoid(prediction[..., 0]) # prediction[..., 0] 维度是8, 3, 13, 13 取tx坐标
|
||||
y = torch.sigmoid(prediction[..., 1]) # ty
|
||||
# -----------------------------------------------#
|
||||
# 先验框的宽高调整参数
|
||||
# -----------------------------------------------#
|
||||
w = prediction[..., 2] # tw
|
||||
h = prediction[..., 3] # th
|
||||
# -----------------------------------------------#
|
||||
# 获得置信度，是否有物体
|
||||
# -----------------------------------------------#
|
||||
conf = torch.sigmoid(prediction[..., 4]) # prediction[..., 4] 是否有目标
|
||||
# -----------------------------------------------#
|
||||
# 种类置信度
|
||||
# -----------------------------------------------#
|
||||
pred_cls = torch.sigmoid(prediction[..., 5:])
|
||||
|
||||
# -----------------------------------------------#
|
||||
# 获得网络应该有的预测结果 y_true是重新建立的真实标签 8, 3, 13, 13, 25. noobj_mask中有目标为0，其他为1. box_loss_scale记录了面积
|
||||
# -----------------------------------------------#
|
||||
y_true, noobj_mask, box_loss_scale = self.get_target(l, targets, scaled_anchors, in_h, in_w)
|
||||
# y_true中，是用 真实框转换为 与网络输出一致的格式。比如，坐标是在输出特征分辨率下的，类别是真实框所在的cell对应的类别。
|
||||
# ---------------------------------------------------------------#
|
||||
# 将预测结果进行解码，判断预测结果和真实值的重合程度
|
||||
# 如果重合程度过大则忽略，因为这些特征点属于预测比较准确的特征点
|
||||
# 作为负样本不合适 # l在这里是三个多尺度特征图的第几个 pred_boxes是生成的网络预测的结果
|
||||
# ----------------------------------------------------------------#
|
||||
noobj_mask, pred_boxes = self.get_ignore(l, x, y, h, w, targets, scaled_anchors, in_h, in_w, noobj_mask)
|
||||
|
||||
if self.cuda:
|
||||
y_true = y_true.type_as(x)
|
||||
noobj_mask = noobj_mask.type_as(x)
|
||||
box_loss_scale = box_loss_scale.type_as(x)
|
||||
# --------------------------------------------------------------------------#
|
||||
# box_loss_scale是真实框宽高的乘积，宽高均在0-1之间，因此乘积也在0-1之间。
|
||||
# 2-宽高的乘积代表真实框越大，比重越小，小框的比重更大。
|
||||
# --------------------------------------------------------------------------#
|
||||
box_loss_scale = 2 - box_loss_scale
|
||||
|
||||
loss = 0
|
||||
obj_mask = y_true[..., 4] == 1
|
||||
n = torch.sum(obj_mask)
|
||||
if n != 0:
|
||||
if self.giou:
|
||||
# ---------------------------------------------------------------#
|
||||
# 计算预测结果和真实结果的giou
|
||||
# ----------------------------------------------------------------#
|
||||
giou = self.box_giou(pred_boxes, y_true[..., :4]).type_as(x)
|
||||
loss_loc = torch.mean((1 - giou)[obj_mask]) # 这里用的GIOU 作为定位误差，而不是论文中的MSE
|
||||
else:
|
||||
# -----------------------------------------------------------#
|
||||
# 计算中心偏移情况的loss，使用BCELoss效果好一些
|
||||
# -----------------------------------------------------------#
|
||||
loss_x = torch.mean(self.BCELoss(x[obj_mask], y_true[..., 0][obj_mask]) * box_loss_scale[obj_mask])
|
||||
loss_y = torch.mean(self.BCELoss(y[obj_mask], y_true[..., 1][obj_mask]) * box_loss_scale[obj_mask])
|
||||
# -----------------------------------------------------------#
|
||||
# 计算宽高调整值的loss
|
||||
# -----------------------------------------------------------#
|
||||
loss_w = torch.mean(self.MSELoss(w[obj_mask], y_true[..., 2][obj_mask]) * box_loss_scale[obj_mask])
|
||||
loss_h = torch.mean(self.MSELoss(h[obj_mask], y_true[..., 3][obj_mask]) * box_loss_scale[obj_mask])
|
||||
loss_loc = (loss_x + loss_y + loss_h + loss_w) * 0.1
|
||||
# pred_cls[obj_mask] 有目标的框数* 20个属性值（20个分类）
|
||||
loss_cls = torch.mean(self.BCELoss(pred_cls[obj_mask], y_true[..., 5:][obj_mask])) # 目标的分类误差
|
||||
loss += loss_loc * self.box_ratio + loss_cls * self.cls_ratio
|
||||
|
||||
loss_conf = torch.mean(self.BCELoss(conf, obj_mask.type_as(conf))[noobj_mask.bool() | obj_mask]) # 忽略掉部分重叠高的但不是最匹配的预测框 的是否有目标的误差
|
||||
loss += loss_conf * self.balance[l] * self.obj_ratio # self.balance[l]不同层的权重不一样 [0.4, 1.0, 4] 表示对小目标损失权重更大
|
||||
# if n != 0:
|
||||
# print(loss_loc * self.box_ratio, loss_cls * self.cls_ratio, loss_conf * self.balance[l] * self.obj_ratio)
|
||||
return loss
|
||||
|
||||
def calculate_iou(self, _box_a, _box_b):
|
||||
# -----------------------------------------------------------#
|
||||
# 计算真实框的左上角和右下角 以0,0为中心点，计算左上角和右下角
|
||||
# -----------------------------------------------------------#
|
||||
b1_x1, b1_x2 = _box_a[:, 0] - _box_a[:, 2] / 2, _box_a[:, 0] + _box_a[:, 2] / 2
|
||||
b1_y1, b1_y2 = _box_a[:, 1] - _box_a[:, 3] / 2, _box_a[:, 1] + _box_a[:, 3] / 2
|
||||
# -----------------------------------------------------------#
|
||||
# 计算先验框获得的预测框的左上角和右下角
|
||||
# -----------------------------------------------------------#
|
||||
b2_x1, b2_x2 = _box_b[:, 0] - _box_b[:, 2] / 2, _box_b[:, 0] + _box_b[:, 2] / 2
|
||||
b2_y1, b2_y2 = _box_b[:, 1] - _box_b[:, 3] / 2, _box_b[:, 1] + _box_b[:, 3] / 2
|
||||
|
||||
# -----------------------------------------------------------#
|
||||
# 将真实框和预测框都转化成左上角右下角的形式
|
||||
# -----------------------------------------------------------#
|
||||
box_a = torch.zeros_like(_box_a)
|
||||
box_b = torch.zeros_like(_box_b)
|
||||
box_a[:, 0], box_a[:, 1], box_a[:, 2], box_a[:, 3] = b1_x1, b1_y1, b1_x2, b1_y2
|
||||
box_b[:, 0], box_b[:, 1], box_b[:, 2], box_b[:, 3] = b2_x1, b2_y1, b2_x2, b2_y2
|
||||
|
||||
# ----------------------------------------------------------- #
|
||||
# A为真实框的数量，B为先验框的数量
|
||||
# ----------------------------------------------------------- #
|
||||
A = box_a.size(0)
|
||||
B = box_b.size(0)
|
||||
|
||||
# ----------------------------------------------------------- #
|
||||
# 计算交的面积 box_a是真实框，左上角和右下角。 box_b是先验框的左上角和右下角
|
||||
# box_a[:, 2:].unsqueeze(1).expand(A, B, 2) 从 5, 1, 2 扩展到5, 9, 2。 这里的5是图中框的数量。每一个组有9个，5个框重复9次
|
||||
# box_b[:, 2:].unsqueeze(0).expand(A, B, 2) 从 1, 9, 2 扩展到5, 9, 2。 这里的每一个组9个是不一样的9个anchor框，重复5次。
|
||||
# ----------------------------------------------------------- #
|
||||
|
||||
# 每一个gt复制 len(anchors)次，然后与所有anchors比较
|
||||
max_xy = torch.min(box_a[:, 2:].unsqueeze(1).expand(A, B, 2), # 计算右下角的最小点
|
||||
box_b[:, 2:].unsqueeze(0).expand(A, B, 2)) # 输出 5，9，2
|
||||
min_xy = torch.max(box_a[:, :2].unsqueeze(1).expand(A, B, 2), # 计算左上角的最大点
|
||||
box_b[:, :2].unsqueeze(0).expand(A, B, 2)) # 输出 5，9，2
|
||||
inter = torch.clamp((max_xy - min_xy), # 这里无法判断两个框不相交的情况。但不相交 U 就大，所以应该不影响结果
|
||||
min=0) # 最小值是0，最大值不限。相减之后，得到宽和高。# input：输入张量 min：范围的最小值，如果不指定的话，会默认无下界 max：范围的最大值，如果不指定的话，会默认无上界
|
||||
inter = inter[:, :, 0] * inter[:, :, 1] # 每个真实框与锚框 相交的面积
|
||||
# ----------------------------------------------------------- #
|
||||
# 计算预测框和真实框各自的面积
|
||||
# ----------------------------------------------------------- #
|
||||
area_a = ((box_a[:, 2] - box_a[:, 0]) * (box_a[:, 3] - box_a[:, 1])).unsqueeze(1).expand_as(
|
||||
inter) # [A,B] 5个值，重复9次
|
||||
area_b = ((box_b[:, 2] - box_b[:, 0]) * (box_b[:, 3] - box_b[:, 1])).unsqueeze(0).expand_as(
|
||||
inter) # [A,B] 9个值，重复5次
|
||||
# ----------------------------------------------------------- #
|
||||
# 求IOU
|
||||
# ----------------------------------------------------------- #
|
||||
union = area_a + area_b - inter
|
||||
return inter / union # [A,B]
|
||||
|
||||
def get_target(self, l, targets, anchors, in_h, in_w):
|
||||
# -----------------------------------------------------#
|
||||
# 计算一共有多少张图片
|
||||
# -----------------------------------------------------#
|
||||
bs = len(targets)
|
||||
# -----------------------------------------------------#
|
||||
# 对每一个grid cell，都需要标记。用于选取哪些先验框不包含物体
|
||||
# -----------------------------------------------------#
|
||||
noobj_mask = torch.ones(bs, len(self.anchors_mask[l]), in_h, in_w, requires_grad=False)
|
||||
# -----------------------------------------------------#
|
||||
# 让网络更加去关注小目标
|
||||
# -----------------------------------------------------#
|
||||
box_loss_scale = torch.zeros(bs, len(self.anchors_mask[l]), in_h, in_w, requires_grad=False)
|
||||
# -----------------------------------------------------#
|
||||
# batch_size, 3, 13, 13, 5 + num_classes
|
||||
# -----------------------------------------------------#
|
||||
y_true = torch.zeros(bs, len(self.anchors_mask[l]), in_h, in_w, self.bbox_attrs, requires_grad=False)
|
||||
for b in range(bs): # 每张图片单独计算
|
||||
if len(targets[b]) == 0: # targets是真实框
|
||||
continue
|
||||
batch_target = torch.zeros_like(targets[b]) # 把0~1之间的targets转换到 特征图大小的 targets
|
||||
# -------------------------------------------------------#
|
||||
# 计算出正样本在特征层上的中心点 # box第0，1维记录中心点 box第2，3维记录宽高 # 这里不知道为何这样做，但结果一样的
|
||||
# -------------------------------------------------------#
|
||||
batch_target[:, [0, 2]] = targets[b][:, [0, 2]] * in_w # 从归一化的box中反解出在 13*13 分辨率下的大小 两个 x 坐标
|
||||
batch_target[:, [1, 3]] = targets[b][:, [1, 3]] * in_h
|
||||
batch_target[:, 4] = targets[b][:, 4]
|
||||
batch_target = batch_target.cpu() # 因为是从targets（放在cuda上）中复制过来的，所以需要执行一次cpu()
|
||||
|
||||
# -------------------------------------------------------#
|
||||
# 将真实框转换一个形式 相当于都放到0, 0, w, h 进行比较
|
||||
# num_true_box, 4 # 把2，3 维，也就是宽和高取出，前面拼两个0
|
||||
# -------------------------------------------------------#
|
||||
gt_box = torch.FloatTensor(torch.cat((torch.zeros((batch_target.size(0), 2)), batch_target[:, 2:4]), 1))
|
||||
# -------------------------------------------------------#
|
||||
# 将先验框转换一个形式
|
||||
# 9, 4 在先验框大小前面加了两个0
|
||||
# -------------------------------------------------------#
|
||||
anchor_shapes = torch.FloatTensor(
|
||||
torch.cat((torch.zeros((len(anchors), 2)), torch.FloatTensor(anchors)), 1))
|
||||
# -------------------------------------------------------#
|
||||
# 计算交并比
|
||||
# self.calculate_iou(gt_box, anchor_shapes) = [num_true_box, 9]每一个真实框和9个先验框的重合情况
|
||||
# best_ns:
|
||||
# [每个真实框最大的重合度max_iou, 每一个真实框最重合的先验框的序号] # self.calculate_iou(gt_box, anchor_shapes) 的结果，是 b x len(anchors)
|
||||
# -------------------------------------------------------#
|
||||
best_ns = torch.argmax(self.calculate_iou(gt_box, anchor_shapes), dim=-1) # 找到每个真实框与所有anchor的IoU，然后取出每个真实框最匹配的anchor下标
|
||||
# 依次遍历每个真实框对应的anchor号数，找到在 所属当前层的3个anchor中的下标
|
||||
for t, best_n in enumerate(best_ns): # l是最后输出的多层特征图第几层
|
||||
if best_n not in self.anchors_mask[l]: # self.anchors_mask的用法是指定当前特征图用的是哪3个anchor
|
||||
continue
|
||||
# ----------------------------------------#
|
||||
# 判断这个先验框是当前特征点的哪一个先验框 l是第几号最后的输出特征图
|
||||
# ----------------------------------------#
|
||||
k = self.anchors_mask[l].index(best_n) # 使用当前层对应anchors的第几号anchor
|
||||
# ----------------------------------------#
|
||||
# 获得真实框属于哪个网格点 获取中心点。因为映射到了13*13分辨率上。 floor不就是左上角的意思？
|
||||
# ----------------------------------------#
|
||||
i = torch.floor(batch_target[t, 0]).long() # t 表示当前是第几个真实框
|
||||
j = torch.floor(batch_target[t, 1]).long()
|
||||
# ----------------------------------------#
|
||||
# 取出真实框的种类
|
||||
# ----------------------------------------#
|
||||
c = batch_target[t, 4].long()
|
||||
|
||||
# ----------------------------------------#
|
||||
# noobj_mask代表无目标的特征点 b是几号batch，k是几号anchor
|
||||
# ----------------------------------------#
|
||||
noobj_mask[b, k, j, i] = 0
|
||||
# ----------------------------------------#
|
||||
# tx、ty代表中心调整参数的真实值
|
||||
# ----------------------------------------#
|
||||
if not self.giou: # 不走这条分支
|
||||
# ----------------------------------------#
|
||||
# tx、ty代表中心调整参数的真实值
|
||||
# ----------------------------------------#
|
||||
y_true[b, k, j, i, 0] = batch_target[t, 0] - i.float()
|
||||
y_true[b, k, j, i, 1] = batch_target[t, 1] - j.float()
|
||||
y_true[b, k, j, i, 2] = math.log(batch_target[t, 2] / anchors[best_n][0])
|
||||
y_true[b, k, j, i, 3] = math.log(batch_target[t, 3] / anchors[best_n][1])
|
||||
y_true[b, k, j, i, 4] = 1
|
||||
y_true[b, k, j, i, c + 5] = 1 # 重新设置标记种类
|
||||
else:
|
||||
# ----------------------------------------#
|
||||
# tx、ty代表中心调整参数的真实值 重新生成的标签 y_true t是当前的图像的第t个真实框
|
||||
# ----------------------------------------#
|
||||
y_true[b, k, j, i, 0] = batch_target[t, 0]
|
||||
y_true[b, k, j, i, 1] = batch_target[t, 1]
|
||||
y_true[b, k, j, i, 2] = batch_target[t, 2]
|
||||
y_true[b, k, j, i, 3] = batch_target[t, 3]
|
||||
y_true[b, k, j, i, 4] = 1 # 有物体
|
||||
y_true[b, k, j, i, c + 5] = 1 # c是种类
|
||||
# ----------------------------------------#
|
||||
# 用于获得xywh的比例
|
||||
# 大目标loss权重小，小目标loss权重大
|
||||
# ----------------------------------------#
|
||||
box_loss_scale[b, k, j, i] = batch_target[t, 2] * batch_target[t, 3] / in_w / in_h # 这里计算出面积，能反应大小目标。又归一化到0~1之间。
|
||||
return y_true, noobj_mask, box_loss_scale
|
||||
|
||||
def get_ignore(self, l, x, y, h, w, targets, scaled_anchors, in_h, in_w, noobj_mask):
|
||||
# -----------------------------------------------------#
|
||||
# 计算一共有多少张图片
|
||||
# -----------------------------------------------------#
|
||||
bs = len(targets)
|
||||
|
||||
# -----------------------------------------------------#
|
||||
# 生成网格，先验框中心，网格左上角 torch.linspace(0, in_w - 1, in_w) 在0, in_w - 1之间分成in_w个点。.repeat(in_h, 1)沿0重复in_h次，沿1重复1次
|
||||
# -----------------------------------------------------#
|
||||
grid_x = torch.linspace(0, in_w - 1, in_w).repeat(in_h, 1).repeat(
|
||||
int(bs * len(self.anchors_mask[l])), 1, 1).view(x.shape).type_as(x) # 这样写 repeat 比较清晰。repeat从右向左分析比较清晰。后两维是沿着竖轴和横轴重复指定次数。
|
||||
grid_y = torch.linspace(0, in_h - 1, in_h).repeat(in_w, 1).t().repeat(
|
||||
int(bs * len(self.anchors_mask[l])), 1, 1).view(y.shape).type_as(x)
|
||||
|
||||
# 生成先验框的宽高
|
||||
scaled_anchors_l = np.array(scaled_anchors)[self.anchors_mask[l]] # 取出对应的3个先验框的具体值
|
||||
anchor_w = torch.Tensor(scaled_anchors_l).index_select(1, torch.LongTensor([0])).type_as(x) # 沿1维度，找到第几维值
|
||||
anchor_h = torch.Tensor(scaled_anchors_l).index_select(1, torch.LongTensor([1])).type_as(x)
|
||||
|
||||
anchor_w = anchor_w.repeat(bs, 1).repeat(1, 1, in_h * in_w).view(w.shape) # 13*13 个一样的，形成一组。3个不一样的13*13。 x8次
|
||||
anchor_h = anchor_h.repeat(bs, 1).repeat(1, 1, in_h * in_w).view(h.shape)
|
||||
# -------------------------------------------------------#
|
||||
# 计算调整后的先验框中心与宽高 x是输出的第0属性，就是x的sigmoid的输出坐标
|
||||
# -------------------------------------------------------#
|
||||
pred_boxes_x = torch.unsqueeze(x + grid_x, -1)
|
||||
pred_boxes_y = torch.unsqueeze(y + grid_y, -1)
|
||||
pred_boxes_w = torch.unsqueeze(torch.exp(w) * anchor_w, -1)
|
||||
pred_boxes_h = torch.unsqueeze(torch.exp(h) * anchor_h, -1)
|
||||
pred_boxes = torch.cat([pred_boxes_x, pred_boxes_y, pred_boxes_w, pred_boxes_h], dim=-1)
|
||||
|
||||
for b in range(bs): # 对一个 batch 里的数据 一张张图像 分别进行操作
|
||||
# -------------------------------------------------------#
|
||||
# 将预测结果转换一个形式
|
||||
# pred_boxes_for_ignore num_anchors, 4
|
||||
# -------------------------------------------------------#
|
||||
pred_boxes_for_ignore = pred_boxes[b].view(-1, 4)
|
||||
# -------------------------------------------------------#
|
||||
# 计算真实框，并把真实框转换成相对于特征层的大小
|
||||
# gt_box num_true_box, 4
|
||||
# -------------------------------------------------------#
|
||||
if len(targets[b]) > 0: # 如果有目标，进行下面的操作。否则 跳到下一张图片。
|
||||
batch_target = torch.zeros_like(targets[b])
|
||||
# -------------------------------------------------------#
|
||||
# 计算出正样本在特征层上的中心点 # 这里地方好像也是把 box当前左上角和右下角的形式，实现已经变成了中心点与宽高的形式。但无论如何，最终的结果没变。
|
||||
# -------------------------------------------------------#
|
||||
batch_target[:, [0, 2]] = targets[b][:, [0, 2]] * in_w
|
||||
batch_target[:, [1, 3]] = targets[b][:, [1, 3]] * in_h
|
||||
batch_target = batch_target[:, :4].type_as(x)
|
||||
# -------------------------------------------------------#
|
||||
# 计算交并比
|
||||
# anch_ious num_true_box, num_anchors
|
||||
# -------------------------------------------------------#
|
||||
anch_ious = self.calculate_iou(batch_target, pred_boxes_for_ignore) # 真实框与预测框的IoU
|
||||
# -------------------------------------------------------#
|
||||
# 每个先验框???对应真实框的最大重合度
|
||||
# anch_ious_max num_anchors
|
||||
# -------------------------------------------------------#
|
||||
anch_ious_max, _ = torch.max(anch_ious, dim=0) # 每个真实框与预测框的最大值。
|
||||
anch_ious_max = anch_ious_max.view(pred_boxes[b].size()[:3])
|
||||
noobj_mask[b][anch_ious_max > self.ignore_threshold] = 0 # 如果大于某个阈值，即使不是最匹配的，也可以忽略这个cell。所以noobj设置为0。
|
||||
return noobj_mask, pred_boxes
|
||||
|
||||
|
||||
def weights_init(net, init_type='normal', init_gain=0.02):
|
||||
def init_func(m):
|
||||
classname = m.__class__.__name__
|
||||
if hasattr(m, 'weight') and classname.find('Conv') != -1:
|
||||
if init_type == 'normal':
|
||||
torch.nn.init.normal_(m.weight.data, 0.0, init_gain)
|
||||
elif init_type == 'xavier':
|
||||
torch.nn.init.xavier_normal_(m.weight.data, gain=init_gain)
|
||||
elif init_type == 'kaiming':
|
||||
torch.nn.init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
|
||||
elif init_type == 'orthogonal':
|
||||
torch.nn.init.orthogonal_(m.weight.data, gain=init_gain)
|
||||
else:
|
||||
raise NotImplementedError('initialization method [%s] is not implemented' % init_type)
|
||||
elif classname.find('BatchNorm2d') != -1:
|
||||
torch.nn.init.normal_(m.weight.data, 1.0, 0.02)
|
||||
torch.nn.init.constant_(m.bias.data, 0.0)
|
||||
|
||||
print('initialize network with %s type' % init_type)
|
||||
net.apply(init_func)
|
||||
|
||||
|
||||
def get_lr_scheduler(lr_decay_type, lr, min_lr, total_iters, warmup_iters_ratio=0.05, warmup_lr_ratio=0.1,
|
||||
no_aug_iter_ratio=0.05, step_num=10):
|
||||
def yolox_warm_cos_lr(lr, min_lr, total_iters, warmup_total_iters, warmup_lr_start, no_aug_iter, iters):
|
||||
if iters <= warmup_total_iters:
|
||||
# lr = (lr - warmup_lr_start) * iters / float(warmup_total_iters) + warmup_lr_start
|
||||
lr = (lr - warmup_lr_start) * pow(iters / float(warmup_total_iters), 2) + warmup_lr_start
|
||||
elif iters >= total_iters - no_aug_iter:
|
||||
lr = min_lr
|
||||
else:
|
||||
lr = min_lr + 0.5 * (lr - min_lr) * (
|
||||
1.0 + math.cos(
|
||||
math.pi * (iters - warmup_total_iters) / (total_iters - warmup_total_iters - no_aug_iter))
|
||||
)
|
||||
return lr
|
||||
|
||||
def step_lr(lr, decay_rate, step_size, iters):
|
||||
if step_size < 1:
|
||||
raise ValueError("step_size must above 1.")
|
||||
n = iters // step_size
|
||||
out_lr = lr * decay_rate ** n
|
||||
return out_lr
|
||||
|
||||
if lr_decay_type == "cos":
|
||||
warmup_total_iters = min(max(warmup_iters_ratio * total_iters, 1), 3)
|
||||
warmup_lr_start = max(warmup_lr_ratio * lr, 1e-6)
|
||||
no_aug_iter = min(max(no_aug_iter_ratio * total_iters, 1), 15)
|
||||
func = partial(yolox_warm_cos_lr, lr, min_lr, total_iters, warmup_total_iters, warmup_lr_start, no_aug_iter)
|
||||
else:
|
||||
decay_rate = (min_lr / lr) ** (1 / (step_num - 1))
|
||||
step_size = total_iters / step_num
|
||||
func = partial(step_lr, lr, decay_rate, step_size)
|
||||
|
||||
return func
|
||||
|
||||
|
||||
def set_optimizer_lr(optimizer, lr_scheduler_func, epoch):
|
||||
lr = lr_scheduler_func(epoch)
|
||||
for param_group in optimizer.param_groups:
|
||||
param_group['lr'] = lr
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
from torch import optim
|
||||
|
||||
|
||||
class BaseConfig(object):
|
||||
"""
|
||||
Default parameters for all config files.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Set the defaults.
|
||||
"""
|
||||
# self.img_dir = "inria/Train/pos"
|
||||
# self.lab_dir = "inria/Train/pos/yolo-labels"
|
||||
self.img_dir = "cctsdb/Train/pos"
|
||||
self.lab_dir = "cctsdb/Train/labels"
|
||||
self.cfgfile = "cfg/yolo.cfg"
|
||||
self.weightfile = "weights/yolo.weights"
|
||||
self.printfile = "non_printability/30values.txt"
|
||||
self.patch_size = 300
|
||||
|
||||
self.start_learning_rate = 0.03
|
||||
|
||||
self.patch_name = 'base'
|
||||
|
||||
self.scheduler_factory = lambda x: optim.lr_scheduler.ReduceLROnPlateau(x, 'min', patience=50)
|
||||
self.max_tv = 0
|
||||
|
||||
self.batch_size = 20
|
||||
|
||||
self.loss_target = lambda obj, cls: obj * cls
|
||||
|
||||
|
||||
class Experiment1(BaseConfig):
|
||||
"""
|
||||
Model that uses a maximum total variation, tv cannot go below this point.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Change stuff...
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
self.patch_name = 'Experiment1'
|
||||
self.max_tv = 0.165
|
||||
|
||||
|
||||
class Experiment2HighRes(Experiment1):
|
||||
"""
|
||||
Higher res
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Change stuff...
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
self.max_tv = 0.165
|
||||
self.patch_size = 400
|
||||
self.patch_name = 'Exp2HighRes'
|
||||
|
||||
|
||||
class Experiment3LowRes(Experiment1):
|
||||
"""
|
||||
Lower res
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Change stuff...
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
self.max_tv = 0.165
|
||||
self.patch_size = 100
|
||||
self.patch_name = "Exp3LowRes"
|
||||
|
||||
|
||||
class Experiment4ClassOnly(Experiment1):
|
||||
"""
|
||||
Only minimise class score.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Change stuff...
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
self.patch_name = 'Experiment4ClassOnly'
|
||||
self.loss_target = lambda obj, cls: cls
|
||||
|
||||
|
||||
class Experiment1Desktop(Experiment1):
|
||||
"""
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Change batch size.
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
self.batch_size = 8
|
||||
self.patch_size = 400
|
||||
|
||||
|
||||
class ReproducePaperObj(BaseConfig):
|
||||
"""
|
||||
Reproduce the results from the paper: Generate a patch that minimises object score.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.batch_size = 8
|
||||
self.patch_size = 300
|
||||
|
||||
self.patch_name = 'ObjectOnlyPaper'
|
||||
self.max_tv = 0.165
|
||||
|
||||
self.loss_target = lambda obj, cls: obj
|
||||
|
||||
|
||||
patch_configs = {
|
||||
"base": BaseConfig,
|
||||
"exp1": Experiment1,
|
||||
"exp1_des": Experiment1Desktop,
|
||||
"exp2_high_res": Experiment2HighRes,
|
||||
"exp3_low_res": Experiment3LowRes,
|
||||
"exp4_class_only": Experiment4ClassOnly,
|
||||
"paper_obj": ReproducePaperObj
|
||||
}
|
||||
|
|
@ -0,0 +1,181 @@
|
|||
# -----------------------------------------------------------------------#
|
||||
# predict.py将单张图片预测、摄像头检测、FPS测试和目录遍历检测等功能
|
||||
# 整合到了一个py文件中，通过指定mode进行模式的修改。
|
||||
# -----------------------------------------------------------------------#
|
||||
import time
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from yolo import YOLO
|
||||
|
||||
if __name__ == "__main__":
|
||||
yolo = YOLO()
|
||||
# ----------------------------------------------------------------------------------------------------------#
|
||||
# mode用于指定测试的模式：
|
||||
# 'predict' 表示单张图片预测，如果想对预测过程进行修改，如保存图片，截取对象等，可以先看下方详细的注释
|
||||
# 'video' 表示视频检测，可调用摄像头或者视频进行检测，详情查看下方注释。
|
||||
# 'fps' 表示测试fps，使用的图片是img里面的street.jpg，详情查看下方注释。
|
||||
# 'dir_predict' 表示遍历文件夹进行检测并保存。默认遍历img文件夹，保存img_out文件夹，详情查看下方注释。
|
||||
# 'heatmap' 表示进行预测结果的热力图可视化，详情查看下方注释。
|
||||
# 'export_onnx' 表示将模型导出为onnx，需要pytorch1.7.1以上。
|
||||
# ----------------------------------------------------------------------------------------------------------#
|
||||
mode = "predict"
|
||||
# -------------------------------------------------------------------------#
|
||||
# crop 指定了是否在单张图片预测后对目标进行截取
|
||||
# count 指定了是否进行目标的计数
|
||||
# crop、count仅在mode='predict'时有效
|
||||
# -------------------------------------------------------------------------#
|
||||
crop = False
|
||||
count = False
|
||||
# ----------------------------------------------------------------------------------------------------------#
|
||||
# video_path 用于指定视频的路径，当video_path=0时表示检测摄像头
|
||||
# 想要检测视频，则设置如video_path = "xxx.mp4"即可，代表读取出根目录下的xxx.mp4文件。
|
||||
# video_save_path 表示视频保存的路径，当video_save_path=""时表示不保存
|
||||
# 想要保存视频，则设置如video_save_path = "yyy.mp4"即可，代表保存为根目录下的yyy.mp4文件。
|
||||
# video_fps 用于保存的视频的fps
|
||||
#
|
||||
# video_path、video_save_path和video_fps仅在mode='video'时有效
|
||||
# 保存视频时需要ctrl+c退出或者运行到最后一帧才会完成完整的保存步骤。
|
||||
# ----------------------------------------------------------------------------------------------------------#
|
||||
video_path = 0
|
||||
video_save_path = ""
|
||||
video_fps = 25.0
|
||||
# ----------------------------------------------------------------------------------------------------------#
|
||||
# test_interval 用于指定测量fps的时候，图片检测的次数。理论上test_interval越大，fps越准确。
|
||||
# fps_image_path 用于指定测试的fps图片
|
||||
#
|
||||
# test_interval和fps_image_path仅在mode='fps'有效
|
||||
# ----------------------------------------------------------------------------------------------------------#
|
||||
test_interval = 100
|
||||
fps_image_path = "img/street.jpg"
|
||||
# -------------------------------------------------------------------------#
|
||||
# dir_origin_path 指定了用于检测的图片的文件夹路径
|
||||
# dir_save_path 指定了检测完图片的保存路径
|
||||
#
|
||||
# dir_origin_path和dir_save_path仅在mode='dir_predict'时有效
|
||||
# -------------------------------------------------------------------------#
|
||||
dir_origin_path = "img/"
|
||||
dir_save_path = "img_out/"
|
||||
# -------------------------------------------------------------------------#
|
||||
# heatmap_save_path 热力图的保存路径，默认保存在model_data下
|
||||
#
|
||||
# heatmap_save_path仅在mode='heatmap'有效
|
||||
# -------------------------------------------------------------------------#
|
||||
heatmap_save_path = "model_data/heatmap_vision.png"
|
||||
# -------------------------------------------------------------------------#
|
||||
# simplify 使用Simplify onnx
|
||||
# onnx_save_path 指定了onnx的保存路径
|
||||
# -------------------------------------------------------------------------#
|
||||
simplify = True
|
||||
onnx_save_path = "model_data/models.onnx"
|
||||
|
||||
if mode == "predict":
|
||||
'''
|
||||
1、如果想要进行检测完的图片的保存，利用r_image.save("img.jpg")即可保存，直接在predict.py里进行修改即可。
|
||||
2、如果想要获得预测框的坐标，可以进入yolo.detect_image函数，在绘图部分读取top，left，bottom，right这四个值。
|
||||
3、如果想要利用预测框截取下目标，可以进入yolo.detect_image函数，在绘图部分利用获取到的top，left，bottom，right这四个值
|
||||
在原图上利用矩阵的方式进行截取。
|
||||
4、如果想要在预测图上写额外的字，比如检测到的特定目标的数量，可以进入yolo.detect_image函数，在绘图部分对predicted_class进行判断，
|
||||
比如判断if predicted_class == 'car': 即可判断当前目标是否为车，然后记录数量即可。利用draw.text即可写字。
|
||||
'''
|
||||
while True:
|
||||
img = input('Input image filename:')
|
||||
# img/street.jpg
|
||||
# img/street_a3.jpg
|
||||
try:
|
||||
image = Image.open(img)
|
||||
except:
|
||||
print('Open Error! Try again!')
|
||||
continue
|
||||
else:
|
||||
r_image = yolo.detect_image(image, crop=crop, count=count)
|
||||
# r_image.show()
|
||||
r_image.save("duffision.png")
|
||||
|
||||
elif mode == "video":
|
||||
capture = cv2.VideoCapture(video_path)
|
||||
if video_save_path != "":
|
||||
fourcc = cv2.VideoWriter_fourcc(*'XVID')
|
||||
size = (int(capture.get(cv2.CAP_PROP_FRAME_WIDTH)), int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT)))
|
||||
out = cv2.VideoWriter(video_save_path, fourcc, video_fps, size)
|
||||
|
||||
ref, frame = capture.read()
|
||||
if not ref:
|
||||
raise ValueError("未能正确读取摄像头（视频），请注意是否正确安装摄像头（是否正确填写视频路径）。")
|
||||
|
||||
fps = 0.0
|
||||
while (True):
|
||||
t1 = time.time()
|
||||
# 读取某一帧
|
||||
ref, frame = capture.read()
|
||||
if not ref:
|
||||
break
|
||||
# 格式转变，BGRtoRGB
|
||||
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
# 转变成Image
|
||||
frame = Image.fromarray(np.uint8(frame))
|
||||
# 进行检测
|
||||
frame = np.array(yolo.detect_image(frame))
|
||||
# RGBtoBGR满足opencv显示格式
|
||||
frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
|
||||
|
||||
fps = (fps + (1. / (time.time() - t1))) / 2
|
||||
print("fps= %.2f" % (fps))
|
||||
frame = cv2.putText(frame, "fps= %.2f" % (fps), (0, 40), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
|
||||
|
||||
cv2.imshow("video", frame)
|
||||
c = cv2.waitKey(1) & 0xff
|
||||
if video_save_path != "":
|
||||
out.write(frame)
|
||||
|
||||
if c == 27:
|
||||
capture.release()
|
||||
break
|
||||
|
||||
print("Video Detection Done!")
|
||||
capture.release()
|
||||
if video_save_path != "":
|
||||
print("Save processed video to the path :" + video_save_path)
|
||||
out.release()
|
||||
cv2.destroyAllWindows()
|
||||
|
||||
elif mode == "fps":
|
||||
img = Image.open(fps_image_path)
|
||||
tact_time = yolo.get_FPS(img, test_interval)
|
||||
print(str(tact_time) + ' seconds, ' + str(1 / tact_time) + 'FPS, @batch_size 1')
|
||||
|
||||
elif mode == "dir_predict":
|
||||
import os
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
img_names = os.listdir(dir_origin_path)
|
||||
for img_name in tqdm(img_names):
|
||||
if img_name.lower().endswith(
|
||||
('.bmp', '.dib', '.png', '.jpg', '.jpeg', '.pbm', '.pgm', '.ppm', '.tif', '.tiff')):
|
||||
image_path = os.path.join(dir_origin_path, img_name)
|
||||
image = Image.open(image_path)
|
||||
r_image = yolo.detect_image(image)
|
||||
if not os.path.exists(dir_save_path):
|
||||
os.makedirs(dir_save_path)
|
||||
r_image.save(os.path.join(dir_save_path, img_name.replace(".jpg", ".png")), quality=95, subsampling=0)
|
||||
|
||||
elif mode == "heatmap":
|
||||
while True:
|
||||
img = input('Input image filename:')
|
||||
try:
|
||||
image = Image.open(img)
|
||||
except:
|
||||
print('Open Error! Try again!')
|
||||
continue
|
||||
else:
|
||||
yolo.detect_heatmap(image, heatmap_save_path)
|
||||
|
||||
elif mode == "export_onnx":
|
||||
yolo.convert_to_onnx(simplify, onnx_save_path)
|
||||
|
||||
else:
|
||||
raise AssertionError(
|
||||
"Please specify the correct mode: 'predict', 'video', 'fps', 'heatmap', 'export_onnx', 'dir_predict'.")
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
import time
|
||||
|
||||
import pyautogui
|
||||
import cv2
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from yolo import YOLO
|
||||
|
||||
if __name__ == "__main__":
|
||||
yolo = YOLO()
|
||||
# ----------------------------------------------------------------------------------------------------------#
|
||||
# mode用于指定测试的模式：
|
||||
# 'predict' 表示单张图片预测，如果想对预测过程进行修改，如保存图片，截取对象等，可以先看下方详细的注释
|
||||
# 'video' 表示视频检测，可调用摄像头或者视频进行检测，详情查看下方注释。
|
||||
# 'fps' 表示测试fps，使用的图片是img里面的street.jpg，详情查看下方注释。
|
||||
# 'dir_predict' 表示遍历文件夹进行检测并保存。默认遍历img文件夹，保存img_out文件夹，详情查看下方注释。
|
||||
# 'heatmap' 表示进行预测结果的热力图可视化，详情查看下方注释。
|
||||
# 'export_onnx' 表示将模型导出为onnx，需要pytorch1.7.1以上。
|
||||
# ----------------------------------------------------------------------------------------------------------#
|
||||
mode = "predict"
|
||||
# -------------------------------------------------------------------------#
|
||||
# crop 指定了是否在单张图片预测后对目标进行截取
|
||||
# count 指定了是否进行目标的计数
|
||||
# crop、count仅在mode='predict'时有效
|
||||
# -------------------------------------------------------------------------#
|
||||
crop = False
|
||||
count = False
|
||||
# ----------------------------------------------------------------------------------------------------------#
|
||||
# video_path 用于指定视频的路径，当video_path=0时表示检测摄像头
|
||||
# 想要检测视频，则设置如video_path = "xxx.mp4"即可，代表读取出根目录下的xxx.mp4文件。
|
||||
# video_save_path 表示视频保存的路径，当video_save_path=""时表示不保存
|
||||
# 想要保存视频，则设置如video_save_path = "yyy.mp4"即可，代表保存为根目录下的yyy.mp4文件。
|
||||
# video_fps 用于保存的视频的fps
|
||||
#
|
||||
# video_path、video_save_path和video_fps仅在mode='video'时有效
|
||||
# 保存视频时需要ctrl+c退出或者运行到最后一帧才会完成完整的保存步骤。
|
||||
# ----------------------------------------------------------------------------------------------------------#
|
||||
video_path = 0
|
||||
video_save_path = ""
|
||||
video_fps = 25.0
|
||||
# ----------------------------------------------------------------------------------------------------------#
|
||||
# test_interval 用于指定测量fps的时候，图片检测的次数。理论上test_interval越大，fps越准确。
|
||||
# fps_image_path 用于指定测试的fps图片
|
||||
#
|
||||
# test_interval和fps_image_path仅在mode='fps'有效
|
||||
# ----------------------------------------------------------------------------------------------------------#
|
||||
test_interval = 100
|
||||
fps_image_path = "img/street.jpg"
|
||||
# -------------------------------------------------------------------------#
|
||||
# dir_origin_path 指定了用于检测的图片的文件夹路径
|
||||
# dir_save_path 指定了检测完图片的保存路径
|
||||
#
|
||||
# dir_origin_path和dir_save_path仅在mode='dir_predict'时有效
|
||||
# -------------------------------------------------------------------------#
|
||||
dir_origin_path = "img/"
|
||||
dir_save_path = "img_out/"
|
||||
# -------------------------------------------------------------------------#
|
||||
# heatmap_save_path 热力图的保存路径，默认保存在model_data下
|
||||
#
|
||||
# heatmap_save_path仅在mode='heatmap'有效
|
||||
# -------------------------------------------------------------------------#
|
||||
heatmap_save_path = "model_data/heatmap_vision.png"
|
||||
# -------------------------------------------------------------------------#
|
||||
# simplify 使用Simplify onnx
|
||||
# onnx_save_path 指定了onnx的保存路径
|
||||
# -------------------------------------------------------------------------#
|
||||
simplify = True
|
||||
onnx_save_path = "model_data/models.onnx"
|
||||
|
||||
if mode == "predict":
|
||||
'''
|
||||
1、如果想要进行检测完的图片的保存，利用r_image.save("img.jpg")即可保存，直接在predict.py里进行修改即可。
|
||||
2、如果想要获得预测框的坐标，可以进入yolo.detect_image函数，在绘图部分读取top，left，bottom，right这四个值。
|
||||
3、如果想要利用预测框截取下目标，可以进入yolo.detect_image函数，在绘图部分利用获取到的top，left，bottom，right这四个值
|
||||
在原图上利用矩阵的方式进行截取。
|
||||
4、如果想要在预测图上写额外的字，比如检测到的特定目标的数量，可以进入yolo.detect_image函数，在绘图部分对predicted_class进行判断，
|
||||
比如判断if predicted_class == 'car': 即可判断当前目标是否为车，然后记录数量即可。利用draw.text即可写字。
|
||||
'''
|
||||
while True:
|
||||
# img = pyautogui.screenshot(region=[300, 50, 200, 100]) # 分别代表：左上角坐标，宽高
|
||||
# img = pyautogui.screenshot() # 分别代表：左上角坐标，宽高
|
||||
# 对获取的图片转换成二维矩阵形式，后再将RGB转成BGR
|
||||
# 因为imshow,默认通道顺序是BGR，而pyautogui默认是RGB所以要转换一下，不然会有点问题
|
||||
# img = cv2.cvtColor(np.asarray(img), cv2.COLOR_RGB2BGR)
|
||||
# img/street.jpg
|
||||
# img/street_a3.jpg
|
||||
try:
|
||||
time.sleep(0.3)
|
||||
# image = Image.fromarray(np.asarray(pyautogui.screenshot(region=[1920/2, 300, 1920/2, 1080])))
|
||||
image = Image.fromarray(np.asarray(pyautogui.screenshot()))
|
||||
except:
|
||||
print('Open Error! Try again!')
|
||||
continue
|
||||
else:
|
||||
r_image = yolo.detect_image(image, crop=crop, count=count)
|
||||
img = cv2.cvtColor(np.asarray(r_image), cv2.COLOR_RGB2BGR)
|
||||
# img = cv2.resize(img, dsize=(1600, 860)) # (宽度,高度)
|
||||
img = cv2.resize(img, dsize=(1920, 1080)) # (宽度,高度)
|
||||
cv2.imshow("screen", img)
|
||||
# time.sleep(1)
|
||||
cv2.waitKey(1)
|
||||
|
||||
c = cv2.waitKey(1) & 0xff
|
||||
# print(c)
|
||||
if c == 113:
|
||||
break
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
scipy==1.2.1
|
||||
numpy==1.17.0
|
||||
matplotlib==3.1.2
|
||||
opencv_python==4.1.2.30
|
||||
torch==1.2.0
|
||||
torchvision==0.4.0
|
||||
tqdm==4.60.0
|
||||
Pillow==8.2.0
|
||||
h5py==2.10.0
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
# --------------------------------------------#
|
||||
# 该部分代码用于看网络结构
|
||||
# --------------------------------------------#
|
||||
import torch
|
||||
# from thop import clever_format, profile
|
||||
from torchsummary import summary
|
||||
|
||||
from nets.yolo import YoloBody
|
||||
|
||||
if __name__ == "__main__":
|
||||
input_shape = [416, 416]
|
||||
anchors_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]
|
||||
num_classes = 80
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
m = YoloBody(anchors_mask, num_classes)
|
||||
print(m)
|
||||
print('-' * 80)
|
||||
|
||||
m = m.to(device)
|
||||
summary(m, (3, input_shape[0], input_shape[1]))
|
||||
|
||||
# dummy_input = torch.randn(1, 3, input_shape[0], input_shape[1]).to(device)
|
||||
# flops, params = profile(m.to(device), (dummy_input,), verbose=False)
|
||||
# --------------------------------------------------------#
|
||||
# flops * 2是因为profile没有将卷积作为两个operations
|
||||
# 有些论文将卷积算乘法、加法两个operations。此时乘2
|
||||
# 有些论文只考虑乘法的运算次数，忽略加法。此时不乘2
|
||||
# 本代码选择乘2，参考YOLOX。
|
||||
# --------------------------------------------------------#
|
||||
# flops = flops * 2
|
||||
# flops, params = clever_format([flops, params], "%.3f")
|
||||
# print('Total GFLOPS: %s' % (flops))
|
||||
# print('Total params: %s' % (params))
|
||||
|
|
@ -0,0 +1,225 @@
|
|||
"""
|
||||
Training code for Adversarial patch training
|
||||
|
||||
|
||||
"""
|
||||
|
||||
import PIL
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
|
||||
# import load_data
|
||||
from tqdm import tqdm
|
||||
|
||||
from load_data import * # 可能导致多次导入问题？
|
||||
import gc
|
||||
import matplotlib.pyplot as plt
|
||||
from torch import autograd
|
||||
from torchvision import transforms
|
||||
|
||||
import subprocess
|
||||
|
||||
import patch_config
|
||||
import sys
|
||||
import time
|
||||
|
||||
from yolo import YOLO
|
||||
|
||||
|
||||
class PatchTrainer(object):
|
||||
def __init__(self, mode):
|
||||
self.config = patch_config.patch_configs[mode]() # 获取对应的配置类
|
||||
|
||||
# self.darknet_model = Darknet(self.config.cfgfile) # 加载yolo模型
|
||||
# self.darknet_model.load_weights(self.config.weightfile) # 默认 YOLOv2 MS COCO weights， person编号是0
|
||||
self.darknet_model = YOLO().net
|
||||
self.darknet_model = self.darknet_model.eval().cuda() # TODO: Why eval?
|
||||
self.patch_applier = PatchApplier().cuda() # 对图像应用对抗补丁
|
||||
self.patch_transformer = PatchTransformer().cuda() # 变换补丁到指定大小并产生抖动
|
||||
# self.prob_extractor = MaxProbExtractor(0, 80, self.config).cuda() # 提取最大类别概率
|
||||
self.prob_extractor = MaxProbExtractor(0, 1, self.config).cuda() # 提取最大类别概率
|
||||
self.nps_calculator = NPSCalculator(self.config.printfile, self.config.patch_size).cuda() # 不可打印分数
|
||||
self.total_variation = TotalVariation().cuda() # 计算补丁的所有变化程度
|
||||
|
||||
self.writer = self.init_tensorboard(mode)
|
||||
|
||||
def init_tensorboard(self, name=None):
|
||||
subprocess.Popen(['tensorboard', '--logdir=runs'])
|
||||
if name is not None:
|
||||
time_str = time.strftime("%Y%m%d-%H%M%S")
|
||||
return SummaryWriter(f'runs/{time_str}_{name}')
|
||||
else:
|
||||
return SummaryWriter()
|
||||
|
||||
def train(self):
|
||||
"""
|
||||
Optimize a patch to generate an adversarial example.
|
||||
:return: Nothing
|
||||
"""
|
||||
|
||||
img_size = self.darknet_model.height # 416
|
||||
# print('batch_size:',batch_size)
|
||||
batch_size = self.config.batch_size # 8
|
||||
n_epochs = 200
|
||||
# n_epochs = 5
|
||||
# max_lab = 20 # label的最大长度
|
||||
max_lab = 8
|
||||
|
||||
time_str = time.strftime("%Y%m%d-%H%M%S")
|
||||
|
||||
# Generate stating point
|
||||
# adv_patch_cpu = self.generate_patch("gray") # 生成一个灰图，初始化为0.5
|
||||
adv_patch_cpu = self.read_image("saved_patches/patchnew0.jpg")
|
||||
|
||||
adv_patch_cpu.requires_grad_(True)
|
||||
|
||||
train_loader = torch.utils.data.DataLoader(
|
||||
InriaDataset(self.config.img_dir, self.config.lab_dir, max_lab, img_size,
|
||||
shuffle=True),
|
||||
batch_size=batch_size,
|
||||
shuffle=True,
|
||||
num_workers=0) # 与 from load_data import * 搭配导致多少导入？
|
||||
self.epoch_length = len(train_loader)
|
||||
print(f'One epoch is {len(train_loader)}')
|
||||
|
||||
optimizer = optim.Adam([adv_patch_cpu], lr=self.config.start_learning_rate, amsgrad=True) # 更新的是那个补丁
|
||||
scheduler = self.config.scheduler_factory(optimizer) # ICLR-2018年最佳论文提出的Adam改进版Amsgrad
|
||||
|
||||
et0 = time.time()
|
||||
for epoch in range(n_epochs):
|
||||
ep_det_loss = 0
|
||||
ep_nps_loss = 0
|
||||
ep_tv_loss = 0
|
||||
ep_loss = 0
|
||||
bt0 = time.time()
|
||||
for i_batch, (img_batch, lab_batch) in tqdm(enumerate(train_loader), desc=f'Running epoch {epoch}',
|
||||
total=self.epoch_length):
|
||||
with autograd.detect_anomaly(): # 1.运行前向时开启异常检测功能，则在反向时会打印引起反向失败的前向操作堆栈 2.反向计算出现“nan”时引发异常
|
||||
img_batch = img_batch.cuda() # 8, 3, 416, 416
|
||||
lab_batch = lab_batch.cuda() # 8, 14, 5 为什么要把人数的标签补到14?
|
||||
# print('TRAINING EPOCH %i, BATCH %i'%(epoch, i_batch))
|
||||
adv_patch = adv_patch_cpu.cuda() # 3, 300, 300
|
||||
adv_batch_t = self.patch_transformer(adv_patch, lab_batch, img_size, do_rotate=True, rand_loc=False)
|
||||
p_img_batch = self.patch_applier(img_batch, adv_batch_t)
|
||||
p_img_batch = F.interpolate(p_img_batch,
|
||||
(self.darknet_model.height, self.darknet_model.width)) # 确保和图片大小一致
|
||||
|
||||
# print('++++++++++++p_img_batch:+++++++++++++',p_img_batch.shape)
|
||||
img = p_img_batch[1, :, :, ]
|
||||
img = transforms.ToPILImage()(img.detach().cpu())
|
||||
# img.show()
|
||||
|
||||
outputs = self.darknet_model(p_img_batch) # 输入8，3，416，416 输出8，425， 13， 13 ,其中425是5*(5+80)
|
||||
max_prob = 0
|
||||
nps = 0
|
||||
tv = 0
|
||||
for l in range(len(outputs)): # 三组不同分辨率大小的输出特征分别计算
|
||||
output = outputs[l]
|
||||
max_prob += self.prob_extractor(output)
|
||||
nps += self.nps_calculator(adv_patch)
|
||||
tv += self.total_variation(adv_patch)
|
||||
|
||||
nps_loss = nps * 0.01
|
||||
tv_loss = tv * 2.5
|
||||
det_loss = torch.mean(max_prob) # 把人的置值度当成损失
|
||||
loss = det_loss + nps_loss + torch.max(tv_loss, torch.tensor(0.1).cuda())
|
||||
|
||||
ep_det_loss += det_loss.detach().cpu().numpy()
|
||||
ep_nps_loss += nps_loss.detach().cpu().numpy()
|
||||
ep_tv_loss += tv_loss.detach().cpu().numpy()
|
||||
ep_loss += loss
|
||||
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
optimizer.zero_grad()
|
||||
adv_patch_cpu.data.clamp_(0, 1) # keep patch in image range
|
||||
|
||||
bt1 = time.time()
|
||||
if i_batch % 5 == 0:
|
||||
iteration = self.epoch_length * epoch + i_batch
|
||||
|
||||
self.writer.add_scalar('total_loss', loss.detach().cpu().numpy(), iteration)
|
||||
self.writer.add_scalar('loss/det_loss', det_loss.detach().cpu().numpy(), iteration)
|
||||
self.writer.add_scalar('loss/nps_loss', nps_loss.detach().cpu().numpy(), iteration)
|
||||
self.writer.add_scalar('loss/tv_loss', tv_loss.detach().cpu().numpy(), iteration)
|
||||
self.writer.add_scalar('misc/epoch', epoch, iteration)
|
||||
self.writer.add_scalar('misc/learning_rate', optimizer.param_groups[0]["lr"], iteration)
|
||||
|
||||
self.writer.add_image('patch', adv_patch_cpu, iteration)
|
||||
if i_batch + 1 >= len(train_loader):
|
||||
print('\n')
|
||||
else:
|
||||
del adv_batch_t, output, max_prob, det_loss, p_img_batch, nps_loss, tv_loss, loss
|
||||
torch.cuda.empty_cache()
|
||||
bt0 = time.time()
|
||||
et1 = time.time()
|
||||
ep_det_loss = ep_det_loss / len(train_loader)
|
||||
ep_nps_loss = ep_nps_loss / len(train_loader)
|
||||
ep_tv_loss = ep_tv_loss / len(train_loader)
|
||||
ep_loss = ep_loss / len(train_loader)
|
||||
|
||||
# im = transforms.ToPILImage('RGB')(adv_patch_cpu)
|
||||
# plt.imshow(im)
|
||||
# plt.savefig(f'pics/{time_str}_{self.config.patch_name}_{epoch}.png')
|
||||
|
||||
scheduler.step(ep_loss)
|
||||
if True:
|
||||
print(' EPOCH NR: ', epoch),
|
||||
print('EPOCH LOSS: ', ep_loss)
|
||||
print(' DET LOSS: ', ep_det_loss)
|
||||
print(' NPS LOSS: ', ep_nps_loss)
|
||||
print(' TV LOSS: ', ep_tv_loss)
|
||||
print('EPOCH TIME: ', et1 - et0)
|
||||
# im = transforms.ToPILImage('RGB')(adv_patch_cpu)
|
||||
# plt.imshow(im)
|
||||
# plt.show()
|
||||
# im.save("saved_patches/patchnew1.jpg")
|
||||
im = transforms.ToPILImage('RGB')(adv_patch_cpu)
|
||||
if epoch >= 3:
|
||||
im.save(f"saved_patches/patchnew1_t1_{epoch}_{time_str}.jpg")
|
||||
del adv_batch_t, output, max_prob, det_loss, p_img_batch, nps_loss, tv_loss, loss
|
||||
torch.cuda.empty_cache()
|
||||
et0 = time.time()
|
||||
|
||||
def generate_patch(self, type):
|
||||
"""
|
||||
Generate a random patch as a starting point for optimization.
|
||||
|
||||
:param type: Can be 'gray' or 'random'. Whether or not generate a gray or a random patch.
|
||||
:return:
|
||||
"""
|
||||
if type == 'gray':
|
||||
adv_patch_cpu = torch.full((3, self.config.patch_size, self.config.patch_size), 0.5)
|
||||
elif type == 'random':
|
||||
adv_patch_cpu = torch.rand((3, self.config.patch_size, self.config.patch_size))
|
||||
|
||||
return adv_patch_cpu
|
||||
|
||||
def read_image(self, path):
|
||||
"""
|
||||
Read an input image to be used as a patch
|
||||
|
||||
:param path: Path to the image to be read.
|
||||
:return: Returns the transformed patch as a pytorch Tensor.
|
||||
"""
|
||||
patch_img = Image.open(path).convert('RGB')
|
||||
tf = transforms.Resize((self.config.patch_size, self.config.patch_size))
|
||||
patch_img = tf(patch_img)
|
||||
tf = transforms.ToTensor()
|
||||
|
||||
adv_patch_cpu = tf(patch_img)
|
||||
return adv_patch_cpu
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print('You need to supply (only) a configuration mode.')
|
||||
print('Possible modes are:')
|
||||
print(patch_config.patch_configs) # 一般传入paper_obj
|
||||
|
||||
# print('sys.argv:',sys.argv)
|
||||
trainer = PatchTrainer(sys.argv[1])
|
||||
trainer.train()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1 @@
|
|||
#
|
||||
|
|
@ -0,0 +1,241 @@
|
|||
import datetime
|
||||
import os
|
||||
|
||||
import torch
|
||||
import matplotlib
|
||||
|
||||
import scipy.signal
|
||||
from matplotlib import pyplot as plt
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
|
||||
import shutil
|
||||
import numpy as np
|
||||
|
||||
from PIL import Image
|
||||
from tqdm import tqdm
|
||||
from .utils import cvtColor, preprocess_input, resize_image
|
||||
from .utils_bbox import DecodeBox
|
||||
from .utils_map import get_coco_map, get_map
|
||||
|
||||
matplotlib.use('Agg')
|
||||
|
||||
|
||||
class LossHistory():
|
||||
def __init__(self, log_dir, model, input_shape):
|
||||
self.log_dir = log_dir
|
||||
self.losses = []
|
||||
self.val_loss = []
|
||||
|
||||
os.makedirs(self.log_dir)
|
||||
self.writer = SummaryWriter(self.log_dir)
|
||||
try:
|
||||
dummy_input = torch.randn(2, 3, input_shape[0], input_shape[1])
|
||||
self.writer.add_graph(model, dummy_input)
|
||||
except:
|
||||
pass
|
||||
|
||||
def append_loss(self, epoch, loss, val_loss):
|
||||
if not os.path.exists(self.log_dir):
|
||||
os.makedirs(self.log_dir)
|
||||
|
||||
self.losses.append(loss)
|
||||
self.val_loss.append(val_loss)
|
||||
|
||||
with open(os.path.join(self.log_dir, "epoch_loss.txt"), 'a') as f:
|
||||
f.write(str(loss))
|
||||
f.write("\n")
|
||||
with open(os.path.join(self.log_dir, "epoch_val_loss.txt"), 'a') as f:
|
||||
f.write(str(val_loss))
|
||||
f.write("\n")
|
||||
|
||||
self.writer.add_scalar('loss', loss, epoch)
|
||||
self.writer.add_scalar('val_loss', val_loss, epoch)
|
||||
self.loss_plot()
|
||||
|
||||
def loss_plot(self):
|
||||
iters = range(len(self.losses))
|
||||
|
||||
plt.figure()
|
||||
plt.plot(iters, self.losses, 'red', linewidth=2, label='train loss')
|
||||
plt.plot(iters, self.val_loss, 'coral', linewidth=2, label='val loss')
|
||||
try:
|
||||
if len(self.losses) < 25:
|
||||
num = 5
|
||||
else:
|
||||
num = 15
|
||||
|
||||
plt.plot(iters, scipy.signal.savgol_filter(self.losses, num, 3), 'green', linestyle='--', linewidth=2,
|
||||
label='smooth train loss')
|
||||
plt.plot(iters, scipy.signal.savgol_filter(self.val_loss, num, 3), '#8B4513', linestyle='--', linewidth=2,
|
||||
label='smooth val loss')
|
||||
except:
|
||||
pass
|
||||
|
||||
plt.grid(True)
|
||||
plt.xlabel('Epoch')
|
||||
plt.ylabel('Loss')
|
||||
plt.legend(loc="upper right")
|
||||
|
||||
plt.savefig(os.path.join(self.log_dir, "epoch_loss.png"))
|
||||
|
||||
plt.cla()
|
||||
plt.close("all")
|
||||
|
||||
|
||||
class EvalCallback():
|
||||
def __init__(self, net, input_shape, anchors, anchors_mask, class_names, num_classes, val_lines, log_dir, cuda, \
|
||||
map_out_path=".temp_map_out", max_boxes=100, confidence=0.05, nms_iou=0.5, letterbox_image=True,
|
||||
MINOVERLAP=0.5, eval_flag=True, period=1):
|
||||
super(EvalCallback, self).__init__()
|
||||
|
||||
self.net = net
|
||||
self.input_shape = input_shape
|
||||
self.anchors = anchors
|
||||
self.anchors_mask = anchors_mask
|
||||
self.class_names = class_names
|
||||
self.num_classes = num_classes
|
||||
self.val_lines = val_lines
|
||||
self.log_dir = log_dir
|
||||
self.cuda = cuda
|
||||
self.map_out_path = map_out_path
|
||||
self.max_boxes = max_boxes
|
||||
self.confidence = confidence
|
||||
self.nms_iou = nms_iou
|
||||
self.letterbox_image = letterbox_image
|
||||
self.MINOVERLAP = MINOVERLAP
|
||||
self.eval_flag = eval_flag
|
||||
self.period = period
|
||||
|
||||
self.bbox_util = DecodeBox(self.anchors, self.num_classes, (self.input_shape[0], self.input_shape[1]),
|
||||
self.anchors_mask)
|
||||
|
||||
self.maps = [0]
|
||||
self.epoches = [0]
|
||||
if self.eval_flag:
|
||||
with open(os.path.join(self.log_dir, "epoch_map.txt"), 'a') as f:
|
||||
f.write(str(0))
|
||||
f.write("\n")
|
||||
|
||||
def get_map_txt(self, image_id, image, class_names, map_out_path):
|
||||
f = open(os.path.join(map_out_path, "detection-results/" + image_id + ".txt"), "w", encoding='utf-8')
|
||||
image_shape = np.array(np.shape(image)[0:2])
|
||||
# ---------------------------------------------------------#
|
||||
# 在这里将图像转换成RGB图像，防止灰度图在预测时报错。
|
||||
# 代码仅仅支持RGB图像的预测，所有其它类型的图像都会转化成RGB
|
||||
# ---------------------------------------------------------#
|
||||
image = cvtColor(image)
|
||||
# ---------------------------------------------------------#
|
||||
# 给图像增加灰条，实现不失真的resize
|
||||
# 也可以直接resize进行识别
|
||||
# ---------------------------------------------------------#
|
||||
image_data = resize_image(image, (self.input_shape[1], self.input_shape[0]), self.letterbox_image)
|
||||
# ---------------------------------------------------------#
|
||||
# 添加上batch_size维度
|
||||
# ---------------------------------------------------------#
|
||||
image_data = np.expand_dims(np.transpose(preprocess_input(np.array(image_data, dtype='float32')), (2, 0, 1)), 0)
|
||||
|
||||
with torch.no_grad():
|
||||
images = torch.from_numpy(image_data)
|
||||
if self.cuda:
|
||||
images = images.cuda()
|
||||
# ---------------------------------------------------------#
|
||||
# 将图像输入网络当中进行预测！
|
||||
# ---------------------------------------------------------#
|
||||
outputs = self.net(images)
|
||||
outputs = self.bbox_util.decode_box(outputs)
|
||||
# ---------------------------------------------------------#
|
||||
# 将预测框进行堆叠，然后进行非极大抑制
|
||||
# ---------------------------------------------------------#
|
||||
results = self.bbox_util.non_max_suppression(torch.cat(outputs, 1), self.num_classes, self.input_shape,
|
||||
image_shape, self.letterbox_image, conf_thres=self.confidence,
|
||||
nms_thres=self.nms_iou)
|
||||
|
||||
if results[0] is None:
|
||||
return
|
||||
|
||||
top_label = np.array(results[0][:, 6], dtype='int32')
|
||||
top_conf = results[0][:, 4] * results[0][:, 5]
|
||||
top_boxes = results[0][:, :4]
|
||||
|
||||
top_100 = np.argsort(top_label)[::-1][:self.max_boxes]
|
||||
top_boxes = top_boxes[top_100]
|
||||
top_conf = top_conf[top_100]
|
||||
top_label = top_label[top_100]
|
||||
|
||||
for i, c in list(enumerate(top_label)):
|
||||
predicted_class = self.class_names[int(c)]
|
||||
box = top_boxes[i]
|
||||
score = str(top_conf[i])
|
||||
|
||||
top, left, bottom, right = box
|
||||
if predicted_class not in class_names:
|
||||
continue
|
||||
|
||||
f.write("%s %s %s %s %s %s\n" % (
|
||||
predicted_class, score[:6], str(int(left)), str(int(top)), str(int(right)), str(int(bottom))))
|
||||
|
||||
f.close()
|
||||
return
|
||||
|
||||
def on_epoch_end(self, epoch, model_eval):
|
||||
if epoch % self.period == 0 and self.eval_flag:
|
||||
self.net = model_eval
|
||||
if not os.path.exists(self.map_out_path):
|
||||
os.makedirs(self.map_out_path)
|
||||
if not os.path.exists(os.path.join(self.map_out_path, "ground-truth")):
|
||||
os.makedirs(os.path.join(self.map_out_path, "ground-truth"))
|
||||
if not os.path.exists(os.path.join(self.map_out_path, "detection-results")):
|
||||
os.makedirs(os.path.join(self.map_out_path, "detection-results"))
|
||||
print("Get map.")
|
||||
for annotation_line in tqdm(self.val_lines):
|
||||
line = annotation_line.split()
|
||||
image_id = os.path.basename(line[0]).split('.')[0]
|
||||
# ------------------------------#
|
||||
# 读取图像并转换成RGB图像
|
||||
# ------------------------------#
|
||||
image = Image.open(line[0])
|
||||
# ------------------------------#
|
||||
# 获得预测框
|
||||
# ------------------------------#
|
||||
gt_boxes = np.array([np.array(list(map(int, box.split(',')))) for box in line[1:]])
|
||||
# ------------------------------#
|
||||
# 获得预测txt
|
||||
# ------------------------------#
|
||||
self.get_map_txt(image_id, image, self.class_names, self.map_out_path)
|
||||
|
||||
# ------------------------------#
|
||||
# 获得真实框txt
|
||||
# ------------------------------#
|
||||
with open(os.path.join(self.map_out_path, "ground-truth/" + image_id + ".txt"), "w") as new_f:
|
||||
for box in gt_boxes:
|
||||
left, top, right, bottom, obj = box
|
||||
obj_name = self.class_names[obj]
|
||||
new_f.write("%s %s %s %s %s\n" % (obj_name, left, top, right, bottom))
|
||||
|
||||
print("Calculate Map.")
|
||||
try:
|
||||
temp_map = get_coco_map(class_names=self.class_names, path=self.map_out_path)[1]
|
||||
except:
|
||||
temp_map = get_map(self.MINOVERLAP, False, path=self.map_out_path)
|
||||
self.maps.append(temp_map)
|
||||
self.epoches.append(epoch)
|
||||
|
||||
with open(os.path.join(self.log_dir, "epoch_map.txt"), 'a') as f:
|
||||
f.write(str(temp_map))
|
||||
f.write("\n")
|
||||
|
||||
plt.figure()
|
||||
plt.plot(self.epoches, self.maps, 'red', linewidth=2, label='train map')
|
||||
|
||||
plt.grid(True)
|
||||
plt.xlabel('Epoch')
|
||||
plt.ylabel('Map %s' % str(self.MINOVERLAP))
|
||||
plt.title('A Map Curve')
|
||||
plt.legend(loc="upper right")
|
||||
|
||||
plt.savefig(os.path.join(self.log_dir, "epoch_map.png"))
|
||||
plt.cla()
|
||||
plt.close("all")
|
||||
|
||||
print("Get map done.")
|
||||
shutil.rmtree(self.map_out_path)
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
import cv2
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torch.utils.data.dataset import Dataset
|
||||
|
||||
from utils.utils import cvtColor, preprocess_input
|
||||
|
||||
|
||||
class YoloDataset(Dataset):
|
||||
def __init__(self, annotation_lines, input_shape, num_classes, train):
|
||||
super(YoloDataset, self).__init__()
|
||||
self.annotation_lines = annotation_lines # 记录训练集或测试集的文件的路径，这个是可以全部载入的
|
||||
self.input_shape = input_shape # 这里是 [416, 416]
|
||||
self.num_classes = num_classes # 这里是20
|
||||
self.length = len(self.annotation_lines) # 数据的数量
|
||||
self.train = train # 是否是训练集的标记
|
||||
|
||||
def __len__(self):
|
||||
return self.length
|
||||
|
||||
def __getitem__(self, index):
|
||||
index = index % self.length
|
||||
# ---------------------------------------------------#
|
||||
# 训练时进行数据的随机增强
|
||||
# 验证时不进行数据的随机增强
|
||||
# ---------------------------------------------------#
|
||||
image, box = self.get_random_data(self.annotation_lines[index], self.input_shape[0:2],
|
||||
random=self.train) # 自定义的数据增强
|
||||
image = np.transpose(preprocess_input(np.array(image, dtype=np.float32)), (2, 0, 1)) # 像素值归到0~1之间，然后变换坐标轴
|
||||
box = np.array(box, dtype=np.float32) # 转为numpy。np中常用的是创建新类型的array。
|
||||
if len(box) != 0:
|
||||
box[:, [0, 2]] = box[:, [0, 2]] / self.input_shape[1] # 把框的坐标归一化
|
||||
box[:, [1, 3]] = box[:, [1, 3]] / self.input_shape[0]
|
||||
|
||||
box[:, 2:4] = box[:, 2:4] - box[:, 0:2] # box第0，1维记录中心点 box第2，3维记录宽高
|
||||
box[:, 0:2] = box[:, 0:2] + box[:, 2:4] / 2 # box第0，1维记录中心点
|
||||
return image, box
|
||||
|
||||
def rand(self, a=0, b=1):
|
||||
return np.random.rand() * (b - a) + a
|
||||
|
||||
def get_random_data(self, annotation_line, input_shape, jitter=.3, hue=.1, sat=0.7, val=0.4, random=True):
|
||||
line = annotation_line.split() # 以空格、回车等分隔字符串
|
||||
# ------------------------------#
|
||||
# 读取图像并转换成RGB图像
|
||||
# ------------------------------#
|
||||
image = Image.open(line[0]) # line[0] 是图片的地址
|
||||
image = cvtColor(image) # 这里啥也没干
|
||||
# ------------------------------#
|
||||
# 获得图像的高宽与目标高宽
|
||||
# ------------------------------#
|
||||
iw, ih = image.size # 获取图像的原始尺寸
|
||||
h, w = input_shape
|
||||
# ------------------------------#
|
||||
# 获得预测框
|
||||
# ------------------------------#
|
||||
box = np.array([np.array(list(map(int, box.split(',')))) for box in line[1:]]) # 从python二维矩阵转到 numpy二维矩阵
|
||||
|
||||
if not random: # 没进入这里面
|
||||
scale = min(w / iw, h / ih)
|
||||
nw = int(iw * scale)
|
||||
nh = int(ih * scale)
|
||||
dx = (w - nw) // 2
|
||||
dy = (h - nh) // 2
|
||||
|
||||
# ---------------------------------#
|
||||
# 将图像多余的部分加上灰条
|
||||
# ---------------------------------#
|
||||
image = image.resize((nw, nh), Image.BICUBIC)
|
||||
new_image = Image.new('RGB', (w, h), (128, 128, 128))
|
||||
new_image.paste(image, (dx, dy))
|
||||
image_data = np.array(new_image, np.float32)
|
||||
|
||||
# ---------------------------------#
|
||||
# 对真实框进行调整
|
||||
# ---------------------------------#
|
||||
if len(box) > 0:
|
||||
np.random.shuffle(box)
|
||||
box[:, [0, 2]] = box[:, [0, 2]] * nw / iw + dx
|
||||
box[:, [1, 3]] = box[:, [1, 3]] * nh / ih + dy
|
||||
box[:, 0:2][box[:, 0:2] < 0] = 0
|
||||
box[:, 2][box[:, 2] > w] = w
|
||||
box[:, 3][box[:, 3] > h] = h
|
||||
box_w = box[:, 2] - box[:, 0]
|
||||
box_h = box[:, 3] - box[:, 1]
|
||||
box = box[np.logical_and(box_w > 1, box_h > 1)] # discard invalid box
|
||||
|
||||
return image_data, box
|
||||
|
||||
# ------------------------------------------#
|
||||
# 对原始图像进行缩放并且进行长和宽的扭曲
|
||||
# ------------------------------------------#
|
||||
new_ar = iw / ih * self.rand(1 - jitter, 1 + jitter) / self.rand(1 - jitter, 1 + jitter) # (iw*随机) / (ih*随机)
|
||||
scale = self.rand(.25, 2) # 随机一个缩放比例
|
||||
if new_ar < 1: # 原图高大
|
||||
nh = int(scale * h) # 新图先缩放高
|
||||
nw = int(nh * new_ar)
|
||||
else: # 原图宽大
|
||||
nw = int(scale * w) # 新的宽从 预期宽中 乘以随机的比例
|
||||
nh = int(nw / new_ar) # 新的宽、高比，也是 new_ar, 也就是也是宽大
|
||||
image = image.resize((nw, nh), Image.BICUBIC)
|
||||
|
||||
# ------------------------------------------#
|
||||
# 将图像多余的部分加上灰条
|
||||
# ------------------------------------------#
|
||||
dx = int(self.rand(0, w - nw)) # 在(0, w - nw)找一个点作为新图的放置点
|
||||
dy = int(self.rand(0, h - nh))
|
||||
new_image = Image.new('RGB', (w, h), (128, 128, 128)) # 画一个 412, 412大小的灰图
|
||||
new_image.paste(image, (dx, dy)) # 在这里看看两者的区别
|
||||
image = new_image
|
||||
|
||||
# ------------------------------------------#
|
||||
# 翻转图像
|
||||
# ------------------------------------------#
|
||||
flip = self.rand() < .5
|
||||
if flip:
|
||||
image = image.transpose(Image.FLIP_LEFT_RIGHT)
|
||||
|
||||
image_data = np.array(image, np.uint8)
|
||||
# ---------------------------------#
|
||||
# 对图像进行色域变换
|
||||
# 计算色域变换的参数
|
||||
# ---------------------------------#
|
||||
r = np.random.uniform(-1, 1, 3) * [hue, sat, val] + 1
|
||||
# ---------------------------------#
|
||||
# 将图像转到HSV上
|
||||
# ---------------------------------#
|
||||
hue, sat, val = cv2.split(cv2.cvtColor(image_data, cv2.COLOR_RGB2HSV))
|
||||
dtype = image_data.dtype
|
||||
# ---------------------------------#
|
||||
# 应用变换
|
||||
# ---------------------------------#
|
||||
x = np.arange(0, 256, dtype=r.dtype)
|
||||
lut_hue = ((x * r[0]) % 180).astype(dtype)
|
||||
lut_sat = np.clip(x * r[1], 0, 255).astype(dtype)
|
||||
lut_val = np.clip(x * r[2], 0, 255).astype(dtype)
|
||||
# LUT是look-up table查找表的意思,cv2.LUT(src, lut, dst=None)的作用是对输入的src执行查找表lut转换
|
||||
image_data = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val)))
|
||||
image_data = cv2.cvtColor(image_data, cv2.COLOR_HSV2RGB) # image_data在这里还是unit8类型
|
||||
|
||||
# ---------------------------------#
|
||||
# 对真实框进行调整
|
||||
# ---------------------------------#
|
||||
if len(box) > 0: # 如果有box
|
||||
np.random.shuffle(box)
|
||||
box[:, [0, 2]] = box[:, [0, 2]] * nw / iw + dx # 所有行的第0列和2列，也就是 x 坐标， 除以iw找到占原图的比例，再乘以nw，是新图的比例，再加dx是新图中的偏移
|
||||
box[:, [1, 3]] = box[:, [1, 3]] * nh / ih + dy
|
||||
if flip:
|
||||
box[:, [0, 2]] = w - box[:, [2, 0]] # 如果有水平翻转，则x坐标变换为416-x，并且x0 和 x1的位置互换一下
|
||||
box[:, 0:2][box[:, 0:2] < 0] = 0 # 对于左上角的点在图像外（小于0），则把对应的位置的坐标置为0 # 右下角的点不会小于0吗？
|
||||
box[:, 2][box[:, 2] > w] = w # 对于右下角的横坐标点超出图的，则置为w # 右下角不会超出图吗？
|
||||
box[:, 3][box[:, 3] > h] = h # 对于右下角的纵坐标点超出图的，则置为h
|
||||
box_w = box[:, 2] - box[:, 0]
|
||||
box_h = box[:, 3] - box[:, 1]
|
||||
box = box[np.logical_and(box_w > 1, box_h > 1)] # 多余的检查？如果宽、高大于至少1，则保留下来
|
||||
|
||||
return image_data, box # box依然是左上角和右下角的形式
|
||||
|
||||
|
||||
# DataLoader中collate_fn使用
|
||||
def yolo_dataset_collate(batch):
|
||||
images = [] # 这是是一个batch大小的列表，每一项是 image_data, box。需要把image放一堆，box放一堆
|
||||
bboxes = []
|
||||
for img, box in batch:
|
||||
images.append(img) # images在这里已经是0~1的float32类型了
|
||||
bboxes.append(box)
|
||||
images = torch.from_numpy(np.array(images)).type(torch.FloatTensor) # 转换为 batch_size, C, H, W 的数据
|
||||
bboxes = [torch.from_numpy(ann).type(torch.FloatTensor) for ann in bboxes] # 转换为一个列表，每个元素是一组二维Tensor
|
||||
return images, bboxes
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
|
||||
# ---------------------------------------------------------#
|
||||
# 将图像转换成RGB图像，防止灰度图在预测时报错。
|
||||
# 代码仅仅支持RGB图像的预测，所有其它类型的图像都会转化成RGB
|
||||
# ---------------------------------------------------------#
|
||||
def cvtColor(image):
|
||||
if len(np.shape(image)) == 3 and np.shape(image)[2] == 3:
|
||||
return image
|
||||
else:
|
||||
image = image.convert('RGB')
|
||||
return image
|
||||
|
||||
# ---------------------------------------------------#
|
||||
|
||||
|
||||
# 对输入图像进行resize
|
||||
# ---------------------------------------------------#
|
||||
def resize_image(image, size, letterbox_image):
|
||||
iw, ih = image.size
|
||||
w, h = size
|
||||
if letterbox_image:
|
||||
scale = min(w / iw, h / ih)
|
||||
nw = int(iw * scale)
|
||||
nh = int(ih * scale)
|
||||
|
||||
image = image.resize((nw, nh), Image.BICUBIC)
|
||||
new_image = Image.new('RGB', size, (128, 128, 128))
|
||||
new_image.paste(image, ((w - nw) // 2, (h - nh) // 2))
|
||||
else:
|
||||
new_image = image.resize((w, h), Image.BICUBIC) # 这里直接用了缩放，而不是加灰条的形式
|
||||
return new_image
|
||||
|
||||
|
||||
# ---------------------------------------------------#
|
||||
# 获得类
|
||||
# ---------------------------------------------------#
|
||||
def get_classes(classes_path):
|
||||
with open(classes_path, encoding='utf-8') as f:
|
||||
class_names = f.readlines()
|
||||
class_names = [c.strip() for c in class_names]
|
||||
return class_names, len(class_names)
|
||||
|
||||
|
||||
# ---------------------------------------------------#
|
||||
# 获得先验框
|
||||
# ---------------------------------------------------#
|
||||
def get_anchors(anchors_path):
|
||||
'''loads the anchors from a file'''
|
||||
with open(anchors_path, encoding='utf-8') as f:
|
||||
anchors = f.readline()
|
||||
anchors = [float(x) for x in anchors.split(',')]
|
||||
anchors = np.array(anchors).reshape(-1, 2)
|
||||
return anchors, len(anchors)
|
||||
|
||||
|
||||
# ---------------------------------------------------#
|
||||
# 获得学习率
|
||||
# ---------------------------------------------------#
|
||||
def get_lr(optimizer):
|
||||
for param_group in optimizer.param_groups:
|
||||
return param_group['lr']
|
||||
|
||||
|
||||
def preprocess_input(image):
|
||||
image /= 255.0
|
||||
return image
|
||||
|
||||
|
||||
def show_config(**kwargs):
|
||||
print('Configurations:')
|
||||
print('-' * 70)
|
||||
print('|%25s | %40s|' % ('keys', 'values'))
|
||||
print('-' * 70)
|
||||
for key, value in kwargs.items():
|
||||
print('|%25s | %40s|' % (str(key), str(value)))
|
||||
print('-' * 70)
|
||||
|
|
@ -0,0 +1,232 @@
|
|||
import torch
|
||||
import torch.nn as nn
|
||||
from torchvision.ops import nms
|
||||
import numpy as np
|
||||
|
||||
|
||||
class DecodeBox():
|
||||
def __init__(self, anchors, num_classes, input_shape, anchors_mask=[[6, 7, 8], [3, 4, 5], [0, 1, 2]]):
|
||||
super(DecodeBox, self).__init__()
|
||||
self.anchors = anchors
|
||||
self.num_classes = num_classes
|
||||
self.bbox_attrs = 5 + num_classes
|
||||
self.input_shape = input_shape
|
||||
# -----------------------------------------------------------#
|
||||
# 13x13的特征层对应的anchor是[116,90],[156,198],[373,326]
|
||||
# 26x26的特征层对应的anchor是[30,61],[62,45],[59,119]
|
||||
# 52x52的特征层对应的anchor是[10,13],[16,30],[33,23]
|
||||
# -----------------------------------------------------------#
|
||||
self.anchors_mask = anchors_mask
|
||||
|
||||
def decode_box(self, inputs):
|
||||
outputs = []
|
||||
for i, input in enumerate(inputs):
|
||||
# -----------------------------------------------#
|
||||
# 输入的input一共有三个，他们的shape分别是
|
||||
# batch_size, 255, 13, 13
|
||||
# batch_size, 255, 26, 26
|
||||
# batch_size, 255, 52, 52
|
||||
# -----------------------------------------------#
|
||||
batch_size = input.size(0)
|
||||
input_height = input.size(2)
|
||||
input_width = input.size(3)
|
||||
|
||||
# -----------------------------------------------#
|
||||
# 输入为416x416时
|
||||
# stride_h = stride_w = 32、16、8
|
||||
# -----------------------------------------------#
|
||||
stride_h = self.input_shape[0] / input_height
|
||||
stride_w = self.input_shape[1] / input_width
|
||||
# -------------------------------------------------#
|
||||
# 此时获得的scaled_anchors大小是相对于特征层的
|
||||
# -------------------------------------------------#
|
||||
scaled_anchors = [(anchor_width / stride_w, anchor_height / stride_h) for anchor_width, anchor_height in
|
||||
self.anchors[self.anchors_mask[i]]]
|
||||
|
||||
# -----------------------------------------------#
|
||||
# 输入的input一共有三个，他们的shape分别是
|
||||
# batch_size, 3, 13, 13, 85
|
||||
# batch_size, 3, 26, 26, 85
|
||||
# batch_size, 3, 52, 52, 85
|
||||
# -----------------------------------------------#
|
||||
prediction = input.view(batch_size, len(self.anchors_mask[i]),
|
||||
self.bbox_attrs, input_height, input_width).permute(0, 1, 3, 4, 2).contiguous()
|
||||
# 调整为 1，3，13，13，25 的形状
|
||||
# -----------------------------------------------#
|
||||
# 先验框的中心位置的调整参数
|
||||
# -----------------------------------------------#
|
||||
x = torch.sigmoid(prediction[..., 0])
|
||||
y = torch.sigmoid(prediction[..., 1])
|
||||
# -----------------------------------------------#
|
||||
# 先验框的宽高调整参数
|
||||
# -----------------------------------------------#
|
||||
w = prediction[..., 2]
|
||||
h = prediction[..., 3]
|
||||
# -----------------------------------------------#
|
||||
# 获得置信度，是否有物体
|
||||
# -----------------------------------------------#
|
||||
conf = torch.sigmoid(prediction[..., 4])
|
||||
# -----------------------------------------------#
|
||||
# 种类置信度
|
||||
# -----------------------------------------------#
|
||||
pred_cls = torch.sigmoid(prediction[..., 5:])
|
||||
|
||||
FloatTensor = torch.cuda.FloatTensor if x.is_cuda else torch.FloatTensor
|
||||
LongTensor = torch.cuda.LongTensor if x.is_cuda else torch.LongTensor
|
||||
|
||||
# ----------------------------------------------------------#
|
||||
# 生成网格，先验框中心，网格左上角
|
||||
# batch_size,3,13,13
|
||||
# ----------------------------------------------------------#
|
||||
grid_x = torch.linspace(0, input_width - 1, input_width).repeat(input_height, 1).repeat(
|
||||
batch_size * len(self.anchors_mask[i]), 1, 1).view(x.shape).type(FloatTensor)
|
||||
grid_y = torch.linspace(0, input_height - 1, input_height).repeat(input_width, 1).t().repeat(
|
||||
batch_size * len(self.anchors_mask[i]), 1, 1).view(y.shape).type(FloatTensor)
|
||||
|
||||
# ----------------------------------------------------------#
|
||||
# 按照网格格式生成先验框的宽高
|
||||
# batch_size,3,13,13
|
||||
# ----------------------------------------------------------#
|
||||
anchor_w = FloatTensor(scaled_anchors).index_select(1, LongTensor([0]))
|
||||
anchor_h = FloatTensor(scaled_anchors).index_select(1, LongTensor([1]))
|
||||
anchor_w = anchor_w.repeat(batch_size, 1).repeat(1, 1, input_height * input_width).view(w.shape)
|
||||
anchor_h = anchor_h.repeat(batch_size, 1).repeat(1, 1, input_height * input_width).view(h.shape)
|
||||
|
||||
# ----------------------------------------------------------#
|
||||
# 利用预测结果对先验框进行调整
|
||||
# 首先调整先验框的中心，从先验框中心向右下角偏移 # ？从先验框左上角向右下角偏移？
|
||||
# 再调整先验框的宽高。
|
||||
# ----------------------------------------------------------#
|
||||
pred_boxes = FloatTensor(prediction[..., :4].shape)
|
||||
pred_boxes[..., 0] = x.data + grid_x
|
||||
pred_boxes[..., 1] = y.data + grid_y
|
||||
pred_boxes[..., 2] = torch.exp(w.data) * anchor_w
|
||||
pred_boxes[..., 3] = torch.exp(h.data) * anchor_h
|
||||
|
||||
# ----------------------------------------------------------#
|
||||
# 将输出结果归一化成小数的形式
|
||||
# ----------------------------------------------------------#
|
||||
_scale = torch.Tensor([input_width, input_height, input_width, input_height]).type(FloatTensor)
|
||||
output = torch.cat((pred_boxes.view(batch_size, -1, 4) / _scale,
|
||||
conf.view(batch_size, -1, 1), pred_cls.view(batch_size, -1, self.num_classes)), -1)
|
||||
# output的shape是 batch_size, -1, attr(25)
|
||||
outputs.append(output.data)
|
||||
return outputs
|
||||
|
||||
def yolo_correct_boxes(self, box_xy, box_wh, input_shape, image_shape, letterbox_image):
|
||||
# -----------------------------------------------------------------#
|
||||
# 把y轴放前面是因为方便预测框和图像的宽高进行相乘
|
||||
# -----------------------------------------------------------------#
|
||||
box_yx = box_xy[..., ::-1]
|
||||
box_hw = box_wh[..., ::-1]
|
||||
input_shape = np.array(input_shape)
|
||||
image_shape = np.array(image_shape)
|
||||
|
||||
if letterbox_image:
|
||||
# -----------------------------------------------------------------#
|
||||
# 这里求出来的offset是图像有效区域相对于图像左上角的偏移情况
|
||||
# new_shape指的是宽高缩放情况
|
||||
# -----------------------------------------------------------------#
|
||||
new_shape = np.round(image_shape * np.min(input_shape / image_shape))
|
||||
offset = (input_shape - new_shape) / 2. / input_shape
|
||||
scale = input_shape / new_shape
|
||||
|
||||
box_yx = (box_yx - offset) * scale
|
||||
box_hw *= scale
|
||||
|
||||
box_mins = box_yx - (box_hw / 2.)
|
||||
box_maxes = box_yx + (box_hw / 2.)
|
||||
boxes = np.concatenate([box_mins[..., 0:1], box_mins[..., 1:2], box_maxes[..., 0:1], box_maxes[..., 1:2]],
|
||||
axis=-1)
|
||||
boxes *= np.concatenate([image_shape, image_shape], axis=-1)
|
||||
return boxes
|
||||
|
||||
def non_max_suppression(self, prediction, num_classes, input_shape, image_shape, letterbox_image, conf_thres=0.5,
|
||||
nms_thres=0.4):
|
||||
# ----------------------------------------------------------#
|
||||
# 将预测结果的格式转换成左上角右下角的格式。
|
||||
# prediction [batch_size, num_anchors, 85]
|
||||
# ----------------------------------------------------------#
|
||||
box_corner = prediction.new(prediction.shape)
|
||||
box_corner[:, :, 0] = prediction[:, :, 0] - prediction[:, :, 2] / 2
|
||||
box_corner[:, :, 1] = prediction[:, :, 1] - prediction[:, :, 3] / 2
|
||||
box_corner[:, :, 2] = prediction[:, :, 0] + prediction[:, :, 2] / 2
|
||||
box_corner[:, :, 3] = prediction[:, :, 1] + prediction[:, :, 3] / 2
|
||||
prediction[:, :, :4] = box_corner[:, :, :4]
|
||||
|
||||
output = [None for _ in range(len(prediction))]
|
||||
for i, image_pred in enumerate(prediction):
|
||||
# ----------------------------------------------------------#
|
||||
# 对种类预测部分取max。 # image_pred 是在prediction中以0维度迭代
|
||||
# class_conf [num_anchors, 1] 种类置信度
|
||||
# class_pred [num_anchors, 1] 种类 image_pred[:, 5:5 + num_classes] 是取出类别
|
||||
# ----------------------------------------------------------#
|
||||
class_conf, class_pred = torch.max(image_pred[:, 5:5 + num_classes], 1, keepdim=True)
|
||||
|
||||
# ----------------------------------------------------------#
|
||||
# 利用置信度进行第一轮筛选
|
||||
# ----------------------------------------------------------#
|
||||
conf_mask = (image_pred[:, 4] * class_conf[:, 0] >= conf_thres).squeeze()
|
||||
|
||||
# ----------------------------------------------------------#
|
||||
# 根据置信度进行预测结果的筛选
|
||||
# ----------------------------------------------------------#
|
||||
image_pred = image_pred[conf_mask]
|
||||
class_conf = class_conf[conf_mask]
|
||||
class_pred = class_pred[conf_mask]
|
||||
if not image_pred.size(0):
|
||||
continue # 如果没有剩下类别，就判断下一张图片
|
||||
# -------------------------------------------------------------------------#
|
||||
# detections [num_anchors, 7]
|
||||
# 7的内容为：x1, y1, x2, y2, obj_conf, class_conf, class_pred
|
||||
# -------------------------------------------------------------------------#
|
||||
detections = torch.cat((image_pred[:, :5], class_conf.float(), class_pred.float()), 1)
|
||||
|
||||
# ------------------------------------------#
|
||||
# 获得预测结果中包含的所有种类
|
||||
# ------------------------------------------#
|
||||
unique_labels = detections[:, -1].cpu().unique()
|
||||
|
||||
if prediction.is_cuda:
|
||||
unique_labels = unique_labels.cuda()
|
||||
detections = detections.cuda()
|
||||
|
||||
for c in unique_labels:
|
||||
# ------------------------------------------#
|
||||
# 获得某一类得分筛选后全部的预测结果
|
||||
# ------------------------------------------#
|
||||
detections_class = detections[detections[:, -1] == c]
|
||||
|
||||
# ------------------------------------------#
|
||||
# 使用官方自带的非极大抑制会速度更快一些！
|
||||
# ------------------------------------------#
|
||||
keep = nms(
|
||||
detections_class[:, :4],
|
||||
detections_class[:, 4] * detections_class[:, 5],
|
||||
nms_thres
|
||||
)
|
||||
max_detections = detections_class[keep]
|
||||
|
||||
# # 按照存在物体的置信度排序
|
||||
# _, conf_sort_index = torch.sort(detections_class[:, 4]*detections_class[:, 5], descending=True)
|
||||
# detections_class = detections_class[conf_sort_index]
|
||||
# # 进行非极大抑制
|
||||
# max_detections = []
|
||||
# while detections_class.size(0):
|
||||
# # 取出这一类置信度最高的，一步一步往下判断，判断重合程度是否大于nms_thres，如果是则去除掉
|
||||
# max_detections.append(detections_class[0].unsqueeze(0))
|
||||
# if len(detections_class) == 1:
|
||||
# break
|
||||
# ious = bbox_iou(max_detections[-1], detections_class[1:])
|
||||
# detections_class = detections_class[1:][ious < nms_thres]
|
||||
# # 堆叠
|
||||
# max_detections = torch.cat(max_detections).data
|
||||
|
||||
# Add max detections to outputs
|
||||
output[i] = max_detections if output[i] is None else torch.cat((output[i], max_detections))
|
||||
|
||||
if output[i] is not None:
|
||||
output[i] = output[i].cpu().numpy()
|
||||
box_xy, box_wh = (output[i][:, 0:2] + output[i][:, 2:4]) / 2, output[i][:, 2:4] - output[i][:, 0:2]
|
||||
output[i][:, :4] = self.yolo_correct_boxes(box_xy, box_wh, input_shape, image_shape, letterbox_image)
|
||||
return output
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
import os
|
||||
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
|
||||
from utils.utils import get_lr
|
||||
|
||||
|
||||
def fit_one_epoch(model_train, model, yolo_loss, loss_history, eval_callback, optimizer, epoch, epoch_step,
|
||||
epoch_step_val, gen, gen_val, Epoch, cuda, fp16, scaler, save_period, save_dir, local_rank=0):
|
||||
loss = 0
|
||||
val_loss = 0
|
||||
|
||||
if local_rank == 0:
|
||||
print('Start Train')
|
||||
pbar = tqdm(total=epoch_step, desc=f'Epoch {epoch + 1}/{Epoch}', postfix=dict, mininterval=0.3)
|
||||
model_train.train() # 调整所有的模块为train模式
|
||||
for iteration, batch in enumerate(gen):
|
||||
if iteration >= epoch_step: # 有什么意义？
|
||||
break
|
||||
|
||||
images, targets = batch[0], batch[1] # targets也是归一化了的
|
||||
with torch.no_grad():
|
||||
if cuda:
|
||||
images = images.cuda(local_rank)
|
||||
targets = [ann.cuda(local_rank) for ann in
|
||||
targets] # targets是一个python的list，里面是tensor，把tensor逐个转到cuda上，然后targets还是python的列表
|
||||
# ----------------------#
|
||||
# 清零梯度
|
||||
# ----------------------#
|
||||
optimizer.zero_grad()
|
||||
if not fp16:
|
||||
# ----------------------#
|
||||
# 前向传播
|
||||
# ----------------------#
|
||||
outputs = model_train(images)
|
||||
|
||||
loss_value_all = 0
|
||||
# ----------------------#
|
||||
# 计算损失
|
||||
# ----------------------#
|
||||
for l in range(len(outputs)): # 三组不同分辨率大小的输出特征分别计算
|
||||
loss_item = yolo_loss(l, outputs[l], targets)
|
||||
loss_value_all += loss_item
|
||||
loss_value = loss_value_all
|
||||
|
||||
# ----------------------#
|
||||
# 反向传播
|
||||
# ----------------------#
|
||||
loss_value.backward()
|
||||
optimizer.step()
|
||||
else: # 不进入这条分支
|
||||
from torch.cuda.amp import autocast
|
||||
with autocast():
|
||||
# ----------------------#
|
||||
# 前向传播
|
||||
# ----------------------#
|
||||
outputs = model_train(images)
|
||||
|
||||
loss_value_all = 0
|
||||
# ----------------------#
|
||||
# 计算损失
|
||||
# ----------------------#
|
||||
for l in range(len(outputs)):
|
||||
loss_item = yolo_loss(l, outputs[l], targets)
|
||||
loss_value_all += loss_item
|
||||
loss_value = loss_value_all
|
||||
|
||||
# ----------------------#
|
||||
# 反向传播
|
||||
# ----------------------#
|
||||
scaler.scale(loss_value).backward()
|
||||
scaler.step(optimizer)
|
||||
scaler.update()
|
||||
|
||||
loss += loss_value.item()
|
||||
|
||||
# # 调试用 begin
|
||||
# if iteration > 2:
|
||||
# break
|
||||
# # 调试用 end
|
||||
|
||||
if local_rank == 0:
|
||||
pbar.set_postfix(**{'loss': loss / (iteration + 1),
|
||||
'lr': get_lr(optimizer)})
|
||||
pbar.update(1)
|
||||
|
||||
if local_rank == 0:
|
||||
pbar.close()
|
||||
print('Finish Train')
|
||||
print('Start Validation')
|
||||
pbar = tqdm(total=epoch_step_val, desc=f'Epoch {epoch + 1}/{Epoch}', postfix=dict, mininterval=0.3)
|
||||
|
||||
model_train.eval()
|
||||
for iteration, batch in enumerate(gen_val):
|
||||
if iteration >= epoch_step_val:
|
||||
break
|
||||
images, targets = batch[0], batch[1]
|
||||
with torch.no_grad():
|
||||
if cuda:
|
||||
images = images.cuda(local_rank)
|
||||
targets = [ann.cuda(local_rank) for ann in targets]
|
||||
# ----------------------#
|
||||
# 清零梯度
|
||||
# ----------------------#
|
||||
optimizer.zero_grad()
|
||||
# ----------------------#
|
||||
# 前向传播
|
||||
# ----------------------#
|
||||
outputs = model_train(images)
|
||||
|
||||
loss_value_all = 0
|
||||
# ----------------------#
|
||||
# 计算损失
|
||||
# ----------------------#
|
||||
for l in range(len(outputs)):
|
||||
loss_item = yolo_loss(l, outputs[l], targets)
|
||||
loss_value_all += loss_item
|
||||
loss_value = loss_value_all
|
||||
|
||||
val_loss += loss_value.item()
|
||||
|
||||
# # 调试用 begin
|
||||
# if iteration > 2:
|
||||
# break
|
||||
# # 调试用 end
|
||||
|
||||
if local_rank == 0:
|
||||
pbar.set_postfix(**{'val_loss': val_loss / (iteration + 1)})
|
||||
pbar.update(1)
|
||||
|
||||
if local_rank == 0:
|
||||
pbar.close()
|
||||
print('Finish Validation')
|
||||
loss_history.append_loss(epoch + 1, loss / epoch_step, val_loss / epoch_step_val)
|
||||
eval_callback.on_epoch_end(epoch + 1, model_train)
|
||||
print('Epoch:' + str(epoch + 1) + '/' + str(Epoch))
|
||||
print('Total Loss: %.3f || Val Loss: %.3f ' % (loss / epoch_step, val_loss / epoch_step_val))
|
||||
|
||||
# -----------------------------------------------#
|
||||
# 保存权值
|
||||
# -----------------------------------------------#
|
||||
if (epoch + 1) % save_period == 0 or epoch + 1 == Epoch:
|
||||
torch.save(model.state_dict(), os.path.join(save_dir, "ep%03d-loss%.3f-val_loss%.3f.pth" % (
|
||||
epoch + 1, loss / epoch_step, val_loss / epoch_step_val)))
|
||||
|
||||
if len(loss_history.val_loss) <= 1 or (val_loss / epoch_step_val) <= min(loss_history.val_loss):
|
||||
print('Save best model to best_epoch_weights.pth')
|
||||
torch.save(model.state_dict(), os.path.join(save_dir, "best_epoch_weights.pth"))
|
||||
|
||||
torch.save(model.state_dict(), os.path.join(save_dir, "last_epoch_weights.pth"))
|
||||
|
|
@ -0,0 +1,963 @@
|
|||
import glob
|
||||
import json
|
||||
import math
|
||||
import operator
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
try:
|
||||
from pycocotools.coco import COCO
|
||||
from pycocotools.cocoeval import COCOeval
|
||||
except:
|
||||
pass
|
||||
import cv2
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use('Agg')
|
||||
from matplotlib import pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
'''
|
||||
0,0 ------> x (width)
|
||||
|
|
||||
| (Left,Top)
|
||||
| *_________
|
||||
| | |
|
||||
| |
|
||||
y |_________|
|
||||
(height) *
|
||||
(Right,Bottom)
|
||||
'''
|
||||
|
||||
|
||||
def log_average_miss_rate(precision, fp_cumsum, num_images):
|
||||
"""
|
||||
log-average miss rate:
|
||||
Calculated by averaging miss rates at 9 evenly spaced FPPI points
|
||||
between 10e-2 and 10e0, in log-space.
|
||||
|
||||
output:
|
||||
lamr | log-average miss rate
|
||||
mr | miss rate
|
||||
fppi | false positives per image
|
||||
|
||||
references:
|
||||
[1] Dollar, Piotr, et al. "Pedestrian Detection: An Evaluation of the
|
||||
State of the Art." Pattern Analysis and Machine Intelligence, IEEE
|
||||
Transactions on 34.4 (2012): 743 - 761.
|
||||
"""
|
||||
|
||||
if precision.size == 0:
|
||||
lamr = 0
|
||||
mr = 1
|
||||
fppi = 0
|
||||
return lamr, mr, fppi
|
||||
|
||||
fppi = fp_cumsum / float(num_images)
|
||||
mr = (1 - precision)
|
||||
|
||||
fppi_tmp = np.insert(fppi, 0, -1.0)
|
||||
mr_tmp = np.insert(mr, 0, 1.0)
|
||||
|
||||
ref = np.logspace(-2.0, 0.0, num=9)
|
||||
for i, ref_i in enumerate(ref):
|
||||
j = np.where(fppi_tmp <= ref_i)[-1][-1]
|
||||
ref[i] = mr_tmp[j]
|
||||
|
||||
lamr = math.exp(np.mean(np.log(np.maximum(1e-10, ref))))
|
||||
|
||||
return lamr, mr, fppi
|
||||
|
||||
|
||||
"""
|
||||
throw error and exit
|
||||
"""
|
||||
|
||||
|
||||
def error(msg):
|
||||
print(msg)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
"""
|
||||
check if the number is a float between 0.0 and 1.0
|
||||
"""
|
||||
|
||||
|
||||
def is_float_between_0_and_1(value):
|
||||
try:
|
||||
val = float(value)
|
||||
if val > 0.0 and val < 1.0:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
"""
|
||||
Calculate the AP given the recall and precision array
|
||||
1st) We compute a version of the measured precision/recall curve with
|
||||
precision monotonically decreasing
|
||||
2nd) We compute the AP as the area under this curve by numerical integration.
|
||||
"""
|
||||
|
||||
|
||||
def voc_ap(rec, prec):
|
||||
"""
|
||||
--- Official matlab code VOC2012---
|
||||
mrec=[0 ; rec ; 1];
|
||||
mpre=[0 ; prec ; 0];
|
||||
for i=numel(mpre)-1:-1:1
|
||||
mpre(i)=max(mpre(i),mpre(i+1));
|
||||
end
|
||||
i=find(mrec(2:end)~=mrec(1:end-1))+1;
|
||||
ap=sum((mrec(i)-mrec(i-1)).*mpre(i));
|
||||
"""
|
||||
rec.insert(0, 0.0) # insert 0.0 at begining of list
|
||||
rec.append(1.0) # insert 1.0 at end of list
|
||||
mrec = rec[:]
|
||||
prec.insert(0, 0.0) # insert 0.0 at begining of list
|
||||
prec.append(0.0) # insert 0.0 at end of list
|
||||
mpre = prec[:]
|
||||
"""
|
||||
This part makes the precision monotonically decreasing
|
||||
(goes from the end to the beginning)
|
||||
matlab: for i=numel(mpre)-1:-1:1
|
||||
mpre(i)=max(mpre(i),mpre(i+1));
|
||||
"""
|
||||
for i in range(len(mpre) - 2, -1, -1):
|
||||
mpre[i] = max(mpre[i], mpre[i + 1])
|
||||
"""
|
||||
This part creates a list of indexes where the recall changes
|
||||
matlab: i=find(mrec(2:end)~=mrec(1:end-1))+1;
|
||||
"""
|
||||
i_list = []
|
||||
for i in range(1, len(mrec)):
|
||||
if mrec[i] != mrec[i - 1]:
|
||||
i_list.append(i) # if it was matlab would be i + 1
|
||||
"""
|
||||
The Average Precision (AP) is the area under the curve
|
||||
(numerical integration)
|
||||
matlab: ap=sum((mrec(i)-mrec(i-1)).*mpre(i));
|
||||
"""
|
||||
ap = 0.0
|
||||
for i in i_list:
|
||||
ap += ((mrec[i] - mrec[i - 1]) * mpre[i])
|
||||
return ap, mrec, mpre
|
||||
|
||||
|
||||
"""
|
||||
Convert the lines of a file to a list
|
||||
"""
|
||||
|
||||
|
||||
def file_lines_to_list(path):
|
||||
# open txt file lines to a list
|
||||
with open(path) as f:
|
||||
content = f.readlines()
|
||||
# remove whitespace characters like `\n` at the end of each line
|
||||
content = [x.strip() for x in content]
|
||||
return content
|
||||
|
||||
|
||||
"""
|
||||
Draws text in image
|
||||
"""
|
||||
|
||||
|
||||
def draw_text_in_image(img, text, pos, color, line_width):
|
||||
font = cv2.FONT_HERSHEY_PLAIN
|
||||
fontScale = 1
|
||||
lineType = 1
|
||||
bottomLeftCornerOfText = pos
|
||||
cv2.putText(img, text,
|
||||
bottomLeftCornerOfText,
|
||||
font,
|
||||
fontScale,
|
||||
color,
|
||||
lineType)
|
||||
text_width, _ = cv2.getTextSize(text, font, fontScale, lineType)[0]
|
||||
return img, (line_width + text_width)
|
||||
|
||||
|
||||
"""
|
||||
Plot - adjust axes
|
||||
"""
|
||||
|
||||
|
||||
def adjust_axes(r, t, fig, axes):
|
||||
# get text width for re-scaling
|
||||
bb = t.get_window_extent(renderer=r)
|
||||
text_width_inches = bb.width / fig.dpi
|
||||
# get axis width in inches
|
||||
current_fig_width = fig.get_figwidth()
|
||||
new_fig_width = current_fig_width + text_width_inches
|
||||
propotion = new_fig_width / current_fig_width
|
||||
# get axis limit
|
||||
x_lim = axes.get_xlim()
|
||||
axes.set_xlim([x_lim[0], x_lim[1] * propotion])
|
||||
|
||||
|
||||
"""
|
||||
Draw plot using Matplotlib
|
||||
"""
|
||||
|
||||
|
||||
def draw_plot_func(dictionary, n_classes, window_title, plot_title, x_label, output_path, to_show, plot_color,
|
||||
true_p_bar):
|
||||
# sort the dictionary by decreasing value, into a list of tuples
|
||||
sorted_dic_by_value = sorted(dictionary.items(), key=operator.itemgetter(1))
|
||||
# unpacking the list of tuples into two lists
|
||||
sorted_keys, sorted_values = zip(*sorted_dic_by_value)
|
||||
#
|
||||
if true_p_bar != "":
|
||||
"""
|
||||
Special case to draw in:
|
||||
- green -> TP: True Positives (object detected and matches ground-truth)
|
||||
- red -> FP: False Positives (object detected but does not match ground-truth)
|
||||
- orange -> FN: False Negatives (object not detected but present in the ground-truth)
|
||||
"""
|
||||
fp_sorted = []
|
||||
tp_sorted = []
|
||||
for key in sorted_keys:
|
||||
fp_sorted.append(dictionary[key] - true_p_bar[key])
|
||||
tp_sorted.append(true_p_bar[key])
|
||||
plt.barh(range(n_classes), fp_sorted, align='center', color='crimson', label='False Positive')
|
||||
plt.barh(range(n_classes), tp_sorted, align='center', color='forestgreen', label='True Positive',
|
||||
left=fp_sorted)
|
||||
# add legend
|
||||
plt.legend(loc='lower right')
|
||||
"""
|
||||
Write number on side of bar
|
||||
"""
|
||||
fig = plt.gcf() # gcf - get current figure
|
||||
axes = plt.gca()
|
||||
r = fig.canvas.get_renderer()
|
||||
for i, val in enumerate(sorted_values):
|
||||
fp_val = fp_sorted[i]
|
||||
tp_val = tp_sorted[i]
|
||||
fp_str_val = " " + str(fp_val)
|
||||
tp_str_val = fp_str_val + " " + str(tp_val)
|
||||
# trick to paint multicolor with offset:
|
||||
# first paint everything and then repaint the first number
|
||||
t = plt.text(val, i, tp_str_val, color='forestgreen', va='center', fontweight='bold')
|
||||
plt.text(val, i, fp_str_val, color='crimson', va='center', fontweight='bold')
|
||||
if i == (len(sorted_values) - 1): # largest bar
|
||||
adjust_axes(r, t, fig, axes)
|
||||
else:
|
||||
plt.barh(range(n_classes), sorted_values, color=plot_color)
|
||||
"""
|
||||
Write number on side of bar
|
||||
"""
|
||||
fig = plt.gcf() # gcf - get current figure
|
||||
axes = plt.gca()
|
||||
r = fig.canvas.get_renderer()
|
||||
for i, val in enumerate(sorted_values):
|
||||
str_val = " " + str(val) # add a space before
|
||||
if val < 1.0:
|
||||
str_val = " {0:.2f}".format(val)
|
||||
t = plt.text(val, i, str_val, color=plot_color, va='center', fontweight='bold')
|
||||
# re-set axes to show number inside the figure
|
||||
if i == (len(sorted_values) - 1): # largest bar
|
||||
adjust_axes(r, t, fig, axes)
|
||||
# set window title
|
||||
fig.canvas.set_window_title(window_title)
|
||||
# write classes in y axis
|
||||
tick_font_size = 12
|
||||
plt.yticks(range(n_classes), sorted_keys, fontsize=tick_font_size)
|
||||
"""
|
||||
Re-scale height accordingly
|
||||
"""
|
||||
init_height = fig.get_figheight()
|
||||
# comput the matrix height in points and inches
|
||||
dpi = fig.dpi
|
||||
height_pt = n_classes * (tick_font_size * 1.4) # 1.4 (some spacing)
|
||||
height_in = height_pt / dpi
|
||||
# compute the required figure height
|
||||
top_margin = 0.15 # in percentage of the figure height
|
||||
bottom_margin = 0.05 # in percentage of the figure height
|
||||
figure_height = height_in / (1 - top_margin - bottom_margin)
|
||||
# set new height
|
||||
if figure_height > init_height:
|
||||
fig.set_figheight(figure_height)
|
||||
|
||||
# set plot title
|
||||
plt.title(plot_title, fontsize=14)
|
||||
# set axis titles
|
||||
# plt.xlabel('classes')
|
||||
plt.xlabel(x_label, fontsize='large')
|
||||
# adjust size of window
|
||||
fig.tight_layout()
|
||||
# save the plot
|
||||
fig.savefig(output_path)
|
||||
# show image
|
||||
if to_show:
|
||||
plt.show()
|
||||
# close the plot
|
||||
plt.close()
|
||||
|
||||
|
||||
def get_map(MINOVERLAP, draw_plot, score_threhold=0.5, path='./map_out'):
|
||||
GT_PATH = os.path.join(path, 'ground-truth')
|
||||
DR_PATH = os.path.join(path, 'detection-results')
|
||||
IMG_PATH = os.path.join(path, 'images-optional')
|
||||
TEMP_FILES_PATH = os.path.join(path, '.temp_files')
|
||||
RESULTS_FILES_PATH = os.path.join(path, 'results')
|
||||
|
||||
show_animation = True
|
||||
if os.path.exists(IMG_PATH):
|
||||
for dirpath, dirnames, files in os.walk(IMG_PATH):
|
||||
if not files:
|
||||
show_animation = False
|
||||
else:
|
||||
show_animation = False
|
||||
|
||||
if not os.path.exists(TEMP_FILES_PATH):
|
||||
os.makedirs(TEMP_FILES_PATH)
|
||||
|
||||
if os.path.exists(RESULTS_FILES_PATH):
|
||||
shutil.rmtree(RESULTS_FILES_PATH)
|
||||
else:
|
||||
os.makedirs(RESULTS_FILES_PATH)
|
||||
if draw_plot:
|
||||
try:
|
||||
matplotlib.use('TkAgg')
|
||||
except:
|
||||
pass
|
||||
os.makedirs(os.path.join(RESULTS_FILES_PATH, "AP"))
|
||||
os.makedirs(os.path.join(RESULTS_FILES_PATH, "F1"))
|
||||
os.makedirs(os.path.join(RESULTS_FILES_PATH, "Recall"))
|
||||
os.makedirs(os.path.join(RESULTS_FILES_PATH, "Precision"))
|
||||
if show_animation:
|
||||
os.makedirs(os.path.join(RESULTS_FILES_PATH, "images", "detections_one_by_one"))
|
||||
|
||||
ground_truth_files_list = glob.glob(GT_PATH + '/*.txt')
|
||||
if len(ground_truth_files_list) == 0:
|
||||
error("Error: No ground-truth files found!")
|
||||
ground_truth_files_list.sort()
|
||||
gt_counter_per_class = {}
|
||||
counter_images_per_class = {}
|
||||
|
||||
for txt_file in ground_truth_files_list:
|
||||
file_id = txt_file.split(".txt", 1)[0]
|
||||
file_id = os.path.basename(os.path.normpath(file_id))
|
||||
temp_path = os.path.join(DR_PATH, (file_id + ".txt"))
|
||||
if not os.path.exists(temp_path):
|
||||
error_msg = "Error. File not found: {}\n".format(temp_path)
|
||||
error(error_msg)
|
||||
lines_list = file_lines_to_list(txt_file)
|
||||
bounding_boxes = []
|
||||
is_difficult = False
|
||||
already_seen_classes = []
|
||||
for line in lines_list:
|
||||
try:
|
||||
if "difficult" in line:
|
||||
class_name, left, top, right, bottom, _difficult = line.split()
|
||||
is_difficult = True
|
||||
else:
|
||||
class_name, left, top, right, bottom = line.split()
|
||||
except:
|
||||
if "difficult" in line:
|
||||
line_split = line.split()
|
||||
_difficult = line_split[-1]
|
||||
bottom = line_split[-2]
|
||||
right = line_split[-3]
|
||||
top = line_split[-4]
|
||||
left = line_split[-5]
|
||||
class_name = ""
|
||||
for name in line_split[:-5]:
|
||||
class_name += name + " "
|
||||
class_name = class_name[:-1]
|
||||
is_difficult = True
|
||||
else:
|
||||
line_split = line.split()
|
||||
bottom = line_split[-1]
|
||||
right = line_split[-2]
|
||||
top = line_split[-3]
|
||||
left = line_split[-4]
|
||||
class_name = ""
|
||||
for name in line_split[:-4]:
|
||||
class_name += name + " "
|
||||
class_name = class_name[:-1]
|
||||
|
||||
bbox = left + " " + top + " " + right + " " + bottom
|
||||
if is_difficult:
|
||||
bounding_boxes.append({"class_name": class_name, "bbox": bbox, "used": False, "difficult": True})
|
||||
is_difficult = False
|
||||
else:
|
||||
bounding_boxes.append({"class_name": class_name, "bbox": bbox, "used": False})
|
||||
if class_name in gt_counter_per_class:
|
||||
gt_counter_per_class[class_name] += 1
|
||||
else:
|
||||
gt_counter_per_class[class_name] = 1
|
||||
|
||||
if class_name not in already_seen_classes:
|
||||
if class_name in counter_images_per_class:
|
||||
counter_images_per_class[class_name] += 1
|
||||
else:
|
||||
counter_images_per_class[class_name] = 1
|
||||
already_seen_classes.append(class_name)
|
||||
|
||||
with open(TEMP_FILES_PATH + "/" + file_id + "_ground_truth.json", 'w') as outfile:
|
||||
json.dump(bounding_boxes, outfile)
|
||||
|
||||
gt_classes = list(gt_counter_per_class.keys())
|
||||
gt_classes = sorted(gt_classes)
|
||||
n_classes = len(gt_classes)
|
||||
|
||||
dr_files_list = glob.glob(DR_PATH + '/*.txt')
|
||||
dr_files_list.sort()
|
||||
for class_index, class_name in enumerate(gt_classes):
|
||||
bounding_boxes = []
|
||||
for txt_file in dr_files_list:
|
||||
file_id = txt_file.split(".txt", 1)[0]
|
||||
file_id = os.path.basename(os.path.normpath(file_id))
|
||||
temp_path = os.path.join(GT_PATH, (file_id + ".txt"))
|
||||
if class_index == 0:
|
||||
if not os.path.exists(temp_path):
|
||||
error_msg = "Error. File not found: {}\n".format(temp_path)
|
||||
error(error_msg)
|
||||
lines = file_lines_to_list(txt_file)
|
||||
for line in lines:
|
||||
try:
|
||||
tmp_class_name, confidence, left, top, right, bottom = line.split()
|
||||
except:
|
||||
line_split = line.split()
|
||||
bottom = line_split[-1]
|
||||
right = line_split[-2]
|
||||
top = line_split[-3]
|
||||
left = line_split[-4]
|
||||
confidence = line_split[-5]
|
||||
tmp_class_name = ""
|
||||
for name in line_split[:-5]:
|
||||
tmp_class_name += name + " "
|
||||
tmp_class_name = tmp_class_name[:-1]
|
||||
|
||||
if tmp_class_name == class_name:
|
||||
bbox = left + " " + top + " " + right + " " + bottom
|
||||
bounding_boxes.append({"confidence": confidence, "file_id": file_id, "bbox": bbox})
|
||||
|
||||
bounding_boxes.sort(key=lambda x: float(x['confidence']), reverse=True)
|
||||
with open(TEMP_FILES_PATH + "/" + class_name + "_dr.json", 'w') as outfile:
|
||||
json.dump(bounding_boxes, outfile)
|
||||
|
||||
sum_AP = 0.0
|
||||
ap_dictionary = {}
|
||||
lamr_dictionary = {}
|
||||
with open(RESULTS_FILES_PATH + "/results.txt", 'w') as results_file:
|
||||
results_file.write("# AP and precision/recall per class\n")
|
||||
count_true_positives = {}
|
||||
|
||||
for class_index, class_name in enumerate(gt_classes):
|
||||
count_true_positives[class_name] = 0
|
||||
dr_file = TEMP_FILES_PATH + "/" + class_name + "_dr.json"
|
||||
dr_data = json.load(open(dr_file))
|
||||
|
||||
nd = len(dr_data)
|
||||
tp = [0] * nd
|
||||
fp = [0] * nd
|
||||
score = [0] * nd
|
||||
score_threhold_idx = 0
|
||||
for idx, detection in enumerate(dr_data):
|
||||
file_id = detection["file_id"]
|
||||
score[idx] = float(detection["confidence"])
|
||||
if score[idx] >= score_threhold:
|
||||
score_threhold_idx = idx
|
||||
|
||||
if show_animation:
|
||||
ground_truth_img = glob.glob1(IMG_PATH, file_id + ".*")
|
||||
if len(ground_truth_img) == 0:
|
||||
error("Error. Image not found with id: " + file_id)
|
||||
elif len(ground_truth_img) > 1:
|
||||
error("Error. Multiple image with id: " + file_id)
|
||||
else:
|
||||
img = cv2.imread(IMG_PATH + "/" + ground_truth_img[0])
|
||||
img_cumulative_path = RESULTS_FILES_PATH + "/images/" + ground_truth_img[0]
|
||||
if os.path.isfile(img_cumulative_path):
|
||||
img_cumulative = cv2.imread(img_cumulative_path)
|
||||
else:
|
||||
img_cumulative = img.copy()
|
||||
bottom_border = 60
|
||||
BLACK = [0, 0, 0]
|
||||
img = cv2.copyMakeBorder(img, 0, bottom_border, 0, 0, cv2.BORDER_CONSTANT, value=BLACK)
|
||||
|
||||
gt_file = TEMP_FILES_PATH + "/" + file_id + "_ground_truth.json"
|
||||
ground_truth_data = json.load(open(gt_file))
|
||||
ovmax = -1
|
||||
gt_match = -1
|
||||
bb = [float(x) for x in detection["bbox"].split()]
|
||||
for obj in ground_truth_data:
|
||||
if obj["class_name"] == class_name:
|
||||
bbgt = [float(x) for x in obj["bbox"].split()]
|
||||
bi = [max(bb[0], bbgt[0]), max(bb[1], bbgt[1]), min(bb[2], bbgt[2]), min(bb[3], bbgt[3])]
|
||||
iw = bi[2] - bi[0] + 1
|
||||
ih = bi[3] - bi[1] + 1
|
||||
if iw > 0 and ih > 0:
|
||||
ua = (bb[2] - bb[0] + 1) * (bb[3] - bb[1] + 1) + (bbgt[2] - bbgt[0]
|
||||
+ 1) * (bbgt[3] - bbgt[1] + 1) - iw * ih
|
||||
ov = iw * ih / ua
|
||||
if ov > ovmax:
|
||||
ovmax = ov
|
||||
gt_match = obj
|
||||
|
||||
if show_animation:
|
||||
status = "NO MATCH FOUND!"
|
||||
|
||||
min_overlap = MINOVERLAP
|
||||
if ovmax >= min_overlap:
|
||||
if "difficult" not in gt_match:
|
||||
if not bool(gt_match["used"]):
|
||||
tp[idx] = 1
|
||||
gt_match["used"] = True
|
||||
count_true_positives[class_name] += 1
|
||||
with open(gt_file, 'w') as f:
|
||||
f.write(json.dumps(ground_truth_data))
|
||||
if show_animation:
|
||||
status = "MATCH!"
|
||||
else:
|
||||
fp[idx] = 1
|
||||
if show_animation:
|
||||
status = "REPEATED MATCH!"
|
||||
else:
|
||||
fp[idx] = 1
|
||||
if ovmax > 0:
|
||||
status = "INSUFFICIENT OVERLAP"
|
||||
|
||||
"""
|
||||
Draw image to show animation
|
||||
"""
|
||||
if show_animation:
|
||||
height, widht = img.shape[:2]
|
||||
white = (255, 255, 255)
|
||||
light_blue = (255, 200, 100)
|
||||
green = (0, 255, 0)
|
||||
light_red = (30, 30, 255)
|
||||
margin = 10
|
||||
# 1nd line
|
||||
v_pos = int(height - margin - (bottom_border / 2.0))
|
||||
text = "Image: " + ground_truth_img[0] + " "
|
||||
img, line_width = draw_text_in_image(img, text, (margin, v_pos), white, 0)
|
||||
text = "Class [" + str(class_index) + "/" + str(n_classes) + "]: " + class_name + " "
|
||||
img, line_width = draw_text_in_image(img, text, (margin + line_width, v_pos), light_blue,
|
||||
line_width)
|
||||
if ovmax != -1:
|
||||
color = light_red
|
||||
if status == "INSUFFICIENT OVERLAP":
|
||||
text = "IoU: {0:.2f}% ".format(ovmax * 100) + "< {0:.2f}% ".format(min_overlap * 100)
|
||||
else:
|
||||
text = "IoU: {0:.2f}% ".format(ovmax * 100) + ">= {0:.2f}% ".format(min_overlap * 100)
|
||||
color = green
|
||||
img, _ = draw_text_in_image(img, text, (margin + line_width, v_pos), color, line_width)
|
||||
# 2nd line
|
||||
v_pos += int(bottom_border / 2.0)
|
||||
rank_pos = str(idx + 1)
|
||||
text = "Detection #rank: " + rank_pos + " confidence: {0:.2f}% ".format(
|
||||
float(detection["confidence"]) * 100)
|
||||
img, line_width = draw_text_in_image(img, text, (margin, v_pos), white, 0)
|
||||
color = light_red
|
||||
if status == "MATCH!":
|
||||
color = green
|
||||
text = "Result: " + status + " "
|
||||
img, line_width = draw_text_in_image(img, text, (margin + line_width, v_pos), color, line_width)
|
||||
|
||||
font = cv2.FONT_HERSHEY_SIMPLEX
|
||||
if ovmax > 0:
|
||||
bbgt = [int(round(float(x))) for x in gt_match["bbox"].split()]
|
||||
cv2.rectangle(img, (bbgt[0], bbgt[1]), (bbgt[2], bbgt[3]), light_blue, 2)
|
||||
cv2.rectangle(img_cumulative, (bbgt[0], bbgt[1]), (bbgt[2], bbgt[3]), light_blue, 2)
|
||||
cv2.putText(img_cumulative, class_name, (bbgt[0], bbgt[1] - 5), font, 0.6, light_blue, 1,
|
||||
cv2.LINE_AA)
|
||||
bb = [int(i) for i in bb]
|
||||
cv2.rectangle(img, (bb[0], bb[1]), (bb[2], bb[3]), color, 2)
|
||||
cv2.rectangle(img_cumulative, (bb[0], bb[1]), (bb[2], bb[3]), color, 2)
|
||||
cv2.putText(img_cumulative, class_name, (bb[0], bb[1] - 5), font, 0.6, color, 1, cv2.LINE_AA)
|
||||
|
||||
cv2.imshow("Animation", img)
|
||||
cv2.waitKey(20)
|
||||
output_img_path = RESULTS_FILES_PATH + "/images/detections_one_by_one/" + class_name + "_detection" + str(
|
||||
idx) + ".jpg"
|
||||
cv2.imwrite(output_img_path, img)
|
||||
cv2.imwrite(img_cumulative_path, img_cumulative)
|
||||
|
||||
cumsum = 0
|
||||
for idx, val in enumerate(fp):
|
||||
fp[idx] += cumsum
|
||||
cumsum += val
|
||||
|
||||
cumsum = 0
|
||||
for idx, val in enumerate(tp):
|
||||
tp[idx] += cumsum
|
||||
cumsum += val
|
||||
|
||||
rec = tp[:]
|
||||
for idx, val in enumerate(tp):
|
||||
rec[idx] = float(tp[idx]) / np.maximum(gt_counter_per_class[class_name], 1)
|
||||
|
||||
prec = tp[:]
|
||||
for idx, val in enumerate(tp):
|
||||
prec[idx] = float(tp[idx]) / np.maximum((fp[idx] + tp[idx]), 1)
|
||||
|
||||
ap, mrec, mprec = voc_ap(rec[:], prec[:])
|
||||
F1 = np.array(rec) * np.array(prec) * 2 / np.where((np.array(prec) + np.array(rec)) == 0, 1,
|
||||
(np.array(prec) + np.array(rec)))
|
||||
|
||||
sum_AP += ap
|
||||
text = "{0:.2f}%".format(
|
||||
ap * 100) + " = " + class_name + " AP " # class_name + " AP = {0:.2f}%".format(ap*100)
|
||||
|
||||
if len(prec) > 0:
|
||||
F1_text = "{0:.2f}".format(F1[score_threhold_idx]) + " = " + class_name + " F1 "
|
||||
Recall_text = "{0:.2f}%".format(rec[score_threhold_idx] * 100) + " = " + class_name + " Recall "
|
||||
Precision_text = "{0:.2f}%".format(prec[score_threhold_idx] * 100) + " = " + class_name + " Precision "
|
||||
else:
|
||||
F1_text = "0.00" + " = " + class_name + " F1 "
|
||||
Recall_text = "0.00%" + " = " + class_name + " Recall "
|
||||
Precision_text = "0.00%" + " = " + class_name + " Precision "
|
||||
|
||||
rounded_prec = ['%.2f' % elem for elem in prec]
|
||||
rounded_rec = ['%.2f' % elem for elem in rec]
|
||||
results_file.write(text + "\n Precision: " + str(rounded_prec) + "\n Recall :" + str(rounded_rec) + "\n\n")
|
||||
|
||||
if len(prec) > 0:
|
||||
print(text + "\t||\tscore_threhold=" + str(score_threhold) + " : " + "F1=" + "{0:.2f}".format(
|
||||
F1[score_threhold_idx]) \
|
||||
+ " ; Recall=" + "{0:.2f}%".format(
|
||||
rec[score_threhold_idx] * 100) + " ; Precision=" + "{0:.2f}%".format(
|
||||
prec[score_threhold_idx] * 100))
|
||||
else:
|
||||
print(text + "\t||\tscore_threhold=" + str(
|
||||
score_threhold) + " : " + "F1=0.00% ; Recall=0.00% ; Precision=0.00%")
|
||||
ap_dictionary[class_name] = ap
|
||||
|
||||
n_images = counter_images_per_class[class_name]
|
||||
lamr, mr, fppi = log_average_miss_rate(np.array(rec), np.array(fp), n_images)
|
||||
lamr_dictionary[class_name] = lamr
|
||||
|
||||
if draw_plot:
|
||||
plt.plot(rec, prec, '-o')
|
||||
area_under_curve_x = mrec[:-1] + [mrec[-2]] + [mrec[-1]]
|
||||
area_under_curve_y = mprec[:-1] + [0.0] + [mprec[-1]]
|
||||
plt.fill_between(area_under_curve_x, 0, area_under_curve_y, alpha=0.2, edgecolor='r')
|
||||
|
||||
fig = plt.gcf()
|
||||
fig.canvas.set_window_title('AP ' + class_name)
|
||||
|
||||
plt.title('class: ' + text)
|
||||
plt.xlabel('Recall')
|
||||
plt.ylabel('Precision')
|
||||
axes = plt.gca()
|
||||
axes.set_xlim([0.0, 1.0])
|
||||
axes.set_ylim([0.0, 1.05])
|
||||
fig.savefig(RESULTS_FILES_PATH + "/AP/" + class_name + ".png")
|
||||
plt.cla()
|
||||
|
||||
plt.plot(score, F1, "-", color='orangered')
|
||||
plt.title('class: ' + F1_text + "\nscore_threhold=" + str(score_threhold))
|
||||
plt.xlabel('Score_Threhold')
|
||||
plt.ylabel('F1')
|
||||
axes = plt.gca()
|
||||
axes.set_xlim([0.0, 1.0])
|
||||
axes.set_ylim([0.0, 1.05])
|
||||
fig.savefig(RESULTS_FILES_PATH + "/F1/" + class_name + ".png")
|
||||
plt.cla()
|
||||
|
||||
plt.plot(score, rec, "-H", color='gold')
|
||||
plt.title('class: ' + Recall_text + "\nscore_threhold=" + str(score_threhold))
|
||||
plt.xlabel('Score_Threhold')
|
||||
plt.ylabel('Recall')
|
||||
axes = plt.gca()
|
||||
axes.set_xlim([0.0, 1.0])
|
||||
axes.set_ylim([0.0, 1.05])
|
||||
fig.savefig(RESULTS_FILES_PATH + "/Recall/" + class_name + ".png")
|
||||
plt.cla()
|
||||
|
||||
plt.plot(score, prec, "-s", color='palevioletred')
|
||||
plt.title('class: ' + Precision_text + "\nscore_threhold=" + str(score_threhold))
|
||||
plt.xlabel('Score_Threhold')
|
||||
plt.ylabel('Precision')
|
||||
axes = plt.gca()
|
||||
axes.set_xlim([0.0, 1.0])
|
||||
axes.set_ylim([0.0, 1.05])
|
||||
fig.savefig(RESULTS_FILES_PATH + "/Precision/" + class_name + ".png")
|
||||
plt.cla()
|
||||
|
||||
if show_animation:
|
||||
cv2.destroyAllWindows()
|
||||
if n_classes == 0:
|
||||
print("未检测到任何种类，请检查标签信息与get_map.py中的classes_path是否修改。")
|
||||
return 0
|
||||
results_file.write("\n# mAP of all classes\n")
|
||||
mAP = sum_AP / n_classes
|
||||
text = "mAP = {0:.2f}%".format(mAP * 100)
|
||||
results_file.write(text + "\n")
|
||||
print(text)
|
||||
|
||||
shutil.rmtree(TEMP_FILES_PATH)
|
||||
|
||||
"""
|
||||
Count total of detection-results
|
||||
"""
|
||||
det_counter_per_class = {}
|
||||
for txt_file in dr_files_list:
|
||||
lines_list = file_lines_to_list(txt_file)
|
||||
for line in lines_list:
|
||||
class_name = line.split()[0]
|
||||
if class_name in det_counter_per_class:
|
||||
det_counter_per_class[class_name] += 1
|
||||
else:
|
||||
det_counter_per_class[class_name] = 1
|
||||
dr_classes = list(det_counter_per_class.keys())
|
||||
|
||||
"""
|
||||
Write number of ground-truth objects per class to results.txt
|
||||
"""
|
||||
with open(RESULTS_FILES_PATH + "/results.txt", 'a') as results_file:
|
||||
results_file.write("\n# Number of ground-truth objects per class\n")
|
||||
for class_name in sorted(gt_counter_per_class):
|
||||
results_file.write(class_name + ": " + str(gt_counter_per_class[class_name]) + "\n")
|
||||
|
||||
"""
|
||||
Finish counting true positives
|
||||
"""
|
||||
for class_name in dr_classes:
|
||||
if class_name not in gt_classes:
|
||||
count_true_positives[class_name] = 0
|
||||
|
||||
"""
|
||||
Write number of detected objects per class to results.txt
|
||||
"""
|
||||
with open(RESULTS_FILES_PATH + "/results.txt", 'a') as results_file:
|
||||
results_file.write("\n# Number of detected objects per class\n")
|
||||
for class_name in sorted(dr_classes):
|
||||
n_det = det_counter_per_class[class_name]
|
||||
text = class_name + ": " + str(n_det)
|
||||
text += " (tp:" + str(count_true_positives[class_name]) + ""
|
||||
text += ", fp:" + str(n_det - count_true_positives[class_name]) + ")\n"
|
||||
results_file.write(text)
|
||||
|
||||
"""
|
||||
Plot the total number of occurences of each class in the ground-truth
|
||||
"""
|
||||
if draw_plot:
|
||||
window_title = "ground-truth-info"
|
||||
plot_title = "ground-truth\n"
|
||||
plot_title += "(" + str(len(ground_truth_files_list)) + " files and " + str(n_classes) + " classes)"
|
||||
x_label = "Number of objects per class"
|
||||
output_path = RESULTS_FILES_PATH + "/ground-truth-info.png"
|
||||
to_show = False
|
||||
plot_color = 'forestgreen'
|
||||
draw_plot_func(
|
||||
gt_counter_per_class,
|
||||
n_classes,
|
||||
window_title,
|
||||
plot_title,
|
||||
x_label,
|
||||
output_path,
|
||||
to_show,
|
||||
plot_color,
|
||||
'',
|
||||
)
|
||||
|
||||
# """
|
||||
# Plot the total number of occurences of each class in the "detection-results" folder
|
||||
# """
|
||||
# if draw_plot:
|
||||
# window_title = "detection-results-info"
|
||||
# # Plot title
|
||||
# plot_title = "detection-results\n"
|
||||
# plot_title += "(" + str(len(dr_files_list)) + " files and "
|
||||
# count_non_zero_values_in_dictionary = sum(int(x) > 0 for x in list(det_counter_per_class.values()))
|
||||
# plot_title += str(count_non_zero_values_in_dictionary) + " detected classes)"
|
||||
# # end Plot title
|
||||
# x_label = "Number of objects per class"
|
||||
# output_path = RESULTS_FILES_PATH + "/detection-results-info.png"
|
||||
# to_show = False
|
||||
# plot_color = 'forestgreen'
|
||||
# true_p_bar = count_true_positives
|
||||
# draw_plot_func(
|
||||
# det_counter_per_class,
|
||||
# len(det_counter_per_class),
|
||||
# window_title,
|
||||
# plot_title,
|
||||
# x_label,
|
||||
# output_path,
|
||||
# to_show,
|
||||
# plot_color,
|
||||
# true_p_bar
|
||||
# )
|
||||
|
||||
"""
|
||||
Draw log-average miss rate plot (Show lamr of all classes in decreasing order)
|
||||
"""
|
||||
if draw_plot:
|
||||
window_title = "lamr"
|
||||
plot_title = "log-average miss rate"
|
||||
x_label = "log-average miss rate"
|
||||
output_path = RESULTS_FILES_PATH + "/lamr.png"
|
||||
to_show = False
|
||||
plot_color = 'royalblue'
|
||||
draw_plot_func(
|
||||
lamr_dictionary,
|
||||
n_classes,
|
||||
window_title,
|
||||
plot_title,
|
||||
x_label,
|
||||
output_path,
|
||||
to_show,
|
||||
plot_color,
|
||||
""
|
||||
)
|
||||
|
||||
"""
|
||||
Draw mAP plot (Show AP's of all classes in decreasing order)
|
||||
"""
|
||||
if draw_plot:
|
||||
window_title = "mAP"
|
||||
plot_title = "mAP = {0:.2f}%".format(mAP * 100)
|
||||
x_label = "Average Precision"
|
||||
output_path = RESULTS_FILES_PATH + "/mAP.png"
|
||||
to_show = True
|
||||
plot_color = 'royalblue'
|
||||
draw_plot_func(
|
||||
ap_dictionary,
|
||||
n_classes,
|
||||
window_title,
|
||||
plot_title,
|
||||
x_label,
|
||||
output_path,
|
||||
to_show,
|
||||
plot_color,
|
||||
""
|
||||
)
|
||||
return mAP
|
||||
|
||||
|
||||
def preprocess_gt(gt_path, class_names):
|
||||
image_ids = os.listdir(gt_path)
|
||||
results = {}
|
||||
|
||||
images = []
|
||||
bboxes = []
|
||||
for i, image_id in enumerate(image_ids):
|
||||
lines_list = file_lines_to_list(os.path.join(gt_path, image_id))
|
||||
boxes_per_image = []
|
||||
image = {}
|
||||
image_id = os.path.splitext(image_id)[0]
|
||||
image['file_name'] = image_id + '.jpg'
|
||||
image['width'] = 1
|
||||
image['height'] = 1
|
||||
# -----------------------------------------------------------------#
|
||||
# 感谢 多学学英语吧 的提醒
|
||||
# 解决了'Results do not correspond to current coco set'问题
|
||||
# -----------------------------------------------------------------#
|
||||
image['id'] = str(image_id)
|
||||
|
||||
for line in lines_list:
|
||||
difficult = 0
|
||||
if "difficult" in line:
|
||||
line_split = line.split()
|
||||
left, top, right, bottom, _difficult = line_split[-5:]
|
||||
class_name = ""
|
||||
for name in line_split[:-5]:
|
||||
class_name += name + " "
|
||||
class_name = class_name[:-1]
|
||||
difficult = 1
|
||||
else:
|
||||
line_split = line.split()
|
||||
left, top, right, bottom = line_split[-4:]
|
||||
class_name = ""
|
||||
for name in line_split[:-4]:
|
||||
class_name += name + " "
|
||||
class_name = class_name[:-1]
|
||||
|
||||
left, top, right, bottom = float(left), float(top), float(right), float(bottom)
|
||||
if class_name not in class_names:
|
||||
continue
|
||||
cls_id = class_names.index(class_name) + 1
|
||||
bbox = [left, top, right - left, bottom - top, difficult, str(image_id), cls_id,
|
||||
(right - left) * (bottom - top) - 10.0]
|
||||
boxes_per_image.append(bbox)
|
||||
images.append(image)
|
||||
bboxes.extend(boxes_per_image)
|
||||
results['images'] = images
|
||||
|
||||
categories = []
|
||||
for i, cls in enumerate(class_names):
|
||||
category = {}
|
||||
category['supercategory'] = cls
|
||||
category['name'] = cls
|
||||
category['id'] = i + 1
|
||||
categories.append(category)
|
||||
results['categories'] = categories
|
||||
|
||||
annotations = []
|
||||
for i, box in enumerate(bboxes):
|
||||
annotation = {}
|
||||
annotation['area'] = box[-1]
|
||||
annotation['category_id'] = box[-2]
|
||||
annotation['image_id'] = box[-3]
|
||||
annotation['iscrowd'] = box[-4]
|
||||
annotation['bbox'] = box[:4]
|
||||
annotation['id'] = i
|
||||
annotations.append(annotation)
|
||||
results['annotations'] = annotations
|
||||
return results
|
||||
|
||||
|
||||
def preprocess_dr(dr_path, class_names):
|
||||
image_ids = os.listdir(dr_path)
|
||||
results = []
|
||||
for image_id in image_ids:
|
||||
lines_list = file_lines_to_list(os.path.join(dr_path, image_id))
|
||||
image_id = os.path.splitext(image_id)[0]
|
||||
for line in lines_list:
|
||||
line_split = line.split()
|
||||
confidence, left, top, right, bottom = line_split[-5:]
|
||||
class_name = ""
|
||||
for name in line_split[:-5]:
|
||||
class_name += name + " "
|
||||
class_name = class_name[:-1]
|
||||
left, top, right, bottom = float(left), float(top), float(right), float(bottom)
|
||||
result = {}
|
||||
result["image_id"] = str(image_id)
|
||||
if class_name not in class_names:
|
||||
continue
|
||||
result["category_id"] = class_names.index(class_name) + 1
|
||||
result["bbox"] = [left, top, right - left, bottom - top]
|
||||
result["score"] = float(confidence)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
def get_coco_map(class_names, path):
|
||||
GT_PATH = os.path.join(path, 'ground-truth')
|
||||
DR_PATH = os.path.join(path, 'detection-results')
|
||||
COCO_PATH = os.path.join(path, 'coco_eval')
|
||||
|
||||
if not os.path.exists(COCO_PATH):
|
||||
os.makedirs(COCO_PATH)
|
||||
|
||||
GT_JSON_PATH = os.path.join(COCO_PATH, 'instances_gt.json')
|
||||
DR_JSON_PATH = os.path.join(COCO_PATH, 'instances_dr.json')
|
||||
|
||||
with open(GT_JSON_PATH, "w") as f:
|
||||
results_gt = preprocess_gt(GT_PATH, class_names)
|
||||
json.dump(results_gt, f, indent=4)
|
||||
|
||||
with open(DR_JSON_PATH, "w") as f:
|
||||
results_dr = preprocess_dr(DR_PATH, class_names)
|
||||
json.dump(results_dr, f, indent=4)
|
||||
if len(results_dr) == 0:
|
||||
print("未检测到任何目标。")
|
||||
return [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
|
||||
|
||||
cocoGt = COCO(GT_JSON_PATH)
|
||||
cocoDt = cocoGt.loadRes(DR_JSON_PATH)
|
||||
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
|
||||
cocoEval.evaluate()
|
||||
cocoEval.accumulate()
|
||||
cocoEval.summarize()
|
||||
|
||||
return cocoEval.stats
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
# -------------------------------------------------------#
|
||||
# 用于处理COCO数据集，根据json文件生成txt文件用于训练
|
||||
# -------------------------------------------------------#
|
||||
import json
|
||||
import os
|
||||
from collections import defaultdict
|
||||
|
||||
# -------------------------------------------------------#
|
||||
# 指向了COCO训练集与验证集图片的路径
|
||||
# -------------------------------------------------------#
|
||||
train_datasets_path = "coco_dataset/train2017"
|
||||
val_datasets_path = "coco_dataset/val2017"
|
||||
|
||||
# -------------------------------------------------------#
|
||||
# 指向了COCO训练集与验证集标签的路径
|
||||
# -------------------------------------------------------#
|
||||
train_annotation_path = "coco_dataset/annotations/instances_train2017.json"
|
||||
val_annotation_path = "coco_dataset/annotations/instances_val2017.json"
|
||||
|
||||
# -------------------------------------------------------#
|
||||
# 生成的txt文件路径
|
||||
# -------------------------------------------------------#
|
||||
train_output_path = "coco_train.txt"
|
||||
val_output_path = "coco_val.txt"
|
||||
|
||||
if __name__ == "__main__":
|
||||
name_box_id = defaultdict(list)
|
||||
id_name = dict()
|
||||
f = open(train_annotation_path, encoding='utf-8')
|
||||
data = json.load(f)
|
||||
|
||||
annotations = data['annotations']
|
||||
for ant in annotations:
|
||||
id = ant['image_id']
|
||||
name = os.path.join(train_datasets_path, '%012d.jpg' % id)
|
||||
cat = ant['category_id']
|
||||
if cat >= 1 and cat <= 11:
|
||||
cat = cat - 1
|
||||
elif cat >= 13 and cat <= 25:
|
||||
cat = cat - 2
|
||||
elif cat >= 27 and cat <= 28:
|
||||
cat = cat - 3
|
||||
elif cat >= 31 and cat <= 44:
|
||||
cat = cat - 5
|
||||
elif cat >= 46 and cat <= 65:
|
||||
cat = cat - 6
|
||||
elif cat == 67:
|
||||
cat = cat - 7
|
||||
elif cat == 70:
|
||||
cat = cat - 9
|
||||
elif cat >= 72 and cat <= 82:
|
||||
cat = cat - 10
|
||||
elif cat >= 84 and cat <= 90:
|
||||
cat = cat - 11
|
||||
name_box_id[name].append([ant['bbox'], cat])
|
||||
|
||||
f = open(train_output_path, 'w')
|
||||
for key in name_box_id.keys():
|
||||
f.write(key)
|
||||
box_infos = name_box_id[key]
|
||||
for info in box_infos:
|
||||
x_min = int(info[0][0])
|
||||
y_min = int(info[0][1])
|
||||
x_max = x_min + int(info[0][2])
|
||||
y_max = y_min + int(info[0][3])
|
||||
|
||||
box_info = " %d,%d,%d,%d,%d" % (
|
||||
x_min, y_min, x_max, y_max, int(info[1]))
|
||||
f.write(box_info)
|
||||
f.write('\n')
|
||||
f.close()
|
||||
|
||||
name_box_id = defaultdict(list)
|
||||
id_name = dict()
|
||||
f = open(val_annotation_path, encoding='utf-8')
|
||||
data = json.load(f)
|
||||
|
||||
annotations = data['annotations']
|
||||
for ant in annotations:
|
||||
id = ant['image_id']
|
||||
name = os.path.join(val_datasets_path, '%012d.jpg' % id)
|
||||
cat = ant['category_id']
|
||||
if cat >= 1 and cat <= 11:
|
||||
cat = cat - 1
|
||||
elif cat >= 13 and cat <= 25:
|
||||
cat = cat - 2
|
||||
elif cat >= 27 and cat <= 28:
|
||||
cat = cat - 3
|
||||
elif cat >= 31 and cat <= 44:
|
||||
cat = cat - 5
|
||||
elif cat >= 46 and cat <= 65:
|
||||
cat = cat - 6
|
||||
elif cat == 67:
|
||||
cat = cat - 7
|
||||
elif cat == 70:
|
||||
cat = cat - 9
|
||||
elif cat >= 72 and cat <= 82:
|
||||
cat = cat - 10
|
||||
elif cat >= 84 and cat <= 90:
|
||||
cat = cat - 11
|
||||
name_box_id[name].append([ant['bbox'], cat])
|
||||
|
||||
f = open(val_output_path, 'w')
|
||||
for key in name_box_id.keys():
|
||||
f.write(key)
|
||||
box_infos = name_box_id[key]
|
||||
for info in box_infos:
|
||||
x_min = int(info[0][0])
|
||||
y_min = int(info[0][1])
|
||||
x_max = x_min + int(info[0][2])
|
||||
y_max = y_min + int(info[0][3])
|
||||
|
||||
box_info = " %d,%d,%d,%d,%d" % (
|
||||
x_min, y_min, x_max, y_max, int(info[1]))
|
||||
f.write(box_info)
|
||||
f.write('\n')
|
||||
f.close()
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
import json
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from pycocotools.coco import COCO
|
||||
from pycocotools.cocoeval import COCOeval
|
||||
from tqdm import tqdm
|
||||
|
||||
from utils.utils import cvtColor, preprocess_input, resize_image
|
||||
from yolo import YOLO
|
||||
|
||||
# ---------------------------------------------------------------------------#
|
||||
# map_mode用于指定该文件运行时计算的内容
|
||||
# map_mode为0代表整个map计算流程，包括获得预测结果、计算map。
|
||||
# map_mode为1代表仅仅获得预测结果。
|
||||
# map_mode为2代表仅仅获得计算map。
|
||||
# ---------------------------------------------------------------------------#
|
||||
map_mode = 0
|
||||
# -------------------------------------------------------#
|
||||
# 指向了验证集标签与图片路径
|
||||
# -------------------------------------------------------#
|
||||
cocoGt_path = 'coco_dataset/annotations/instances_val2017.json'
|
||||
dataset_img_path = 'coco_dataset/val2017'
|
||||
# -------------------------------------------------------#
|
||||
# 结果输出的文件夹，默认为map_out
|
||||
# -------------------------------------------------------#
|
||||
temp_save_path = 'map_out/coco_eval'
|
||||
|
||||
|
||||
class mAP_YOLO(YOLO):
|
||||
# ---------------------------------------------------#
|
||||
# 检测图片
|
||||
# ---------------------------------------------------#
|
||||
def detect_image(self, image_id, image, results):
|
||||
# ---------------------------------------------------#
|
||||
# 计算输入图片的高和宽
|
||||
# ---------------------------------------------------#
|
||||
image_shape = np.array(np.shape(image)[0:2])
|
||||
# ---------------------------------------------------------#
|
||||
# 在这里将图像转换成RGB图像，防止灰度图在预测时报错。
|
||||
# 代码仅仅支持RGB图像的预测，所有其它类型的图像都会转化成RGB
|
||||
# ---------------------------------------------------------#
|
||||
image = cvtColor(image)
|
||||
# ---------------------------------------------------------#
|
||||
# 给图像增加灰条，实现不失真的resize
|
||||
# 也可以直接resize进行识别
|
||||
# ---------------------------------------------------------#
|
||||
image_data = resize_image(image, (self.input_shape[1], self.input_shape[0]), self.letterbox_image)
|
||||
# ---------------------------------------------------------#
|
||||
# 添加上batch_size维度
|
||||
# ---------------------------------------------------------#
|
||||
image_data = np.expand_dims(np.transpose(preprocess_input(np.array(image_data, dtype='float32')), (2, 0, 1)), 0)
|
||||
|
||||
with torch.no_grad():
|
||||
images = torch.from_numpy(image_data)
|
||||
if self.cuda:
|
||||
images = images.cuda()
|
||||
# ---------------------------------------------------------#
|
||||
# 将图像输入网络当中进行预测！
|
||||
# ---------------------------------------------------------#
|
||||
outputs = self.net(images)
|
||||
outputs = self.bbox_util.decode_box(outputs)
|
||||
# ---------------------------------------------------------#
|
||||
# 将预测框进行堆叠，然后进行非极大抑制
|
||||
# ---------------------------------------------------------#
|
||||
outputs = self.bbox_util.non_max_suppression(torch.cat(outputs, 1), self.num_classes, self.input_shape,
|
||||
image_shape, self.letterbox_image, conf_thres=self.confidence,
|
||||
nms_thres=self.nms_iou)
|
||||
|
||||
if outputs[0] is None:
|
||||
return results
|
||||
|
||||
top_label = np.array(outputs[0][:, 6], dtype='int32')
|
||||
top_conf = outputs[0][:, 4] * outputs[0][:, 5]
|
||||
top_boxes = outputs[0][:, :4]
|
||||
|
||||
for i, c in enumerate(top_label):
|
||||
result = {}
|
||||
top, left, bottom, right = top_boxes[i]
|
||||
|
||||
result["image_id"] = int(image_id)
|
||||
result["category_id"] = clsid2catid[c]
|
||||
result["bbox"] = [float(left), float(top), float(right - left), float(bottom - top)]
|
||||
result["score"] = float(top_conf[i])
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not os.path.exists(temp_save_path):
|
||||
os.makedirs(temp_save_path)
|
||||
|
||||
cocoGt = COCO(cocoGt_path)
|
||||
ids = list(cocoGt.imgToAnns.keys())
|
||||
clsid2catid = cocoGt.getCatIds()
|
||||
|
||||
if map_mode == 0 or map_mode == 1:
|
||||
yolo = mAP_YOLO(confidence=0.001, nms_iou=0.65)
|
||||
|
||||
with open(os.path.join(temp_save_path, 'eval_results.json'), "w") as f:
|
||||
results = []
|
||||
for image_id in tqdm(ids):
|
||||
image_path = os.path.join(dataset_img_path, cocoGt.loadImgs(image_id)[0]['file_name'])
|
||||
image = Image.open(image_path)
|
||||
results = yolo.detect_image(image_id, image, results)
|
||||
json.dump(results, f)
|
||||
|
||||
if map_mode == 0 or map_mode == 2:
|
||||
cocoDt = cocoGt.loadRes(os.path.join(temp_save_path, 'eval_results.json'))
|
||||
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
|
||||
cocoEval.evaluate()
|
||||
cocoEval.accumulate()
|
||||
cocoEval.summarize()
|
||||
print("Get map done.")
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
import os
|
||||
import random
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import numpy as np
|
||||
|
||||
from utils.utils import get_classes
|
||||
|
||||
# --------------------------------------------------------------------------------------------------------------------------------#
|
||||
# annotation_mode用于指定该文件运行时计算的内容
|
||||
# annotation_mode为0代表整个标签处理过程，包括获得VOCdevkit/VOC2007/ImageSets里面的txt以及训练用的2007_train.txt、2007_val.txt
|
||||
# annotation_mode为1代表获得VOCdevkit/VOC2007/ImageSets里面的txt
|
||||
# annotation_mode为2代表获得训练用的2007_train.txt、2007_val.txt
|
||||
# --------------------------------------------------------------------------------------------------------------------------------#
|
||||
annotation_mode = 0
|
||||
# -------------------------------------------------------------------#
|
||||
# 必须要修改，用于生成2007_train.txt、2007_val.txt的目标信息
|
||||
# 与训练和预测所用的classes_path一致即可
|
||||
# 如果生成的2007_train.txt里面没有目标信息
|
||||
# 那么就是因为classes没有设定正确
|
||||
# 仅在annotation_mode为0和2的时候有效
|
||||
# -------------------------------------------------------------------#
|
||||
classes_path = 'model_data/voc_classes.txt' # 这里定义的名字是xml的物体的名字，出现的顺序是训练时的onehot顺序。
|
||||
# --------------------------------------------------------------------------------------------------------------------------------#
|
||||
# trainval_percent用于指定(训练集+验证集)与测试集的比例，默认情况下 (训练集+验证集):测试集 = 9:1
|
||||
# train_percent用于指定(训练集+验证集)中训练集与验证集的比例，默认情况下 训练集:验证集 = 9:1
|
||||
# 仅在annotation_mode为0和1的时候有效
|
||||
# --------------------------------------------------------------------------------------------------------------------------------#
|
||||
trainval_percent = 0.9
|
||||
train_percent = 0.9
|
||||
# -------------------------------------------------------#
|
||||
# 指向VOC数据集所在的文件夹
|
||||
# 默认指向根目录下的VOC数据集
|
||||
# -------------------------------------------------------#
|
||||
VOCdevkit_path = 'VOCdevkit'
|
||||
|
||||
VOCdevkit_sets = [('2007', 'train'), ('2007', 'val')]
|
||||
classes, _ = get_classes(classes_path)
|
||||
|
||||
# -------------------------------------------------------#
|
||||
# 统计目标数量
|
||||
# -------------------------------------------------------#
|
||||
photo_nums = np.zeros(len(VOCdevkit_sets)) # 生成train的数目，val的数目
|
||||
nums = np.zeros(len(classes)) # 统计各个类别的数量
|
||||
|
||||
|
||||
def convert_annotation(year, image_id, list_file):
|
||||
in_file = open(os.path.join(VOCdevkit_path, 'VOC%s/Annotations/%s.xml' % (year, image_id)), encoding='utf-8') # 'VOCdevkit\\VOC2007/Annotations/000001.xml'
|
||||
tree = ET.parse(in_file)
|
||||
root = tree.getroot()
|
||||
|
||||
for obj in root.iter('object'):
|
||||
difficult = 0
|
||||
if obj.find('difficult') != None:
|
||||
difficult = obj.find('difficult').text
|
||||
cls = obj.find('name').text
|
||||
if cls not in classes or int(difficult) == 1: # 不在classes里或者difficult为1，跳过当前类别
|
||||
continue
|
||||
cls_id = classes.index(cls) # 类别对应于classes文件的下标，是类别的id属性
|
||||
xmlbox = obj.find('bndbox')
|
||||
b = (int(float(xmlbox.find('xmin').text)), int(float(xmlbox.find('ymin').text)),
|
||||
int(float(xmlbox.find('xmax').text)), int(float(xmlbox.find('ymax').text)))
|
||||
list_file.write(" " + ",".join([str(a) for a in b]) + ',' + str(cls_id))
|
||||
# list_file的每一行，前面先写了图片的全路径，接着一个空格，依次写各个物体的 以,分隔的坐标，和id
|
||||
nums[classes.index(cls)] = nums[classes.index(cls)] + 1 # 统计各个类别的个数
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
random.seed(0)
|
||||
if " " in os.path.abspath(VOCdevkit_path):
|
||||
raise ValueError("数据集存放的文件夹路径与图片名称中不可以存在空格，否则会影响正常的模型训练，请注意修改。")
|
||||
|
||||
if annotation_mode == 0 or annotation_mode == 1:
|
||||
print("Generate txt in ImageSets.")
|
||||
xmlfilepath = os.path.join(VOCdevkit_path, 'VOC2007/Annotations')
|
||||
saveBasePath = os.path.join(VOCdevkit_path, 'VOC2007/ImageSets/Main')
|
||||
temp_xml = os.listdir(xmlfilepath)
|
||||
total_xml = [xml for xml in temp_xml if xml.endswith(".xml")]
|
||||
|
||||
num = len(total_xml) # 取得原始数据集中的总数，从总数中划分数据集
|
||||
list = range(num)
|
||||
tv = int(num * trainval_percent) # 训练+验证集 总数
|
||||
tr = int(tv * train_percent) # 训练+验证集中 训练集的总数
|
||||
trainval = random.sample(list, tv) # 在总数里采样
|
||||
train = random.sample(trainval, tr) # 在tv中采样tr
|
||||
|
||||
print("train and val size", tv)
|
||||
print("train size", tr)
|
||||
ftrainval = open(os.path.join(saveBasePath, 'trainval.txt'), 'w')
|
||||
ftest = open(os.path.join(saveBasePath, 'test.txt'), 'w')
|
||||
ftrain = open(os.path.join(saveBasePath, 'train.txt'), 'w')
|
||||
fval = open(os.path.join(saveBasePath, 'val.txt'), 'w')
|
||||
|
||||
for i in list:
|
||||
name = total_xml[i][:-4] + '\n' # 取出除了后缀的文件名字
|
||||
if i in trainval:
|
||||
ftrainval.write(name)
|
||||
if i in train:
|
||||
ftrain.write(name)
|
||||
else:
|
||||
fval.write(name)
|
||||
else:
|
||||
ftest.write(name)
|
||||
|
||||
ftrainval.close()
|
||||
ftrain.close()
|
||||
fval.close()
|
||||
ftest.close()
|
||||
print("Generate txt in ImageSets done.")
|
||||
|
||||
if annotation_mode == 0 or annotation_mode == 2:
|
||||
print("Generate 2007_train.txt and 2007_val.txt for train.")
|
||||
type_index = 0
|
||||
for year, image_set in VOCdevkit_sets:
|
||||
image_ids = open(os.path.join(VOCdevkit_path, 'VOC%s/ImageSets/Main/%s.txt' % (year, image_set)), # 'VOCdevkit\\VOC2007/ImageSets/Main/train.txt'
|
||||
encoding='utf-8').read().strip().split()
|
||||
list_file = open('%s_%s.txt' % (year, image_set), 'w', encoding='utf-8') # '2007_train.txt'
|
||||
for image_id in image_ids:
|
||||
list_file.write( # 'C:\\my_code\\a_python\\YOLO_all\\yolo_v3\\VOCdevkit/VOC2007/JPEGImages/000001.jpg'
|
||||
'%s/VOC%s/JPEGImages/%s.jpg' % (os.path.abspath(VOCdevkit_path), year, image_id)) # 文件全路径名字是拼出来的
|
||||
convert_annotation(year, image_id, list_file)
|
||||
list_file.write('\n')
|
||||
photo_nums[type_index] = len(image_ids) # 记录训练集总数和验证集总数
|
||||
type_index += 1 # 用来标记是操作 训练集还是验证集
|
||||
list_file.close()
|
||||
print("Generate 2007_train.txt and 2007_val.txt for train done.")
|
||||
|
||||
|
||||
def printTable(List1, List2):
|
||||
# for i in range(len(List1[0])):
|
||||
for i, _ in enumerate(List1[0]):
|
||||
print("|", end=' ')
|
||||
for j in range(len(List1)): # len(List1)为2
|
||||
print(List1[j][i].rjust(int(List2[j])), end=' ')
|
||||
print("|", end=' ')
|
||||
print()
|
||||
|
||||
|
||||
str_nums = [str(int(x)) for x in nums] # 每个类别的数目
|
||||
tableData = [
|
||||
classes, str_nums # 类别与数目对应
|
||||
]
|
||||
colWidths = [0] * len(tableData) # 计算列宽，共有len(tableData)列，这里是2
|
||||
len1 = 0
|
||||
for i in range(len(tableData)):
|
||||
for j in range(len(tableData[i])):
|
||||
if len(tableData[i][j]) > colWidths[i]:
|
||||
colWidths[i] = len(tableData[i][j]) # 每列中，每个元素的最大长度赋值给colWidths
|
||||
printTable(tableData, colWidths)
|
||||
|
||||
if photo_nums[0] <= 500:
|
||||
print("训练集数量小于500，属于较小的数据量，请注意设置较大的训练世代（Epoch）以满足足够的梯度下降次数（Step）。")
|
||||
|
||||
if np.sum(nums) == 0:
|
||||
print("在数据集中并未获得任何目标，请注意修改classes_path对应自己的数据集，并且保证标签名字正确，否则训练将会没有任何效果！")
|
||||
print("在数据集中并未获得任何目标，请注意修改classes_path对应自己的数据集，并且保证标签名字正确，否则训练将会没有任何效果！")
|
||||
print("在数据集中并未获得任何目标，请注意修改classes_path对应自己的数据集，并且保证标签名字正确，否则训练将会没有任何效果！")
|
||||
print("（重要的事情说三遍）。")
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
import time
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from yolo import YOLO
|
||||
|
||||
yolo = YOLO()
|
||||
|
||||
capture = cv2.VideoCapture(0)
|
||||
# 1 就是外接摄像头 0 就是自己的摄像头
|
||||
ref, frame = capture.read()
|
||||
fps = 0.0
|
||||
while (True):
|
||||
t1 = time.time()
|
||||
# 读取某一帧
|
||||
ref, frame = capture.read()
|
||||
if not ref:
|
||||
break
|
||||
# 格式转变，BGRtoRGB
|
||||
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
# 转变成Image
|
||||
frame = Image.fromarray(np.uint8(frame))
|
||||
# 进行检测
|
||||
frame = np.array(yolo.detect_image(frame))
|
||||
# RGBtoBGR满足opencv显示格式
|
||||
frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
|
||||
fps = (fps + (1. / (time.time() - t1))) / 2
|
||||
# print("fps= %.2f" % (fps))
|
||||
frame = cv2.putText(frame, "fps= %.2f" % (fps), (0, 40), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
|
||||
|
||||
cv2.imshow("video", frame)
|
||||
c = cv2.waitKey(1) & 0xff
|
||||
# print(c)
|
||||
if c == 113:
|
||||
capture.release()
|
||||
break
|
||||
|
||||
capture.release()
|
||||
cv2.destroyAllWindows()
|
||||
|
|
@ -0,0 +1,425 @@
|
|||
import colorsys
|
||||
import os
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from PIL import ImageDraw, ImageFont
|
||||
|
||||
from nets.yolo import YoloBody
|
||||
from utils.utils import (cvtColor, get_anchors, get_classes, preprocess_input,
|
||||
resize_image, show_config)
|
||||
from utils.utils_bbox import DecodeBox
|
||||
|
||||
'''
|
||||
训练自己的数据集必看注释！
|
||||
'''
|
||||
|
||||
|
||||
class YOLO(object):
|
||||
_defaults = {
|
||||
# --------------------------------------------------------------------------#
|
||||
# 使用自己训练好的模型进行预测一定要修改model_path和classes_path！
|
||||
# model_path指向logs文件夹下的权值文件，classes_path指向model_data下的txt
|
||||
#
|
||||
# 训练好后logs文件夹下存在多个权值文件，选择验证集损失较低的即可。
|
||||
# 验证集损失较低不代表mAP较高，仅代表该权值在验证集上泛化性能较好。
|
||||
# 如果出现shape不匹配，同时要注意训练时的model_path和classes_path参数的修改
|
||||
# --------------------------------------------------------------------------#
|
||||
# "model_path": 'model_data/yolo_weights.pth',
|
||||
# "classes_path": 'model_data/coco_classes.txt',
|
||||
"model_path": 'logs/best_epoch_weights.pth',
|
||||
"classes_path": 'model_data/cctsdb_classes.txt',
|
||||
# ---------------------------------------------------------------------#
|
||||
# anchors_path代表先验框对应的txt文件，一般不修改。
|
||||
# anchors_mask用于帮助代码找到对应的先验框，一般不修改。
|
||||
# ---------------------------------------------------------------------#
|
||||
"anchors_path": 'model_data/yolo_anchors.txt',
|
||||
"anchors_mask": [[6, 7, 8], [3, 4, 5], [0, 1, 2]],
|
||||
# ---------------------------------------------------------------------#
|
||||
# 输入图片的大小，必须为32的倍数。
|
||||
# ---------------------------------------------------------------------#
|
||||
"input_shape": [416, 416],
|
||||
# ---------------------------------------------------------------------#
|
||||
# 只有得分大于置信度的预测框会被保留下来
|
||||
# ---------------------------------------------------------------------#
|
||||
"confidence": 0.5,
|
||||
# ---------------------------------------------------------------------#
|
||||
# 非极大抑制所用到的nms_iou大小
|
||||
# ---------------------------------------------------------------------#
|
||||
"nms_iou": 0.3,
|
||||
# ---------------------------------------------------------------------#
|
||||
# 该变量用于控制是否使用letterbox_image对输入图像进行不失真的resize，
|
||||
# 在多次测试后，发现关闭letterbox_image直接resize的效果更好
|
||||
# ---------------------------------------------------------------------#
|
||||
"letterbox_image": False,
|
||||
# -------------------------------#
|
||||
# 是否使用Cuda
|
||||
# 没有GPU可以设置成False
|
||||
# -------------------------------#
|
||||
"cuda": True
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_defaults(cls, n):
|
||||
if n in cls._defaults:
|
||||
return cls._defaults[n]
|
||||
else:
|
||||
return "Unrecognized attribute name '" + n + "'"
|
||||
|
||||
# ---------------------------------------------------#
|
||||
# 初始化YOLO
|
||||
# ---------------------------------------------------#
|
||||
def __init__(self, **kwargs):
|
||||
self.__dict__.update(self._defaults) # 用类的_defaults变量更新当前对象的属性字典
|
||||
for name, value in kwargs.items():
|
||||
setattr(self, name, value)
|
||||
|
||||
# ---------------------------------------------------#
|
||||
# 获得种类和先验框的数量
|
||||
# ---------------------------------------------------#
|
||||
self.class_names, self.num_classes = get_classes(self.classes_path)
|
||||
self.anchors, self.num_anchors = get_anchors(self.anchors_path)
|
||||
self.bbox_util = DecodeBox(self.anchors, self.num_classes, (self.input_shape[0], self.input_shape[1]),
|
||||
self.anchors_mask)
|
||||
|
||||
# ---------------------------------------------------#
|
||||
# 画框设置不同的颜色
|
||||
# ---------------------------------------------------#
|
||||
hsv_tuples = [(x / self.num_classes, 1., 1.) for x in range(self.num_classes)]
|
||||
self.colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
|
||||
self.colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), self.colors))
|
||||
self.generate()
|
||||
|
||||
show_config(**self._defaults)
|
||||
|
||||
# ---------------------------------------------------#
|
||||
# 生成模型
|
||||
# ---------------------------------------------------#
|
||||
def generate(self, onnx=False):
|
||||
# ---------------------------------------------------#
|
||||
# 建立yolov3模型，载入yolov3模型的权重
|
||||
# ---------------------------------------------------#
|
||||
self.net = YoloBody(self.anchors_mask, self.num_classes)
|
||||
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
||||
self.net.load_state_dict(torch.load(self.model_path, map_location=device))
|
||||
self.net = self.net.eval()
|
||||
print('{} model, anchors, and classes loaded.'.format(self.model_path))
|
||||
# if not onnx:
|
||||
# if self.cuda:
|
||||
# self.net = nn.DataParallel(self.net)
|
||||
# self.net = self.net.cuda()
|
||||
|
||||
if not onnx:
|
||||
if self.cuda:
|
||||
self.net = self.net.cuda()
|
||||
|
||||
# ---------------------------------------------------#
|
||||
# 检测图片
|
||||
# ---------------------------------------------------#
|
||||
def detect_image(self, image, crop=False, count=False):
|
||||
image_shape = np.array(np.shape(image)[0:2]) # np.shape(image) 的形状 h,w,c
|
||||
# ---------------------------------------------------------#
|
||||
# 在这里将图像转换成RGB图像，防止灰度图在预测时报错。
|
||||
# 代码仅仅支持RGB图像的预测，所有其它类型的图像都会转化成RGB
|
||||
# ---------------------------------------------------------#
|
||||
image = cvtColor(image)
|
||||
# ---------------------------------------------------------#
|
||||
# 给图像增加灰条，实现不失真的resize
|
||||
# 也可以直接resize进行识别
|
||||
# ---------------------------------------------------------#
|
||||
image_data = resize_image(image, (self.input_shape[1], self.input_shape[0]), self.letterbox_image)
|
||||
# ---------------------------------------------------------#
|
||||
# 添加上batch_size维度
|
||||
# ---------------------------------------------------------#
|
||||
image_data = np.expand_dims(np.transpose(preprocess_input(np.array(image_data, dtype='float32')), (2, 0, 1)), 0)
|
||||
# image_data 变换后的维度是 1, 3, 416, 416
|
||||
with torch.no_grad():
|
||||
images = torch.from_numpy(image_data)
|
||||
if self.cuda:
|
||||
images = images.cuda()
|
||||
# ---------------------------------------------------------#
|
||||
# 将图像输入网络当中进行预测！
|
||||
# ---------------------------------------------------------#
|
||||
outputs = self.net(images)
|
||||
outputs = self.bbox_util.decode_box(outputs)
|
||||
# ---------------------------------------------------------#
|
||||
# 将预测框进行堆叠，然后进行非极大抑制
|
||||
# ---------------------------------------------------------#
|
||||
results = self.bbox_util.non_max_suppression(torch.cat(outputs, 1), self.num_classes, self.input_shape,
|
||||
image_shape, self.letterbox_image, conf_thres=self.confidence,
|
||||
nms_thres=self.nms_iou)
|
||||
|
||||
if results[0] is None:
|
||||
return image
|
||||
|
||||
top_label = np.array(results[0][:, 6], dtype='int32')
|
||||
top_conf = results[0][:, 4] * results[0][:, 5]
|
||||
top_boxes = results[0][:, :4]
|
||||
# ---------------------------------------------------------#
|
||||
# 设置字体与边框厚度
|
||||
# ---------------------------------------------------------#
|
||||
font = ImageFont.truetype(font='model_data/simhei.ttf',
|
||||
size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32'))
|
||||
thickness = int(max((image.size[0] + image.size[1]) // np.mean(self.input_shape), 1))
|
||||
# ---------------------------------------------------------#
|
||||
# 计数
|
||||
# ---------------------------------------------------------#
|
||||
if count:
|
||||
print("top_label:", top_label)
|
||||
classes_nums = np.zeros([self.num_classes])
|
||||
for i in range(self.num_classes):
|
||||
num = np.sum(top_label == i)
|
||||
if num > 0:
|
||||
print(self.class_names[i], " : ", num)
|
||||
classes_nums[i] = num
|
||||
print("classes_nums:", classes_nums)
|
||||
# ---------------------------------------------------------#
|
||||
# 是否进行目标的裁剪
|
||||
# ---------------------------------------------------------#
|
||||
if crop:
|
||||
for i, c in list(enumerate(top_label)):
|
||||
top, left, bottom, right = top_boxes[i]
|
||||
top = max(0, np.floor(top).astype('int32'))
|
||||
left = max(0, np.floor(left).astype('int32'))
|
||||
bottom = min(image.size[1], np.floor(bottom).astype('int32'))
|
||||
right = min(image.size[0], np.floor(right).astype('int32'))
|
||||
|
||||
dir_save_path = "img_crop"
|
||||
if not os.path.exists(dir_save_path):
|
||||
os.makedirs(dir_save_path)
|
||||
crop_image = image.crop([left, top, right, bottom])
|
||||
crop_image.save(os.path.join(dir_save_path, "crop_" + str(i) + ".png"), quality=95, subsampling=0)
|
||||
print("save crop_" + str(i) + ".png to " + dir_save_path)
|
||||
# ---------------------------------------------------------#
|
||||
# 图像绘制
|
||||
# ---------------------------------------------------------#
|
||||
for i, c in list(enumerate(top_label)):
|
||||
predicted_class = self.class_names[int(c)]
|
||||
box = top_boxes[i]
|
||||
score = top_conf[i]
|
||||
|
||||
top, left, bottom, right = box
|
||||
|
||||
top = max(0, np.floor(top).astype('int32'))
|
||||
left = max(0, np.floor(left).astype('int32'))
|
||||
bottom = min(image.size[1], np.floor(bottom).astype('int32'))
|
||||
right = min(image.size[0], np.floor(right).astype('int32'))
|
||||
|
||||
label = '{} {:.2f}'.format(predicted_class, score)
|
||||
draw = ImageDraw.Draw(image)
|
||||
label_size = draw.textsize(label, font)
|
||||
label = label.encode('utf-8')
|
||||
# print(label, top, left, bottom, right)
|
||||
|
||||
if top - label_size[1] >= 0: # 框到顶的距离大于 label_size，就是可以在顶部放标签
|
||||
text_origin = np.array([left, top - label_size[1]])
|
||||
else: # 否则放在框内部
|
||||
text_origin = np.array([left, top + 1])
|
||||
|
||||
for i in range(thickness): # 画粗细的实现？是画6次？
|
||||
draw.rectangle([left + i, top + i, right - i, bottom - i], outline=self.colors[c])
|
||||
draw.rectangle([tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c])
|
||||
draw.text(text_origin, str(label, 'UTF-8'), fill=(0, 0, 0), font=font)
|
||||
del draw
|
||||
|
||||
return image
|
||||
|
||||
def get_FPS(self, image, test_interval):
|
||||
image_shape = np.array(np.shape(image)[0:2])
|
||||
# ---------------------------------------------------------#
|
||||
# 在这里将图像转换成RGB图像，防止灰度图在预测时报错。
|
||||
# 代码仅仅支持RGB图像的预测，所有其它类型的图像都会转化成RGB
|
||||
# ---------------------------------------------------------#
|
||||
image = cvtColor(image)
|
||||
# ---------------------------------------------------------#
|
||||
# 给图像增加灰条，实现不失真的resize
|
||||
# 也可以直接resize进行识别
|
||||
# ---------------------------------------------------------#
|
||||
image_data = resize_image(image, (self.input_shape[1], self.input_shape[0]), self.letterbox_image)
|
||||
# ---------------------------------------------------------#
|
||||
# 添加上batch_size维度
|
||||
# ---------------------------------------------------------#
|
||||
image_data = np.expand_dims(np.transpose(preprocess_input(np.array(image_data, dtype='float32')), (2, 0, 1)), 0)
|
||||
|
||||
with torch.no_grad():
|
||||
images = torch.from_numpy(image_data)
|
||||
if self.cuda:
|
||||
images = images.cuda()
|
||||
# ---------------------------------------------------------#
|
||||
# 将图像输入网络当中进行预测！
|
||||
# ---------------------------------------------------------#
|
||||
outputs = self.net(images)
|
||||
outputs = self.bbox_util.decode_box(outputs)
|
||||
# ---------------------------------------------------------#
|
||||
# 将预测框进行堆叠，然后进行非极大抑制
|
||||
# ---------------------------------------------------------#
|
||||
results = self.bbox_util.non_max_suppression(torch.cat(outputs, 1), self.num_classes, self.input_shape,
|
||||
image_shape, self.letterbox_image, conf_thres=self.confidence,
|
||||
nms_thres=self.nms_iou)
|
||||
|
||||
t1 = time.time()
|
||||
for _ in range(test_interval):
|
||||
with torch.no_grad():
|
||||
# ---------------------------------------------------------#
|
||||
# 将图像输入网络当中进行预测！
|
||||
# ---------------------------------------------------------#
|
||||
outputs = self.net(images)
|
||||
outputs = self.bbox_util.decode_box(outputs)
|
||||
# ---------------------------------------------------------#
|
||||
# 将预测框进行堆叠，然后进行非极大抑制
|
||||
# ---------------------------------------------------------#
|
||||
results = self.bbox_util.non_max_suppression(torch.cat(outputs, 1), self.num_classes, self.input_shape,
|
||||
image_shape, self.letterbox_image,
|
||||
conf_thres=self.confidence, nms_thres=self.nms_iou)
|
||||
|
||||
t2 = time.time()
|
||||
tact_time = (t2 - t1) / test_interval
|
||||
return tact_time
|
||||
|
||||
def detect_heatmap(self, image, heatmap_save_path):
|
||||
import cv2
|
||||
import matplotlib.pyplot as plt
|
||||
def sigmoid(x):
|
||||
y = 1.0 / (1.0 + np.exp(-x))
|
||||
return y
|
||||
|
||||
# ---------------------------------------------------------#
|
||||
# 在这里将图像转换成RGB图像，防止灰度图在预测时报错。
|
||||
# 代码仅仅支持RGB图像的预测，所有其它类型的图像都会转化成RGB
|
||||
# ---------------------------------------------------------#
|
||||
image = cvtColor(image)
|
||||
# ---------------------------------------------------------#
|
||||
# 给图像增加灰条，实现不失真的resize
|
||||
# 也可以直接resize进行识别
|
||||
# ---------------------------------------------------------#
|
||||
image_data = resize_image(image, (self.input_shape[1], self.input_shape[0]), self.letterbox_image)
|
||||
# ---------------------------------------------------------#
|
||||
# 添加上batch_size维度
|
||||
# ---------------------------------------------------------#
|
||||
image_data = np.expand_dims(np.transpose(preprocess_input(np.array(image_data, dtype='float32')), (2, 0, 1)), 0)
|
||||
|
||||
with torch.no_grad():
|
||||
images = torch.from_numpy(image_data)
|
||||
if self.cuda:
|
||||
images = images.cuda()
|
||||
# ---------------------------------------------------------#
|
||||
# 将图像输入网络当中进行预测！
|
||||
# ---------------------------------------------------------#
|
||||
outputs = self.net(images)
|
||||
|
||||
plt.imshow(image, alpha=1)
|
||||
plt.axis('off')
|
||||
mask = np.zeros((image.size[1], image.size[0]))
|
||||
for sub_output in outputs:
|
||||
sub_output = sub_output.cpu().numpy()
|
||||
b, c, h, w = np.shape(sub_output)
|
||||
sub_output = np.transpose(np.reshape(sub_output, [b, 3, -1, h, w]), [0, 3, 4, 1, 2])[0]
|
||||
score = np.max(sigmoid(sub_output[..., 4]), -1)
|
||||
score = cv2.resize(score, (image.size[0], image.size[1]))
|
||||
normed_score = (score * 255).astype('uint8')
|
||||
mask = np.maximum(mask, normed_score)
|
||||
|
||||
plt.imshow(mask, alpha=0.5, interpolation='nearest', cmap="jet")
|
||||
|
||||
plt.axis('off')
|
||||
plt.subplots_adjust(top=1, bottom=0, right=1, left=0, hspace=0, wspace=0)
|
||||
plt.margins(0, 0)
|
||||
plt.savefig(heatmap_save_path, dpi=200, bbox_inches='tight', pad_inches=-0.1)
|
||||
print("Save to the " + heatmap_save_path)
|
||||
plt.show()
|
||||
|
||||
def convert_to_onnx(self, simplify, model_path):
|
||||
import onnx
|
||||
self.generate(onnx=True)
|
||||
|
||||
im = torch.zeros(1, 3, *self.input_shape).to('cpu') # image size(1, 3, 512, 512) BCHW
|
||||
input_layer_names = ["images"]
|
||||
output_layer_names = ["output"]
|
||||
|
||||
# Export the model
|
||||
print(f'Starting export with onnx {onnx.__version__}.')
|
||||
torch.onnx.export(self.net,
|
||||
im,
|
||||
f=model_path,
|
||||
verbose=False,
|
||||
opset_version=12,
|
||||
training=torch.onnx.TrainingMode.EVAL,
|
||||
do_constant_folding=True,
|
||||
input_names=input_layer_names,
|
||||
output_names=output_layer_names,
|
||||
dynamic_axes=None)
|
||||
|
||||
# Checks
|
||||
model_onnx = onnx.load(model_path) # load onnx model
|
||||
onnx.checker.check_model(model_onnx) # check onnx model
|
||||
|
||||
# Simplify onnx
|
||||
if simplify:
|
||||
import onnxsim
|
||||
print(f'Simplifying with onnx-simplifier {onnxsim.__version__}.')
|
||||
model_onnx, check = onnxsim.simplify(
|
||||
model_onnx,
|
||||
dynamic_input_shape=False,
|
||||
input_shapes=None)
|
||||
assert check, 'assert check failed'
|
||||
onnx.save(model_onnx, model_path)
|
||||
|
||||
print('Onnx model save as {}'.format(model_path))
|
||||
|
||||
def get_map_txt(self, image_id, image, class_names, map_out_path):
|
||||
f = open(os.path.join(map_out_path, "detection-results/" + image_id + ".txt"), "w")
|
||||
image_shape = np.array(np.shape(image)[0:2])
|
||||
# ---------------------------------------------------------#
|
||||
# 在这里将图像转换成RGB图像，防止灰度图在预测时报错。
|
||||
# 代码仅仅支持RGB图像的预测，所有其它类型的图像都会转化成RGB
|
||||
# ---------------------------------------------------------#
|
||||
image = cvtColor(image)
|
||||
# ---------------------------------------------------------#
|
||||
# 给图像增加灰条，实现不失真的resize
|
||||
# 也可以直接resize进行识别
|
||||
# ---------------------------------------------------------#
|
||||
image_data = resize_image(image, (self.input_shape[1], self.input_shape[0]), self.letterbox_image)
|
||||
# ---------------------------------------------------------#
|
||||
# 添加上batch_size维度
|
||||
# ---------------------------------------------------------#
|
||||
image_data = np.expand_dims(np.transpose(preprocess_input(np.array(image_data, dtype='float32')), (2, 0, 1)), 0)
|
||||
|
||||
with torch.no_grad():
|
||||
images = torch.from_numpy(image_data)
|
||||
if self.cuda:
|
||||
images = images.cuda()
|
||||
# ---------------------------------------------------------#
|
||||
# 将图像输入网络当中进行预测！
|
||||
# ---------------------------------------------------------#
|
||||
outputs = self.net(images)
|
||||
outputs = self.bbox_util.decode_box(outputs)
|
||||
# ---------------------------------------------------------#
|
||||
# 将预测框进行堆叠，然后进行非极大抑制
|
||||
# ---------------------------------------------------------#
|
||||
results = self.bbox_util.non_max_suppression(torch.cat(outputs, 1), self.num_classes, self.input_shape,
|
||||
image_shape, self.letterbox_image, conf_thres=self.confidence,
|
||||
nms_thres=self.nms_iou)
|
||||
|
||||
if results[0] is None:
|
||||
return
|
||||
|
||||
top_label = np.array(results[0][:, 6], dtype='int32')
|
||||
top_conf = results[0][:, 4] * results[0][:, 5]
|
||||
top_boxes = results[0][:, :4]
|
||||
|
||||
for i, c in list(enumerate(top_label)):
|
||||
predicted_class = self.class_names[int(c)]
|
||||
box = top_boxes[i]
|
||||
score = str(top_conf[i])
|
||||
|
||||
top, left, bottom, right = box
|
||||
if predicted_class not in class_names:
|
||||
continue
|
||||
|
||||
f.write("%s %s %s %s %s %s\n" % (
|
||||
predicted_class, score[:6], str(int(left)), str(int(top)), str(int(right)), str(int(bottom))))
|
||||
|
||||
f.close()
|
||||
return
|
||||
Loading…
Reference in New Issue