1703 lines
67 KiB
Python
1703 lines
67 KiB
Python
|
# -----------------------------------------------------------
|
||
|
|
# "BCAN++: Cross-modal Retrieval With Bidirectional Correct Attention Network"
|
||
|
|
# Yang Liu, Hong Liu, Huaqiu Wang, Fanyang Meng, Mengyuan Liu*
|
||
|
|
#
|
||
|
|
# ---------------------------------------------------------------
|
||
|
|
"""Evaluation"""
|
||
|
|
|
||
|
|
from __future__ import print_function
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
import numpy as np
|
||
|
|
import torch
|
||
|
|
from collections import OrderedDict
|
||
|
|
import time
|
||
|
|
from torch.autograd import Variable
|
||
|
|
from transformers import BertTokenizer
|
||
|
|
|
||
|
|
from data import get_test_loader
|
||
|
|
from data2 import get_test_loader2
|
||
|
|
from lib.datasets import image_caption
|
||
|
|
from lib.evaluation import eval_cxc
|
||
|
|
from lib.vse import VSEModel
|
||
|
|
from model import SCAN
|
||
|
|
from model2 import SCAN2
|
||
|
|
from model3 import SCAN3
|
||
|
|
from vocab import deserialize_vocab
|
||
|
|
|
||
|
|
|
||
|
|
class AverageMeter(object):
|
||
|
|
"""Computes and stores the average and current value"""
|
||
|
|
def __init__(self):
|
||
|
|
self.reset()
|
||
|
|
|
||
|
|
def reset(self):
|
||
|
|
self.val = 0
|
||
|
|
self.avg = 0
|
||
|
|
self.sum = 0
|
||
|
|
self.count = 0
|
||
|
|
|
||
|
|
def update(self, val, n=0):
|
||
|
|
self.val = val
|
||
|
|
self.sum += val * n
|
||
|
|
self.count += n
|
||
|
|
self.avg = self.sum / (.0001 + self.count)
|
||
|
|
|
||
|
|
def __str__(self):
|
||
|
|
"""String representation for logging
|
||
|
|
"""
|
||
|
|
# for values that should be recorded exactly e.g. iteration number
|
||
|
|
if self.count == 0:
|
||
|
|
return str(self.val)
|
||
|
|
# for stats
|
||
|
|
return '%.4f (%.4f)' % (self.val, self.avg)
|
||
|
|
|
||
|
|
|
||
|
|
class LogCollector(object):
|
||
|
|
"""A collection of logging objects that can change from train to val"""
|
||
|
|
def __init__(self):
|
||
|
|
# to keep the order of logged variables deterministic
|
||
|
|
self.meters = OrderedDict()
|
||
|
|
|
||
|
|
def update(self, k, v, n=0):
|
||
|
|
# create a new meter if previously not recorded
|
||
|
|
if k not in self.meters:
|
||
|
|
self.meters[k] = AverageMeter()
|
||
|
|
self.meters[k].update(v, n)
|
||
|
|
|
||
|
|
def __str__(self):
|
||
|
|
"""Concatenate the meters in one log line
|
||
|
|
"""
|
||
|
|
s = ''
|
||
|
|
for i, (k, v) in enumerate(self.meters.items()):
|
||
|
|
if i > 0:
|
||
|
|
s += ' '
|
||
|
|
s += k + ' ' + str(v)
|
||
|
|
return s
|
||
|
|
|
||
|
|
def tb_log(self, tb_logger, prefix='', step=None):
|
||
|
|
"""Log using tensorboard
|
||
|
|
"""
|
||
|
|
for k, v in self.meters.items():
|
||
|
|
tb_logger.log_value(prefix + k, v.val, step=step)
|
||
|
|
|
||
|
|
def encode_data2(model, data_loader, log_step=100, logging=print):
|
||
|
|
"""Encode all images and captions loadable by `data_loader`
|
||
|
|
"""
|
||
|
|
batch_time = AverageMeter()
|
||
|
|
val_logger = LogCollector()
|
||
|
|
|
||
|
|
# switch to evaluate mode
|
||
|
|
model.eval()
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
|
||
|
|
# np array to keep all the embeddings
|
||
|
|
img_embs = None
|
||
|
|
cap_embs = None
|
||
|
|
cap_lens = None
|
||
|
|
|
||
|
|
max_n_word = 0
|
||
|
|
for i, (images, captions, lengths, ids) in enumerate(data_loader):
|
||
|
|
max_n_word = max(max_n_word, max(lengths))
|
||
|
|
# lengths = lengths.cpu().numpy().tolist()
|
||
|
|
# l = [len(l) for l in lengths]
|
||
|
|
# max_n_word = max(max_n_word, max(l))
|
||
|
|
|
||
|
|
with torch.no_grad():
|
||
|
|
for i, (images, captions, lengths, ids) in enumerate(data_loader):
|
||
|
|
# make sure val logger is used
|
||
|
|
model.logger = val_logger
|
||
|
|
lengths = lengths.cpu().numpy().tolist()
|
||
|
|
images = images.cuda()
|
||
|
|
captions = captions.cuda()
|
||
|
|
# pos = pos.cuda()
|
||
|
|
# compute the embeddings
|
||
|
|
img_emb, img_mean, cap_emb, cap_len, cap_mean = model.module.forward_emb(images, captions, lengths)
|
||
|
|
# img_emb, cap_emb, cap_len = model.forward_emb(images, captions, pos, lengths)
|
||
|
|
# print(img_emb)
|
||
|
|
if img_embs is None:
|
||
|
|
if img_emb.dim() == 3:
|
||
|
|
img_embs = np.zeros((len(data_loader.dataset), img_emb.size(1), img_emb.size(2)))
|
||
|
|
else:
|
||
|
|
img_embs = np.zeros((len(data_loader.dataset), img_emb.size(1)))
|
||
|
|
cap_embs = np.zeros((len(data_loader.dataset), max_n_word, cap_emb.size(2)))
|
||
|
|
img_means = np.zeros((len(data_loader.dataset), img_mean.size(1)))
|
||
|
|
# tags = np.zeros((len(data_loader.dataset), max_n_word))
|
||
|
|
cap_lens = [0] * len(data_loader.dataset)
|
||
|
|
cap_means = np.zeros((len(data_loader.dataset), cap_mean.size(1)))
|
||
|
|
# cache embeddings
|
||
|
|
# print(img_embs.shape,type(ids))
|
||
|
|
# print(img_emb.shape)
|
||
|
|
img_embs[ids] = img_emb.data.cpu().numpy().copy()
|
||
|
|
img_means[ids] = img_mean.data.cpu().numpy().copy()
|
||
|
|
cap_means[ids] = cap_mean.data.cpu().numpy().copy()
|
||
|
|
cap_embs[ids, :cap_emb.size(1), :] = cap_emb.data.cpu().numpy().copy()
|
||
|
|
for j, nid in enumerate(ids):
|
||
|
|
cap_lens[nid] = cap_len[j]
|
||
|
|
|
||
|
|
# measure elapsed time
|
||
|
|
batch_time.update(time.time() - end)
|
||
|
|
end = time.time()
|
||
|
|
|
||
|
|
if i % log_step == 0:
|
||
|
|
logging('Test: [{0}/{1}]\t'
|
||
|
|
'{e_log}\t'
|
||
|
|
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
|
||
|
|
.format(
|
||
|
|
i, len(data_loader), batch_time=batch_time,
|
||
|
|
e_log=str(model.logger)))
|
||
|
|
del images, captions
|
||
|
|
return img_embs, img_means, cap_embs, cap_lens, cap_means,
|
||
|
|
|
||
|
|
def encode_data(model, data_loader, log_step=100, logging=print):
|
||
|
|
"""Encode all images and captions loadable by `data_loader`
|
||
|
|
"""
|
||
|
|
batch_time = AverageMeter()
|
||
|
|
val_logger = LogCollector()
|
||
|
|
|
||
|
|
# switch to evaluate mode
|
||
|
|
model.eval()
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
|
||
|
|
# np array to keep all the embeddings
|
||
|
|
img_embs = None
|
||
|
|
cap_embs = None
|
||
|
|
cap_lens = None
|
||
|
|
|
||
|
|
max_n_word = 0
|
||
|
|
for i, (images, img_lengths, captions, lengths, ids) in enumerate(data_loader):
|
||
|
|
max_n_word = max(max_n_word, max(lengths))
|
||
|
|
# lengths = lengths.cpu().numpy().tolist()
|
||
|
|
# l = [len(l) for l in lengths]
|
||
|
|
# max_n_word = max(max_n_word, max(l))
|
||
|
|
|
||
|
|
with torch.no_grad():
|
||
|
|
for i, (images, img_lengths, captions, lengths, ids) in enumerate(data_loader):
|
||
|
|
# make sure val logger is used
|
||
|
|
model.logger = val_logger
|
||
|
|
lengths = lengths.cpu().numpy().tolist()
|
||
|
|
images = images.cuda()
|
||
|
|
img_lengths = img_lengths.cuda()
|
||
|
|
captions = captions.cuda()
|
||
|
|
# pos = pos.cuda()
|
||
|
|
# compute the embeddings
|
||
|
|
img_emb, img_mean, cap_emb, cap_len, cap_mean = model.module.forward_emb(images, img_lengths, captions, lengths)
|
||
|
|
# img_emb, cap_emb, cap_len = model.forward_emb(images, captions, pos, lengths)
|
||
|
|
# print(img_emb)
|
||
|
|
if img_embs is None:
|
||
|
|
if img_emb.dim() == 3:
|
||
|
|
img_embs = np.zeros((len(data_loader.dataset), img_emb.size(1), img_emb.size(2)))
|
||
|
|
else:
|
||
|
|
img_embs = np.zeros((len(data_loader.dataset), img_emb.size(1)))
|
||
|
|
cap_embs = np.zeros((len(data_loader.dataset), max_n_word, cap_emb.size(2)))
|
||
|
|
img_means = np.zeros((len(data_loader.dataset), img_mean.size(1)))
|
||
|
|
# tags = np.zeros((len(data_loader.dataset), max_n_word))
|
||
|
|
cap_lens = [0] * len(data_loader.dataset)
|
||
|
|
cap_means = np.zeros((len(data_loader.dataset), cap_mean.size(1)))
|
||
|
|
# cache embeddings
|
||
|
|
# print(img_embs.shape,type(ids))
|
||
|
|
# print(img_emb.shape)
|
||
|
|
img_embs[ids] = img_emb.data.cpu().numpy().copy()
|
||
|
|
img_means[ids] = img_mean.data.cpu().numpy().copy()
|
||
|
|
cap_means[ids] = cap_mean.data.cpu().numpy().copy()
|
||
|
|
cap_embs[ids, :cap_emb.size(1), :] = cap_emb.data.cpu().numpy().copy()
|
||
|
|
for j, nid in enumerate(ids):
|
||
|
|
cap_lens[nid] = cap_len[j]
|
||
|
|
|
||
|
|
# measure elapsed time
|
||
|
|
batch_time.update(time.time() - end)
|
||
|
|
end = time.time()
|
||
|
|
|
||
|
|
if i % log_step == 0:
|
||
|
|
logging('Test: [{0}/{1}]\t'
|
||
|
|
'{e_log}\t'
|
||
|
|
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
|
||
|
|
.format(
|
||
|
|
i, len(data_loader), batch_time=batch_time,
|
||
|
|
e_log=str(model.logger)))
|
||
|
|
del images, captions
|
||
|
|
return img_embs, img_means, cap_embs, cap_lens, cap_means,
|
||
|
|
|
||
|
|
def encode_data_vse(model, data_loader, log_step=10, logging=print, backbone=False):
|
||
|
|
"""Encode all images and captions loadable by `data_loader`
|
||
|
|
"""
|
||
|
|
batch_time = AverageMeter()
|
||
|
|
val_logger = LogCollector()
|
||
|
|
|
||
|
|
# switch to evaluate mode
|
||
|
|
model.val_start()
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
|
||
|
|
# np array to keep all the embeddings
|
||
|
|
img_embs = None
|
||
|
|
cap_embs = None
|
||
|
|
|
||
|
|
for i, data_i in enumerate(data_loader):
|
||
|
|
# make sure val logger is used
|
||
|
|
if not backbone:
|
||
|
|
images, image_lengths, captions, lengths, ids = data_i
|
||
|
|
else:
|
||
|
|
images, captions, lengths, ids = data_i
|
||
|
|
model.logger = val_logger
|
||
|
|
|
||
|
|
# compute the embeddings
|
||
|
|
if not backbone:
|
||
|
|
img_emb, cap_emb = model.forward_emb(images, captions, lengths, image_lengths=image_lengths)
|
||
|
|
else:
|
||
|
|
img_emb, cap_emb = model.forward_emb(images, captions, lengths)
|
||
|
|
|
||
|
|
if img_embs is None:
|
||
|
|
if img_emb.dim() == 3:
|
||
|
|
img_embs = np.zeros((len(data_loader.dataset), img_emb.size(1), img_emb.size(2)))
|
||
|
|
else:
|
||
|
|
img_embs = np.zeros((len(data_loader.dataset), img_emb.size(1)))
|
||
|
|
cap_embs = np.zeros((len(data_loader.dataset), cap_emb.size(1)))
|
||
|
|
cap_lens = [0] * len(data_loader.dataset)
|
||
|
|
# cache embeddings
|
||
|
|
img_embs[ids] = img_emb.data.cpu().numpy().copy()
|
||
|
|
cap_embs[ids, :] = cap_emb.data.cpu().numpy().copy()
|
||
|
|
|
||
|
|
# measure accuracy and record loss
|
||
|
|
model.forward_loss(img_emb, cap_emb)
|
||
|
|
|
||
|
|
# measure elapsed time
|
||
|
|
batch_time.update(time.time() - end)
|
||
|
|
end = time.time()
|
||
|
|
|
||
|
|
if i % log_step == 0:
|
||
|
|
logging('Test: [{0}/{1}]\t'
|
||
|
|
'{e_log}\t'
|
||
|
|
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
|
||
|
|
.format(
|
||
|
|
i, len(data_loader.dataset) // data_loader.batch_size + 1, batch_time=batch_time,
|
||
|
|
e_log=str(model.logger)))
|
||
|
|
|
||
|
|
del images, captions
|
||
|
|
return img_embs, cap_embs
|
||
|
|
|
||
|
|
def compute_sim(images, captions):
|
||
|
|
similarities = np.matmul(images, np.matrix.transpose(captions))
|
||
|
|
return similarities
|
||
|
|
|
||
|
|
def evalrank_vse(model_path, data_path=None, split='dev', fold5=False):
|
||
|
|
"""
|
||
|
|
Evaluate a trained model on either dev or test. If `fold5=True`, 5 fold
|
||
|
|
cross-validation is done (only for MSCOCO). Otherwise, the full data is
|
||
|
|
used for evaluation.
|
||
|
|
"""
|
||
|
|
# load model and options
|
||
|
|
checkpoint = torch.load(model_path)
|
||
|
|
opt = checkpoint['opt']
|
||
|
|
print(opt)
|
||
|
|
if data_path is not None:
|
||
|
|
opt.data_path = data_path
|
||
|
|
|
||
|
|
|
||
|
|
# load vocabulary used by the model
|
||
|
|
vocab = deserialize_vocab(os.path.join(opt.vocab_path, '%s_vocab.json' % opt.data_name))
|
||
|
|
word2idx = vocab.word2idx
|
||
|
|
opt.vocab_size = len(vocab)
|
||
|
|
|
||
|
|
model = VSEModel(opt)
|
||
|
|
|
||
|
|
# load model state
|
||
|
|
model.load_state_dict(checkpoint['model'])
|
||
|
|
|
||
|
|
|
||
|
|
print('Loading dataset')
|
||
|
|
opt.batch_size = 64
|
||
|
|
data_loader = get_test_loader(split, opt.data_name, vocab,
|
||
|
|
opt.batch_size, 2, opt)
|
||
|
|
|
||
|
|
print('Computing results...')
|
||
|
|
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)
|
||
|
|
logging.info('train')
|
||
|
|
os.makedirs(opt.logger_name, exist_ok=True)
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
model.val_start()
|
||
|
|
with torch.no_grad():
|
||
|
|
# compute the encoding for all the validation images and captions
|
||
|
|
img_embs, cap_embs = encode_data_vse(
|
||
|
|
model, data_loader, opt.log_step, backbone=opt.precomp_enc_type == 'backbone')
|
||
|
|
|
||
|
|
print('Images: %d, Captions: %d' %
|
||
|
|
(img_embs.shape[0] / 5, cap_embs.shape[0]))
|
||
|
|
img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 5)])
|
||
|
|
|
||
|
|
start = time.time()
|
||
|
|
sims = compute_sim(img_embs, cap_embs)
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
logger.info("calculate similarity time: {}".format(end - start))
|
||
|
|
|
||
|
|
# caption retrieval
|
||
|
|
npts = img_embs.shape[0]
|
||
|
|
# (r1, r5, r10, medr, meanr) = i2t(img_embs, cap_embs, cap_lens, sims)
|
||
|
|
(r1, r5, r10, medr, meanr) = i2t_vse(npts, sims)
|
||
|
|
# logging.info("Image to text: %.1f, %.1f, %.1f, %.1f, %.1f" %
|
||
|
|
# (r1, r5, r10, medr, meanr))
|
||
|
|
# image retrieval
|
||
|
|
# (r1i, r5i, r10i, medri, meanr) = t2i(img_embs, cap_embs, cap_lens, sims)
|
||
|
|
(r1i, r5i, r10i, medri, meanr) = t2i_vse(npts, sims)
|
||
|
|
# logging.info("Text to image: %.1f, %.1f, %.1f, %.1f, %.1f" %
|
||
|
|
# (r1i, r5i, r10i, medri, meanr))
|
||
|
|
# sum of recalls to be used for early stopping
|
||
|
|
currscore = r1 + r5 + r10 + r1i + r5i + r10i
|
||
|
|
#logger.info('Current rsum is {}'.format(currscore))
|
||
|
|
print("rsum: %.1f" % currscore)
|
||
|
|
ar = (r1 + r5 + r10) / 3
|
||
|
|
ari = (r1i + r5i + r10i) / 3
|
||
|
|
print("Average i2t Recall: %.1f" % ar)
|
||
|
|
print("Image to text: %.1f, %.1f, %.1f, %.1f, %.1f" %
|
||
|
|
(r1, r5, r10, medr, meanr))
|
||
|
|
print("Average t2i Recall: %.1f" % ari)
|
||
|
|
print("Text to image: %.1f, %.1f, %.1f, %.1f, %.1f" %
|
||
|
|
(r1i, r5i, r10i, medri, meanr))
|
||
|
|
|
||
|
|
|
||
|
|
def evalrank_f_c(data_path,dataset):
|
||
|
|
parser = argparse.ArgumentParser()
|
||
|
|
#parser.add_argument('--dataset', default='f30k',help='coco or f30k')
|
||
|
|
#parser.add_argument('--data_path', default='./data/f30k')
|
||
|
|
#parser.add_argument('--data_path', default='../pycharmProject/data/f30k_precomp')
|
||
|
|
#parser.add_argument('--data_path', default='../SCAN-master/data/coco_precomp')
|
||
|
|
#parser.add_argument('--data_path', default='../SCAN-master/data/f30k_precomp')
|
||
|
|
parser.add_argument('--save_results', action='store_true')
|
||
|
|
parser.add_argument('--evaluate_cxc', action='store_true')
|
||
|
|
opt = parser.parse_args()
|
||
|
|
opt.dataset = dataset
|
||
|
|
opt.data_path=data_path
|
||
|
|
|
||
|
|
if opt.dataset == 'coco':
|
||
|
|
weights_bases = [
|
||
|
|
'./runs/coco_model',
|
||
|
|
]
|
||
|
|
elif opt.dataset == 'f30k':
|
||
|
|
weights_bases = [
|
||
|
|
'./runs/f30k_model',
|
||
|
|
]
|
||
|
|
else:
|
||
|
|
raise ValueError('Invalid dataset argument {}'.format(opt.dataset))
|
||
|
|
|
||
|
|
for base in weights_bases:
|
||
|
|
#model_path = os.path.join(base, 'model_best.pth')
|
||
|
|
model_path = os.path.join(base, 'checkpoint.pth')
|
||
|
|
if opt.save_results: # Save the final results for computing ensemble results
|
||
|
|
save_path = os.path.join(base, 'results_{}.npy'.format(opt.dataset))
|
||
|
|
else:
|
||
|
|
save_path = None
|
||
|
|
|
||
|
|
if opt.dataset == 'coco':
|
||
|
|
if not opt.evaluate_cxc:
|
||
|
|
# Evaluate COCO 5-fold 1K
|
||
|
|
evalrank_f30k_coco(model_path, data_path=opt.data_path, split='test', fold5=False)
|
||
|
|
# Evaluate COCO 5K
|
||
|
|
#evalrank_f30k_coco(model_path, data_path=opt.data_path, split='testall', fold5=False, save_path=save_path)
|
||
|
|
else:
|
||
|
|
# Evaluate COCO-trained models on CxC
|
||
|
|
evalrank_f30k_coco(model_path, data_path=opt.data_path, split='testall', fold5=True, cxc=True)
|
||
|
|
elif opt.dataset == 'f30k':
|
||
|
|
# Evaluate Flickr30K
|
||
|
|
evalrank_f30k_coco(model_path, data_path=opt.data_path, split='test', fold5=False, save_path=save_path)
|
||
|
|
|
||
|
|
def evalrank_f_c2(data_path,dataset):
|
||
|
|
parser = argparse.ArgumentParser()
|
||
|
|
#parser.add_argument('--dataset', default='f30k',help='coco or f30k')
|
||
|
|
#parser.add_argument('--data_path', default='./data/f30k')
|
||
|
|
#parser.add_argument('--data_path', default='../pycharmProject/data/f30k_precomp')
|
||
|
|
#parser.add_argument('--data_path', default='../SCAN-master/data/coco_precomp')
|
||
|
|
#parser.add_argument('--data_path', default='../SCAN-master/data/f30k_precomp')
|
||
|
|
parser.add_argument('--save_results', action='store_true')
|
||
|
|
parser.add_argument('--evaluate_cxc', action='store_true')
|
||
|
|
opt = parser.parse_args()
|
||
|
|
opt.dataset = dataset
|
||
|
|
opt.data_path=data_path
|
||
|
|
|
||
|
|
if opt.dataset == 'coco':
|
||
|
|
weights_bases = [
|
||
|
|
'./runs/coco_model',
|
||
|
|
]
|
||
|
|
elif opt.dataset == 'f30k':
|
||
|
|
weights_bases = [
|
||
|
|
'./runs/f30k_model',
|
||
|
|
]
|
||
|
|
else:
|
||
|
|
raise ValueError('Invalid dataset argument {}'.format(opt.dataset))
|
||
|
|
|
||
|
|
for base in weights_bases:
|
||
|
|
#model_path = os.path.join(base, 'model_best.pth')
|
||
|
|
model_path = os.path.join(base, 'checkpoint.pth')
|
||
|
|
if opt.save_results: # Save the final results for computing ensemble results
|
||
|
|
save_path = os.path.join(base, 'results_{}.npy'.format(opt.dataset))
|
||
|
|
else:
|
||
|
|
save_path = None
|
||
|
|
|
||
|
|
if opt.dataset == 'coco':
|
||
|
|
if not opt.evaluate_cxc:
|
||
|
|
# Evaluate COCO 5-fold 1K
|
||
|
|
evalrank_f30k_coco2(model_path, data_path=opt.data_path, split='test', fold5=False)
|
||
|
|
# Evaluate COCO 5K
|
||
|
|
#evalrank_f30k_coco(model_path, data_path=opt.data_path, split='testall', fold5=False, save_path=save_path)
|
||
|
|
else:
|
||
|
|
# Evaluate COCO-trained models on CxC
|
||
|
|
evalrank_f30k_coco2(model_path, data_path=opt.data_path, split='testall', fold5=True, cxc=True)
|
||
|
|
elif opt.dataset == 'f30k':
|
||
|
|
# Evaluate Flickr30K
|
||
|
|
evalrank_f30k_coco2(model_path, data_path=opt.data_path, split='test', fold5=False, save_path=save_path)
|
||
|
|
|
||
|
|
def evalrank_f30k_coco2(model_path, data_path=None, split='dev', fold5=False, save_path=None, cxc=False):
|
||
|
|
"""
|
||
|
|
Evaluate a trained model on either dev or test. If `fold5=True`, 5 fold
|
||
|
|
cross-validation is done (only for MSCOCO). Otherwise, the full data is
|
||
|
|
used for evaluation.
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
# load model and options
|
||
|
|
checkpoint = torch.load(model_path)
|
||
|
|
opt = checkpoint['opt']
|
||
|
|
opt.workers = 5
|
||
|
|
|
||
|
|
if not hasattr(opt, 'caption_loss'):
|
||
|
|
opt.caption_loss = False
|
||
|
|
|
||
|
|
# load vocabulary used by the model
|
||
|
|
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
|
||
|
|
vocab = tokenizer.vocab
|
||
|
|
opt.vocab_size = len(vocab)
|
||
|
|
|
||
|
|
opt.backbone_path = '/tmp/data/weights/original_updown_backbone.pth'
|
||
|
|
if data_path is not None:
|
||
|
|
opt.data_path = data_path
|
||
|
|
|
||
|
|
# construct model
|
||
|
|
model = VSEModel(opt)
|
||
|
|
|
||
|
|
model.make_data_parallel()
|
||
|
|
# load model state
|
||
|
|
model.load_state_dict(checkpoint['model'])
|
||
|
|
model.val_start()
|
||
|
|
|
||
|
|
# checkpoint2 = torch.load("./runs/bert_adam_bcan_gpo_vseinfty_bcan/model_best.pth.tar")
|
||
|
|
# # checkpoint2 = torch.load("../pycharmProject/runs/bigru_bcan_adam_mean_base2/model_best.pth.tar")
|
||
|
|
# opt2 = checkpoint2['opt']
|
||
|
|
# print(opt)
|
||
|
|
#
|
||
|
|
# # load vocabulary used by the model
|
||
|
|
# vocab = deserialize_vocab(os.path.join(opt2.vocab_path, '%s_vocab.json' % opt.data_name))
|
||
|
|
# word2idx = vocab.word2idx
|
||
|
|
# opt2.vocab_size = len(vocab)
|
||
|
|
|
||
|
|
# model2 = SCAN(word2idx, opt2)
|
||
|
|
# model2 = torch.nn.DataParallel(model2)
|
||
|
|
# model2.cuda()
|
||
|
|
|
||
|
|
|
||
|
|
# load model state
|
||
|
|
#model2.load_state_dict(checkpoint2['model'])
|
||
|
|
|
||
|
|
# print('Loading dataset')
|
||
|
|
# opt2.batch_size = 64
|
||
|
|
# data_loader2 = get_test_loader(split, opt2.data_name, vocab,
|
||
|
|
# opt2.batch_size, 0, opt2)
|
||
|
|
|
||
|
|
# print('Computing results...')
|
||
|
|
# img_embs2, img_means2, cap_embs2, cap_lens2, cap_means2 = encode_data(model2, data_loader2)
|
||
|
|
|
||
|
|
print('Loading dataset')
|
||
|
|
data_loader = image_caption.get_test_loader(split, opt.data_name, tokenizer,
|
||
|
|
opt.batch_size, opt.workers, opt)
|
||
|
|
|
||
|
|
print('Computing results...')
|
||
|
|
with torch.no_grad():
|
||
|
|
if opt.precomp_enc_type == 'basic':
|
||
|
|
img_embs, cap_embs = encode_data_vse(model, data_loader)
|
||
|
|
else:
|
||
|
|
img_embs, cap_embs = encode_data_vse(model, data_loader, backbone=True)
|
||
|
|
print('Images: %d, Captions: %d' %
|
||
|
|
(img_embs.shape[0] / 5, cap_embs.shape[0]))
|
||
|
|
|
||
|
|
if cxc:
|
||
|
|
eval_cxc(img_embs, cap_embs, data_path)
|
||
|
|
else:
|
||
|
|
if not fold5:
|
||
|
|
# no cross-validation, full evaluation
|
||
|
|
|
||
|
|
img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 5)])
|
||
|
|
start = time.time()
|
||
|
|
sims = compute_sim(img_embs, cap_embs)
|
||
|
|
|
||
|
|
#img_embs2 = np.array([img_embs2[i] for i in range(0, len(img_embs2), 5)])
|
||
|
|
# img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 20)])
|
||
|
|
# print(img_embs[:10])
|
||
|
|
#sims2 = shard_xattn(model2, img_embs2, img_means2, cap_embs2, cap_lens2, cap_means2, opt2, shard_size=500)
|
||
|
|
# sims = sims + sims2
|
||
|
|
# print(sims.shape)
|
||
|
|
npts = img_embs.shape[0]
|
||
|
|
|
||
|
|
if save_path is not None:
|
||
|
|
np.save(save_path, {'npts': npts, 'sims': sims})
|
||
|
|
print('Save the similarity into {}'.format(save_path))
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time: {}".format(end - start))
|
||
|
|
|
||
|
|
r, rt = i2t_vse(npts, sims, return_ranks=True)
|
||
|
|
ri, rti = t2i_vse(npts, sims, return_ranks=True)
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f" % rsum)
|
||
|
|
print("Average i2t Recall: %.1f" % ar)
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" % r)
|
||
|
|
print("Average t2i Recall: %.1f" % ari)
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" % ri)
|
||
|
|
else:
|
||
|
|
# 5fold cross-validation, only for MSCOCO
|
||
|
|
results = []
|
||
|
|
for i in range(5):
|
||
|
|
img_embs_shard = img_embs[i * 5000:(i + 1) * 5000:5]
|
||
|
|
cap_embs_shard = cap_embs[i * 5000:(i + 1) * 5000]
|
||
|
|
start = time.time()
|
||
|
|
sims = compute_sim(img_embs_shard, cap_embs_shard)
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time: {}".format(end - start))
|
||
|
|
|
||
|
|
npts = img_embs_shard.shape[0]
|
||
|
|
r, rt0 = i2t_vse(npts, sims, return_ranks=True)
|
||
|
|
print("Image to text: %.1f, %.1f, %.1f, %.1f, %.1f" % r)
|
||
|
|
ri, rti0 = t2i_vse(npts, sims, return_ranks=True)
|
||
|
|
print("Text to image: %.1f, %.1f, %.1f, %.1f, %.1f" % ri)
|
||
|
|
|
||
|
|
if i == 0:
|
||
|
|
rt, rti = rt0, rti0
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f ar: %.1f ari: %.1f" % (rsum, ar, ari))
|
||
|
|
results += [list(r) + list(ri) + [ar, ari, rsum]]
|
||
|
|
|
||
|
|
print("-----------------------------------")
|
||
|
|
print("Mean metrics: ")
|
||
|
|
mean_metrics = tuple(np.array(results).mean(axis=0).flatten())
|
||
|
|
print("rsum: %.1f" % (mean_metrics[12]))
|
||
|
|
print("Average i2t Recall: %.1f" % mean_metrics[10])
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[:5])
|
||
|
|
print("Average t2i Recall: %.1f" % mean_metrics[11])
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[5:10])
|
||
|
|
|
||
|
|
|
||
|
|
def evalrank_f30k_coco(model_path, data_path=None, split='dev', fold5=False, save_path=None, cxc=False):
|
||
|
|
"""
|
||
|
|
Evaluate a trained model on either dev or test. If `fold5=True`, 5 fold
|
||
|
|
cross-validation is done (only for MSCOCO). Otherwise, the full data is
|
||
|
|
used for evaluation.
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
# load model and options
|
||
|
|
checkpoint = torch.load(model_path)
|
||
|
|
opt = checkpoint['opt']
|
||
|
|
opt.workers = 5
|
||
|
|
|
||
|
|
if not hasattr(opt, 'caption_loss'):
|
||
|
|
opt.caption_loss = False
|
||
|
|
|
||
|
|
# load vocabulary used by the model
|
||
|
|
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
|
||
|
|
vocab = tokenizer.vocab
|
||
|
|
opt.vocab_size = len(vocab)
|
||
|
|
|
||
|
|
opt.backbone_path = '/tmp/data/weights/original_updown_backbone.pth'
|
||
|
|
if data_path is not None:
|
||
|
|
opt.data_path = data_path
|
||
|
|
|
||
|
|
print(opt)
|
||
|
|
|
||
|
|
# construct model
|
||
|
|
model = VSEModel(opt)
|
||
|
|
|
||
|
|
model.make_data_parallel()
|
||
|
|
# load model state
|
||
|
|
model.load_state_dict(checkpoint['model'])
|
||
|
|
model.val_start()
|
||
|
|
|
||
|
|
# checkpoint2 = torch.load("./runs/bert_adam_bcan_gpo_vseinfty_bcan/model_best.pth.tar")
|
||
|
|
# # checkpoint2 = torch.load("../pycharmProject/runs/bigru_bcan_adam_mean_base2/model_best.pth.tar")
|
||
|
|
# opt2 = checkpoint2['opt']
|
||
|
|
# print(opt)
|
||
|
|
#
|
||
|
|
# # load vocabulary used by the model
|
||
|
|
# vocab = deserialize_vocab(os.path.join(opt2.vocab_path, '%s_vocab.json' % opt.data_name))
|
||
|
|
# word2idx = vocab.word2idx
|
||
|
|
# opt2.vocab_size = len(vocab)
|
||
|
|
|
||
|
|
# model2 = SCAN(word2idx, opt2)
|
||
|
|
# model2 = torch.nn.DataParallel(model2)
|
||
|
|
# model2.cuda()
|
||
|
|
|
||
|
|
|
||
|
|
# load model state
|
||
|
|
#model2.load_state_dict(checkpoint2['model'])
|
||
|
|
|
||
|
|
# print('Loading dataset')
|
||
|
|
# opt2.batch_size = 64
|
||
|
|
# data_loader2 = get_test_loader(split, opt2.data_name, vocab,
|
||
|
|
# opt2.batch_size, 0, opt2)
|
||
|
|
|
||
|
|
# print('Computing results...')
|
||
|
|
# img_embs2, img_means2, cap_embs2, cap_lens2, cap_means2 = encode_data(model2, data_loader2)
|
||
|
|
|
||
|
|
print('Loading dataset')
|
||
|
|
data_loader = image_caption.get_test_loader(split, opt.data_name, tokenizer,
|
||
|
|
opt.batch_size, opt.workers, opt)
|
||
|
|
|
||
|
|
print('Computing results...')
|
||
|
|
with torch.no_grad():
|
||
|
|
if opt.precomp_enc_type == 'basic':
|
||
|
|
img_embs, cap_embs = encode_data_vse(model, data_loader)
|
||
|
|
else:
|
||
|
|
img_embs, cap_embs = encode_data_vse(model, data_loader, backbone=True)
|
||
|
|
print('Images: %d, Captions: %d' %
|
||
|
|
(img_embs.shape[0] / 5, cap_embs.shape[0]))
|
||
|
|
|
||
|
|
if cxc:
|
||
|
|
eval_cxc(img_embs, cap_embs, data_path)
|
||
|
|
else:
|
||
|
|
if not fold5:
|
||
|
|
# no cross-validation, full evaluation
|
||
|
|
|
||
|
|
img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 5)])
|
||
|
|
start = time.time()
|
||
|
|
sims = compute_sim(img_embs, cap_embs)
|
||
|
|
|
||
|
|
#img_embs2 = np.array([img_embs2[i] for i in range(0, len(img_embs2), 5)])
|
||
|
|
# img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 20)])
|
||
|
|
# print(img_embs[:10])
|
||
|
|
#sims2 = shard_xattn(model2, img_embs2, img_means2, cap_embs2, cap_lens2, cap_means2, opt2, shard_size=500)
|
||
|
|
# sims = sims + sims2
|
||
|
|
# print(sims.shape)
|
||
|
|
npts = img_embs.shape[0]
|
||
|
|
|
||
|
|
if save_path is not None:
|
||
|
|
np.save(save_path, {'npts': npts, 'sims': sims})
|
||
|
|
print('Save the similarity into {}'.format(save_path))
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time: {}".format(end - start))
|
||
|
|
|
||
|
|
r, rt = i2t_vse(npts, sims, return_ranks=True)
|
||
|
|
ri, rti = t2i_vse(npts, sims, return_ranks=True)
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f" % rsum)
|
||
|
|
print("Average i2t Recall: %.1f" % ar)
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" % r)
|
||
|
|
print("Average t2i Recall: %.1f" % ari)
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" % ri)
|
||
|
|
else:
|
||
|
|
# 5fold cross-validation, only for MSCOCO
|
||
|
|
results = []
|
||
|
|
for i in range(5):
|
||
|
|
img_embs_shard = img_embs[i * 5000:(i + 1) * 5000:5]
|
||
|
|
cap_embs_shard = cap_embs[i * 5000:(i + 1) * 5000]
|
||
|
|
start = time.time()
|
||
|
|
sims = compute_sim(img_embs_shard, cap_embs_shard)
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time: {}".format(end - start))
|
||
|
|
|
||
|
|
npts = img_embs_shard.shape[0]
|
||
|
|
r, rt0 = i2t_vse(npts, sims, return_ranks=True)
|
||
|
|
print("Image to text: %.1f, %.1f, %.1f, %.1f, %.1f" % r)
|
||
|
|
ri, rti0 = t2i_vse(npts, sims, return_ranks=True)
|
||
|
|
print("Text to image: %.1f, %.1f, %.1f, %.1f, %.1f" % ri)
|
||
|
|
|
||
|
|
if i == 0:
|
||
|
|
rt, rti = rt0, rti0
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f ar: %.1f ari: %.1f" % (rsum, ar, ari))
|
||
|
|
results += [list(r) + list(ri) + [ar, ari, rsum]]
|
||
|
|
|
||
|
|
print("-----------------------------------")
|
||
|
|
print("Mean metrics: ")
|
||
|
|
mean_metrics = tuple(np.array(results).mean(axis=0).flatten())
|
||
|
|
print("rsum: %.1f" % (mean_metrics[12]))
|
||
|
|
print("Average i2t Recall: %.1f" % mean_metrics[10])
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[:5])
|
||
|
|
print("Average t2i Recall: %.1f" % mean_metrics[11])
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[5:10])
|
||
|
|
|
||
|
|
def evalrank(model_path, data_path=None, split='dev', fold5=False):
|
||
|
|
"""
|
||
|
|
Evaluate a trained model on either dev or test. If `fold5=True`, 5 fold
|
||
|
|
cross-validation is done (only for MSCOCO). Otherwise, the full data is
|
||
|
|
used for evaluation.
|
||
|
|
"""
|
||
|
|
# load model and options
|
||
|
|
checkpoint = torch.load(model_path)
|
||
|
|
opt = checkpoint['opt']
|
||
|
|
print(opt)
|
||
|
|
if data_path is not None:
|
||
|
|
opt.data_path = data_path
|
||
|
|
|
||
|
|
|
||
|
|
# load vocabulary used by the model
|
||
|
|
vocab = deserialize_vocab(os.path.join(opt.vocab_path, '%s_vocab.json' % opt.data_name))
|
||
|
|
word2idx = vocab.word2idx
|
||
|
|
opt.vocab_size = len(vocab)
|
||
|
|
|
||
|
|
model = SCAN(word2idx, opt)
|
||
|
|
model = torch.nn.DataParallel(model)
|
||
|
|
model.cuda()
|
||
|
|
|
||
|
|
# load model state
|
||
|
|
model.load_state_dict(checkpoint['model'])
|
||
|
|
|
||
|
|
print('Loading dataset')
|
||
|
|
opt.batch_size = 64
|
||
|
|
data_loader = get_test_loader(split, opt.data_name, vocab,
|
||
|
|
opt.batch_size, 2, opt)
|
||
|
|
|
||
|
|
print('Computing results...')
|
||
|
|
img_embs, img_means, cap_embs, cap_lens, cap_means= encode_data(model, data_loader)
|
||
|
|
print(img_embs.shape, cap_embs.shape)
|
||
|
|
print(img_means.shape)
|
||
|
|
|
||
|
|
print('Images: %d, Captions: %d' %
|
||
|
|
(img_embs.shape[0] / 5, cap_embs.shape[0]))
|
||
|
|
|
||
|
|
if not fold5:
|
||
|
|
# no cross-validation, full evaluation
|
||
|
|
img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 5)])
|
||
|
|
#shuffle_test
|
||
|
|
#img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 20)])
|
||
|
|
print(img_embs.shape)
|
||
|
|
#print(img_embs[:10])
|
||
|
|
start = time.time()
|
||
|
|
|
||
|
|
sims = shard_xattn(model, img_embs, img_means, cap_embs, cap_lens, cap_means,opt, shard_size=500)
|
||
|
|
print(sims.shape)
|
||
|
|
#print(sims[:10])
|
||
|
|
# np.save('f30k_dev', sims)
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time:", end - start)
|
||
|
|
|
||
|
|
r, rt = i2t(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
ri, rti = t2i(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
# shuffle_test
|
||
|
|
# r, rt = i2t_shuffle(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
# ri, rti = t2i_shuffle(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f" % rsum)
|
||
|
|
print("Average i2t Recall: %.1f" % ar)
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" % r)
|
||
|
|
print("Average t2i Recall: %.1f" % ari)
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" % ri)
|
||
|
|
for ele in sims[:10]:
|
||
|
|
inds = np.argsort(ele)[::-1]
|
||
|
|
print(inds[:10])
|
||
|
|
inds = np.argsort(sims[1])[::-1]
|
||
|
|
print(inds[:10])
|
||
|
|
else:
|
||
|
|
# 5fold cross-validation, only for MSCOCO
|
||
|
|
results = []
|
||
|
|
for i in range(5):
|
||
|
|
img_embs_shard = img_embs[i * 5000:(i + 1) * 5000:5]
|
||
|
|
img_means_shard = img_means[i * 5000:(i + 1) * 5000:5]
|
||
|
|
cap_embs_shard = cap_embs[i * 5000:(i + 1) * 5000]
|
||
|
|
cap_lens_shard = cap_lens[i * 5000:(i + 1) * 5000]
|
||
|
|
cap_means_shard = cap_means[i * 5000:(i + 1) * 5000]
|
||
|
|
start = time.time()
|
||
|
|
sims = shard_xattn(model, img_embs_shard, img_means_shard, cap_embs_shard, cap_lens_shard, cap_means_shard, opt, shard_size=128)
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time:", end - start)
|
||
|
|
|
||
|
|
r, rt0 = i2t(img_embs_shard, cap_embs_shard, cap_lens_shard, sims, return_ranks=True)
|
||
|
|
print("Image to text: %.1f, %.1f, %.1f, %.1f, %.1f" % r)
|
||
|
|
ri, rti0 = t2i(img_embs_shard, cap_embs_shard, cap_lens_shard, sims, return_ranks=True)
|
||
|
|
print("Text to image: %.1f, %.1f, %.1f, %.1f, %.1f" % ri)
|
||
|
|
|
||
|
|
if i == 0:
|
||
|
|
rt, rti = rt0, rti0
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f ar: %.1f ari: %.1f" % (rsum, ar, ari))
|
||
|
|
results += [list(r) + list(ri) + [ar, ari, rsum]]
|
||
|
|
|
||
|
|
print("-----------------------------------")
|
||
|
|
print("Mean metrics: ")
|
||
|
|
mean_metrics = tuple(np.array(results).mean(axis=0).flatten())
|
||
|
|
print("rsum: %.1f" % (mean_metrics[10] * 6))
|
||
|
|
print("Average i2t Recall: %.1f" % mean_metrics[11])
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[:5])
|
||
|
|
print("Average t2i Recall: %.1f" % mean_metrics[12])
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[5:10])
|
||
|
|
|
||
|
|
torch.save({'rt': rt, 'rti': rti}, 'ranks.pth.tar')
|
||
|
|
|
||
|
|
def evalrank_fanhua(model_path, data_path=None, split='dev', fold5=False):
|
||
|
|
"""
|
||
|
|
Evaluate a trained model on either dev or test. If `fold5=True`, 5 fold
|
||
|
|
cross-validation is done (only for MSCOCO). Otherwise, the full data is
|
||
|
|
used for evaluation.
|
||
|
|
"""
|
||
|
|
# load model and options
|
||
|
|
checkpoint = torch.load(model_path)
|
||
|
|
opt = checkpoint['opt']
|
||
|
|
print(opt)
|
||
|
|
if data_path is not None:
|
||
|
|
opt.data_path = data_path
|
||
|
|
|
||
|
|
|
||
|
|
# load vocabulary used by the model
|
||
|
|
vocab = deserialize_vocab(os.path.join(opt.vocab_path, '%s_vocab.json' % opt.data_name))
|
||
|
|
word2idx = vocab.word2idx
|
||
|
|
opt.vocab_size = len(vocab)
|
||
|
|
|
||
|
|
model = SCAN(word2idx, opt)
|
||
|
|
model = torch.nn.DataParallel(model)
|
||
|
|
model.cuda()
|
||
|
|
|
||
|
|
# load model state
|
||
|
|
model.load_state_dict(checkpoint['model'])
|
||
|
|
|
||
|
|
print('Loading dataset')
|
||
|
|
opt.batch_size = 64
|
||
|
|
data_loader = get_test_loader(split, "f30k_precomp", vocab,
|
||
|
|
opt.batch_size, 2, opt)
|
||
|
|
|
||
|
|
print('Computing results...')
|
||
|
|
img_embs, img_means, cap_embs, cap_lens, cap_means= encode_data(model, data_loader)
|
||
|
|
print(img_embs.shape, cap_embs.shape)
|
||
|
|
print(img_means.shape)
|
||
|
|
|
||
|
|
print('Images: %d, Captions: %d' %
|
||
|
|
(img_embs.shape[0] / 5, cap_embs.shape[0]))
|
||
|
|
|
||
|
|
if not fold5:
|
||
|
|
# no cross-validation, full evaluation
|
||
|
|
img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 5)])
|
||
|
|
#shuffle_test
|
||
|
|
#img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 20)])
|
||
|
|
print(img_embs.shape)
|
||
|
|
#print(img_embs[:10])
|
||
|
|
start = time.time()
|
||
|
|
|
||
|
|
sims = shard_xattn(model, img_embs, img_means, cap_embs, cap_lens, cap_means,opt, shard_size=500)
|
||
|
|
print(sims.shape)
|
||
|
|
#print(sims[:10])
|
||
|
|
# np.save('f30k_dev', sims)
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time:", end - start)
|
||
|
|
|
||
|
|
r, rt = i2t(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
ri, rti = t2i(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
# shuffle_test
|
||
|
|
# r, rt = i2t_shuffle(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
# ri, rti = t2i_shuffle(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f" % rsum)
|
||
|
|
print("Average i2t Recall: %.1f" % ar)
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" % r)
|
||
|
|
print("Average t2i Recall: %.1f" % ari)
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" % ri)
|
||
|
|
for ele in sims[:10]:
|
||
|
|
inds = np.argsort(ele)[::-1]
|
||
|
|
print(inds[:10])
|
||
|
|
inds = np.argsort(sims[1])[::-1]
|
||
|
|
print(inds[:10])
|
||
|
|
else:
|
||
|
|
# 5fold cross-validation, only for MSCOCO
|
||
|
|
results = []
|
||
|
|
for i in range(5):
|
||
|
|
img_embs_shard = img_embs[i * 5000:(i + 1) * 5000:5]
|
||
|
|
img_means_shard = img_means[i * 5000:(i + 1) * 5000:5]
|
||
|
|
cap_embs_shard = cap_embs[i * 5000:(i + 1) * 5000]
|
||
|
|
cap_lens_shard = cap_lens[i * 5000:(i + 1) * 5000]
|
||
|
|
cap_means_shard = cap_means[i * 5000:(i + 1) * 5000]
|
||
|
|
start = time.time()
|
||
|
|
sims = shard_xattn(model, img_embs_shard, img_means_shard, cap_embs_shard, cap_lens_shard, cap_means_shard, opt, shard_size=128)
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time:", end - start)
|
||
|
|
|
||
|
|
r, rt0 = i2t(img_embs_shard, cap_embs_shard, cap_lens_shard, sims, return_ranks=True)
|
||
|
|
print("Image to text: %.1f, %.1f, %.1f, %.1f, %.1f" % r)
|
||
|
|
ri, rti0 = t2i(img_embs_shard, cap_embs_shard, cap_lens_shard, sims, return_ranks=True)
|
||
|
|
print("Text to image: %.1f, %.1f, %.1f, %.1f, %.1f" % ri)
|
||
|
|
|
||
|
|
if i == 0:
|
||
|
|
rt, rti = rt0, rti0
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f ar: %.1f ari: %.1f" % (rsum, ar, ari))
|
||
|
|
results += [list(r) + list(ri) + [ar, ari, rsum]]
|
||
|
|
|
||
|
|
print("-----------------------------------")
|
||
|
|
print("Mean metrics: ")
|
||
|
|
mean_metrics = tuple(np.array(results).mean(axis=0).flatten())
|
||
|
|
print("rsum: %.1f" % (mean_metrics[10] * 6))
|
||
|
|
print("Average i2t Recall: %.1f" % mean_metrics[11])
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[:5])
|
||
|
|
print("Average t2i Recall: %.1f" % mean_metrics[12])
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[5:10])
|
||
|
|
|
||
|
|
torch.save({'rt': rt, 'rti': rti}, 'ranks.pth.tar')
|
||
|
|
|
||
|
|
def evalrank2(model_path, data_path=None, split='dev', fold5=False):
|
||
|
|
"""
|
||
|
|
Evaluate a trained model on either dev or test. If `fold5=True`, 5 fold
|
||
|
|
cross-validation is done (only for MSCOCO). Otherwise, the full data is
|
||
|
|
used for evaluation.
|
||
|
|
"""
|
||
|
|
# load model and options
|
||
|
|
checkpoint = torch.load(model_path)
|
||
|
|
#checkpoint2 = torch.load("../pycharmProject/runs/bigru_bcan_adam_mean_base2/model_best.pth.tar")
|
||
|
|
#checkpoint2 = torch.load("../bcan_gpo/runs/bigru_adam_bcan_mean2/model_best.pth.tar")
|
||
|
|
checkpoint2 = torch.load("../bcan_gpo/runs/bigru_adam_bcan_mean/model_best.pth.tar")
|
||
|
|
opt = checkpoint['opt']
|
||
|
|
opt2 = checkpoint2['opt']
|
||
|
|
print(opt)
|
||
|
|
print(opt2)
|
||
|
|
if data_path is not None:
|
||
|
|
opt.data_path = data_path
|
||
|
|
|
||
|
|
|
||
|
|
# load vocabulary used by the model
|
||
|
|
vocab = deserialize_vocab(os.path.join(opt.vocab_path, '%s_vocab.json' % opt.data_name))
|
||
|
|
word2idx = vocab.word2idx
|
||
|
|
opt.vocab_size = len(vocab)
|
||
|
|
|
||
|
|
model = SCAN(word2idx, opt)
|
||
|
|
model = torch.nn.DataParallel(model)
|
||
|
|
model.cuda()
|
||
|
|
|
||
|
|
model2 = SCAN2(word2idx, opt2)
|
||
|
|
model2 = torch.nn.DataParallel(model2)
|
||
|
|
model2.cuda()
|
||
|
|
|
||
|
|
# load model state
|
||
|
|
model.load_state_dict(checkpoint['model'])
|
||
|
|
model2.load_state_dict(checkpoint2['model'])
|
||
|
|
|
||
|
|
print('Loading dataset')
|
||
|
|
opt.batch_size = 64
|
||
|
|
data_loader = get_test_loader(split, opt.data_name, vocab,
|
||
|
|
opt.batch_size, 0, opt)
|
||
|
|
data_loader2 = get_test_loader2(split, opt.data_name, vocab,
|
||
|
|
opt.batch_size, 0, opt)
|
||
|
|
|
||
|
|
print('Computing results...')
|
||
|
|
img_embs, img_means, cap_embs, cap_lens, cap_means= encode_data(model, data_loader)
|
||
|
|
img_embs2, img_means2, cap_embs2, cap_lens2, cap_means2 = encode_data2(model2, data_loader2)
|
||
|
|
print(img_embs.shape, cap_embs.shape)
|
||
|
|
print(img_means.shape)
|
||
|
|
|
||
|
|
print('Images: %d, Captions: %d' %
|
||
|
|
(img_embs.shape[0] / 5, cap_embs.shape[0]))
|
||
|
|
|
||
|
|
if not fold5:
|
||
|
|
# no cross-validation, full evaluation
|
||
|
|
img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 5)])
|
||
|
|
img_embs2 = np.array([img_embs2[i] for i in range(0, len(img_embs2), 5)])
|
||
|
|
#shuffle_test
|
||
|
|
#img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 20)])
|
||
|
|
print(img_embs.shape)
|
||
|
|
#print(img_embs[:10])
|
||
|
|
start = time.time()
|
||
|
|
|
||
|
|
sims = shard_xattn(model, img_embs, img_means, cap_embs, cap_lens, cap_means,opt, shard_size=500)
|
||
|
|
sims2 = shard_xattn(model2, img_embs2, img_means2, cap_embs2, cap_lens2, cap_means2, opt2, shard_size=500)
|
||
|
|
sims = sims + sims2
|
||
|
|
print(sims.shape)
|
||
|
|
#print(sims[:10])
|
||
|
|
# np.save('f30k_dev', sims)
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time:", end - start)
|
||
|
|
|
||
|
|
r, rt = i2t(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
ri, rti = t2i(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
# shuffle_test
|
||
|
|
# r, rt = i2t_shuffle(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
# ri, rti = t2i_shuffle(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f" % rsum)
|
||
|
|
print("Average i2t Recall: %.1f" % ar)
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" % r)
|
||
|
|
print("Average t2i Recall: %.1f" % ari)
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" % ri)
|
||
|
|
for ele in sims[:10]:
|
||
|
|
inds = np.argsort(ele)[::-1]
|
||
|
|
print(inds[:10])
|
||
|
|
inds = np.argsort(sims[1])[::-1]
|
||
|
|
print(inds[:10])
|
||
|
|
else:
|
||
|
|
# 5fold cross-validation, only for MSCOCO
|
||
|
|
results = []
|
||
|
|
for i in range(5):
|
||
|
|
img_embs_shard = img_embs[i * 5000:(i + 1) * 5000:5]
|
||
|
|
img_means_shard = img_means[i * 5000:(i + 1) * 5000:5]
|
||
|
|
cap_embs_shard = cap_embs[i * 5000:(i + 1) * 5000]
|
||
|
|
cap_lens_shard = cap_lens[i * 5000:(i + 1) * 5000]
|
||
|
|
cap_means_shard = cap_means[i * 5000:(i + 1) * 5000]
|
||
|
|
start = time.time()
|
||
|
|
sims = shard_xattn(model, img_embs_shard, img_means_shard, cap_embs_shard, cap_lens_shard, cap_means_shard, opt, shard_size=128)
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time:", end - start)
|
||
|
|
|
||
|
|
r, rt0 = i2t(img_embs_shard, cap_embs_shard, cap_lens_shard, sims, return_ranks=True)
|
||
|
|
print("Image to text: %.1f, %.1f, %.1f, %.1f, %.1f" % r)
|
||
|
|
ri, rti0 = t2i(img_embs_shard, cap_embs_shard, cap_lens_shard, sims, return_ranks=True)
|
||
|
|
print("Text to image: %.1f, %.1f, %.1f, %.1f, %.1f" % ri)
|
||
|
|
|
||
|
|
if i == 0:
|
||
|
|
rt, rti = rt0, rti0
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f ar: %.1f ari: %.1f" % (rsum, ar, ari))
|
||
|
|
results += [list(r) + list(ri) + [ar, ari, rsum]]
|
||
|
|
|
||
|
|
print("-----------------------------------")
|
||
|
|
print("Mean metrics: ")
|
||
|
|
mean_metrics = tuple(np.array(results).mean(axis=0).flatten())
|
||
|
|
print("rsum: %.1f" % (mean_metrics[10] * 6))
|
||
|
|
print("Average i2t Recall: %.1f" % mean_metrics[11])
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[:5])
|
||
|
|
print("Average t2i Recall: %.1f" % mean_metrics[12])
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[5:10])
|
||
|
|
|
||
|
|
torch.save({'rt': rt, 'rti': rti}, 'ranks.pth.tar')
|
||
|
|
|
||
|
|
def evalrank_avgpool(model_path, data_path=None, split='dev', fold5=False):
|
||
|
|
"""
|
||
|
|
Evaluate a trained model on either dev or test. If `fold5=True`, 5 fold
|
||
|
|
cross-validation is done (only for MSCOCO). Otherwise, the full data is
|
||
|
|
used for evaluation.
|
||
|
|
"""
|
||
|
|
# load model and options
|
||
|
|
checkpoint = torch.load(model_path)
|
||
|
|
#checkpoint2 = torch.load("../pycharmProject/runs/bigru_bcan_adam_mean_base2/model_best.pth.tar")
|
||
|
|
checkpoint2 = torch.load("../bcan_gpo/runs/bigru_adam_bcan_mean2/model_best.pth.tar")
|
||
|
|
opt = checkpoint['opt']
|
||
|
|
opt2 = checkpoint2['opt']
|
||
|
|
print(opt)
|
||
|
|
print(opt2)
|
||
|
|
if data_path is not None:
|
||
|
|
opt.data_path = data_path
|
||
|
|
|
||
|
|
|
||
|
|
# load vocabulary used by the model
|
||
|
|
vocab = deserialize_vocab(os.path.join(opt.vocab_path, '%s_vocab.json' % opt.data_name))
|
||
|
|
word2idx = vocab.word2idx
|
||
|
|
opt.vocab_size = len(vocab)
|
||
|
|
|
||
|
|
model = SCAN(word2idx, opt)
|
||
|
|
model = torch.nn.DataParallel(model)
|
||
|
|
model.cuda()
|
||
|
|
|
||
|
|
model2 = SCAN2(word2idx, opt2)
|
||
|
|
model2 = torch.nn.DataParallel(model2)
|
||
|
|
model2.cuda()
|
||
|
|
|
||
|
|
# load model state
|
||
|
|
model.load_state_dict(checkpoint['model'])
|
||
|
|
model2.load_state_dict(checkpoint2['model'])
|
||
|
|
|
||
|
|
print('Loading dataset')
|
||
|
|
opt.batch_size = 64
|
||
|
|
data_loader = get_test_loader(split, opt.data_name, vocab,
|
||
|
|
opt.batch_size, 0, opt)
|
||
|
|
data_loader2 = get_test_loader2(split, opt.data_name, vocab,
|
||
|
|
opt.batch_size, 0, opt)
|
||
|
|
|
||
|
|
print('Computing results...')
|
||
|
|
img_embs, img_means, cap_embs, cap_lens, cap_means= encode_data(model, data_loader)
|
||
|
|
img_embs2, img_means2, cap_embs2, cap_lens2, cap_means2 = encode_data2(model2, data_loader2)
|
||
|
|
print(img_embs.shape, cap_embs.shape)
|
||
|
|
print(img_means.shape)
|
||
|
|
|
||
|
|
print('Images: %d, Captions: %d' %
|
||
|
|
(img_embs.shape[0] / 5, cap_embs.shape[0]))
|
||
|
|
|
||
|
|
if not fold5:
|
||
|
|
# no cross-validation, full evaluation
|
||
|
|
img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 5)])
|
||
|
|
img_embs2 = np.array([img_embs2[i] for i in range(0, len(img_embs2), 5)])
|
||
|
|
#shuffle_test
|
||
|
|
#img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 20)])
|
||
|
|
print(img_embs.shape)
|
||
|
|
#print(img_embs[:10])
|
||
|
|
start = time.time()
|
||
|
|
|
||
|
|
sims = shard_xattn(model, img_embs, img_means, cap_embs, cap_lens, cap_means,opt, shard_size=500)
|
||
|
|
sims2 = shard_xattn(model2, img_embs2, img_means2, cap_embs2, cap_lens2, cap_means2, opt2, shard_size=500)
|
||
|
|
sims = sims + sims2
|
||
|
|
print(sims.shape)
|
||
|
|
#print(sims[:10])
|
||
|
|
# np.save('f30k_dev', sims)
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time:", end - start)
|
||
|
|
|
||
|
|
r, rt = i2t(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
ri, rti = t2i(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
# shuffle_test
|
||
|
|
# r, rt = i2t_shuffle(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
# ri, rti = t2i_shuffle(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f" % rsum)
|
||
|
|
print("Average i2t Recall: %.1f" % ar)
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" % r)
|
||
|
|
print("Average t2i Recall: %.1f" % ari)
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" % ri)
|
||
|
|
for ele in sims[:10]:
|
||
|
|
inds = np.argsort(ele)[::-1]
|
||
|
|
print(inds[:10])
|
||
|
|
inds = np.argsort(sims[1])[::-1]
|
||
|
|
print(inds[:10])
|
||
|
|
else:
|
||
|
|
# 5fold cross-validation, only for MSCOCO
|
||
|
|
results = []
|
||
|
|
for i in range(5):
|
||
|
|
img_embs_shard = img_embs[i * 5000:(i + 1) * 5000:5]
|
||
|
|
img_means_shard = img_means[i * 5000:(i + 1) * 5000:5]
|
||
|
|
cap_embs_shard = cap_embs[i * 5000:(i + 1) * 5000]
|
||
|
|
cap_lens_shard = cap_lens[i * 5000:(i + 1) * 5000]
|
||
|
|
cap_means_shard = cap_means[i * 5000:(i + 1) * 5000]
|
||
|
|
start = time.time()
|
||
|
|
sims = shard_xattn(model, img_embs_shard, img_means_shard, cap_embs_shard, cap_lens_shard, cap_means_shard, opt, shard_size=128)
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time:", end - start)
|
||
|
|
|
||
|
|
r, rt0 = i2t(img_embs_shard, cap_embs_shard, cap_lens_shard, sims, return_ranks=True)
|
||
|
|
print("Image to text: %.1f, %.1f, %.1f, %.1f, %.1f" % r)
|
||
|
|
ri, rti0 = t2i(img_embs_shard, cap_embs_shard, cap_lens_shard, sims, return_ranks=True)
|
||
|
|
print("Text to image: %.1f, %.1f, %.1f, %.1f, %.1f" % ri)
|
||
|
|
|
||
|
|
if i == 0:
|
||
|
|
rt, rti = rt0, rti0
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f ar: %.1f ari: %.1f" % (rsum, ar, ari))
|
||
|
|
results += [list(r) + list(ri) + [ar, ari, rsum]]
|
||
|
|
|
||
|
|
print("-----------------------------------")
|
||
|
|
print("Mean metrics: ")
|
||
|
|
mean_metrics = tuple(np.array(results).mean(axis=0).flatten())
|
||
|
|
print("rsum: %.1f" % (mean_metrics[10] * 6))
|
||
|
|
print("Average i2t Recall: %.1f" % mean_metrics[11])
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[:5])
|
||
|
|
print("Average t2i Recall: %.1f" % mean_metrics[12])
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[5:10])
|
||
|
|
|
||
|
|
torch.save({'rt': rt, 'rti': rti}, 'ranks.pth.tar')
|
||
|
|
def evalrank_maxpool(model_path, data_path=None, split='dev', fold5=False):
|
||
|
|
"""
|
||
|
|
Evaluate a trained model on either dev or test. If `fold5=True`, 5 fold
|
||
|
|
cross-validation is done (only for MSCOCO). Otherwise, the full data is
|
||
|
|
used for evaluation.
|
||
|
|
"""
|
||
|
|
# load model and options
|
||
|
|
checkpoint = torch.load(model_path)
|
||
|
|
#checkpoint2 = torch.load("../pycharmProject/runs/bigru_bcan_adam_mean_base2/model_best.pth.tar")
|
||
|
|
checkpoint2 = torch.load("../bcan_gpo/runs/bigru_bcan_adam_max_36/model_best.pth.tar")
|
||
|
|
opt = checkpoint['opt']
|
||
|
|
opt2 = checkpoint2['opt']
|
||
|
|
print(opt)
|
||
|
|
print(opt2)
|
||
|
|
if data_path is not None:
|
||
|
|
opt.data_path = data_path
|
||
|
|
|
||
|
|
|
||
|
|
# load vocabulary used by the model
|
||
|
|
vocab = deserialize_vocab(os.path.join(opt.vocab_path, '%s_vocab.json' % opt.data_name))
|
||
|
|
word2idx = vocab.word2idx
|
||
|
|
opt.vocab_size = len(vocab)
|
||
|
|
|
||
|
|
model = SCAN(word2idx, opt)
|
||
|
|
model = torch.nn.DataParallel(model)
|
||
|
|
model.cuda()
|
||
|
|
|
||
|
|
model2 = SCAN3(word2idx, opt2)
|
||
|
|
model2 = torch.nn.DataParallel(model2)
|
||
|
|
model2.cuda()
|
||
|
|
|
||
|
|
# load model state
|
||
|
|
model.load_state_dict(checkpoint['model'])
|
||
|
|
model2.load_state_dict(checkpoint2['model'])
|
||
|
|
|
||
|
|
print('Loading dataset')
|
||
|
|
opt.batch_size = 64
|
||
|
|
data_loader = get_test_loader(split, opt.data_name, vocab,
|
||
|
|
opt.batch_size, 0, opt)
|
||
|
|
data_loader2 = get_test_loader2(split, opt.data_name, vocab,
|
||
|
|
opt.batch_size, 0, opt)
|
||
|
|
|
||
|
|
print('Computing results...')
|
||
|
|
img_embs, img_means, cap_embs, cap_lens, cap_means= encode_data(model, data_loader)
|
||
|
|
img_embs2, img_means2, cap_embs2, cap_lens2, cap_means2 = encode_data2(model2, data_loader2)
|
||
|
|
print(img_embs.shape, cap_embs.shape)
|
||
|
|
print(img_means.shape)
|
||
|
|
|
||
|
|
print('Images: %d, Captions: %d' %
|
||
|
|
(img_embs.shape[0] / 5, cap_embs.shape[0]))
|
||
|
|
|
||
|
|
if not fold5:
|
||
|
|
# no cross-validation, full evaluation
|
||
|
|
img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 5)])
|
||
|
|
img_embs2 = np.array([img_embs2[i] for i in range(0, len(img_embs2), 5)])
|
||
|
|
#shuffle_test
|
||
|
|
#img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 20)])
|
||
|
|
print(img_embs.shape)
|
||
|
|
#print(img_embs[:10])
|
||
|
|
start = time.time()
|
||
|
|
|
||
|
|
sims = shard_xattn(model, img_embs, img_means, cap_embs, cap_lens, cap_means,opt, shard_size=500)
|
||
|
|
sims2 = shard_xattn(model2, img_embs2, img_means2, cap_embs2, cap_lens2, cap_means2, opt2, shard_size=500)
|
||
|
|
sims = sims + sims2
|
||
|
|
print(sims.shape)
|
||
|
|
#print(sims[:10])
|
||
|
|
# np.save('f30k_dev', sims)
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time:", end - start)
|
||
|
|
|
||
|
|
r, rt = i2t(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
ri, rti = t2i(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
# shuffle_test
|
||
|
|
# r, rt = i2t_shuffle(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
# ri, rti = t2i_shuffle(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f" % rsum)
|
||
|
|
print("Average i2t Recall: %.1f" % ar)
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" % r)
|
||
|
|
print("Average t2i Recall: %.1f" % ari)
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" % ri)
|
||
|
|
for ele in sims[:10]:
|
||
|
|
inds = np.argsort(ele)[::-1]
|
||
|
|
print(inds[:10])
|
||
|
|
inds = np.argsort(sims[1])[::-1]
|
||
|
|
print(inds[:10])
|
||
|
|
else:
|
||
|
|
# 5fold cross-validation, only for MSCOCO
|
||
|
|
results = []
|
||
|
|
for i in range(5):
|
||
|
|
img_embs_shard = img_embs[i * 5000:(i + 1) * 5000:5]
|
||
|
|
img_means_shard = img_means[i * 5000:(i + 1) * 5000:5]
|
||
|
|
cap_embs_shard = cap_embs[i * 5000:(i + 1) * 5000]
|
||
|
|
cap_lens_shard = cap_lens[i * 5000:(i + 1) * 5000]
|
||
|
|
cap_means_shard = cap_means[i * 5000:(i + 1) * 5000]
|
||
|
|
start = time.time()
|
||
|
|
sims = shard_xattn(model, img_embs_shard, img_means_shard, cap_embs_shard, cap_lens_shard, cap_means_shard, opt, shard_size=128)
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time:", end - start)
|
||
|
|
|
||
|
|
r, rt0 = i2t(img_embs_shard, cap_embs_shard, cap_lens_shard, sims, return_ranks=True)
|
||
|
|
print("Image to text: %.1f, %.1f, %.1f, %.1f, %.1f" % r)
|
||
|
|
ri, rti0 = t2i(img_embs_shard, cap_embs_shard, cap_lens_shard, sims, return_ranks=True)
|
||
|
|
print("Text to image: %.1f, %.1f, %.1f, %.1f, %.1f" % ri)
|
||
|
|
|
||
|
|
if i == 0:
|
||
|
|
rt, rti = rt0, rti0
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f ar: %.1f ari: %.1f" % (rsum, ar, ari))
|
||
|
|
results += [list(r) + list(ri) + [ar, ari, rsum]]
|
||
|
|
|
||
|
|
print("-----------------------------------")
|
||
|
|
print("Mean metrics: ")
|
||
|
|
mean_metrics = tuple(np.array(results).mean(axis=0).flatten())
|
||
|
|
print("rsum: %.1f" % (mean_metrics[10] * 6))
|
||
|
|
print("Average i2t Recall: %.1f" % mean_metrics[11])
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[:5])
|
||
|
|
print("Average t2i Recall: %.1f" % mean_metrics[12])
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[5:10])
|
||
|
|
|
||
|
|
torch.save({'rt': rt, 'rti': rti}, 'ranks.pth.tar')
|
||
|
|
|
||
|
|
def evalrank3(model_path, data_path=None, split='dev', fold5=False):
|
||
|
|
"""
|
||
|
|
Evaluate a trained model on either dev or test. If `fold5=True`, 5 fold
|
||
|
|
cross-validation is done (only for MSCOCO). Otherwise, the full data is
|
||
|
|
used for evaluation.
|
||
|
|
"""
|
||
|
|
# load model and options
|
||
|
|
checkpoint = torch.load(model_path)
|
||
|
|
opt = checkpoint['opt']
|
||
|
|
print(opt)
|
||
|
|
if data_path is not None:
|
||
|
|
opt.data_path = data_path
|
||
|
|
|
||
|
|
|
||
|
|
# load vocabulary used by the model
|
||
|
|
vocab = deserialize_vocab(os.path.join(opt.vocab_path, '%s_vocab.json' % opt.data_name))
|
||
|
|
word2idx = vocab.word2idx
|
||
|
|
opt.vocab_size = len(vocab)
|
||
|
|
|
||
|
|
model = SCAN2(word2idx, opt)
|
||
|
|
model = torch.nn.DataParallel(model)
|
||
|
|
model.cuda()
|
||
|
|
|
||
|
|
# load model state
|
||
|
|
model.load_state_dict(checkpoint['model'])
|
||
|
|
|
||
|
|
print('Loading dataset')
|
||
|
|
data_loader = get_test_loader2(split, opt.data_name, vocab,
|
||
|
|
opt.batch_size, 0, opt)
|
||
|
|
|
||
|
|
print('Computing results...')
|
||
|
|
img_embs, img_means, cap_embs, cap_lens, cap_means= encode_data2(model, data_loader)
|
||
|
|
print(img_embs.shape, cap_embs.shape)
|
||
|
|
print('Images: %d, Captions: %d' %
|
||
|
|
(img_embs.shape[0] / 5, cap_embs.shape[0]))
|
||
|
|
|
||
|
|
if not fold5:
|
||
|
|
# no cross-validation, full evaluation
|
||
|
|
img_embs = np.array([img_embs[i] for i in range(0, len(img_embs), 5)])
|
||
|
|
start = time.time()
|
||
|
|
|
||
|
|
sims = shard_xattn(model, img_embs, img_means, cap_embs, cap_lens, cap_means,opt, shard_size=500)
|
||
|
|
print(sims.shape)
|
||
|
|
# np.save('f30k_dev', sims)
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time:", end - start)
|
||
|
|
|
||
|
|
r, rt = i2t(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
ri, rti = t2i(img_embs, cap_embs, cap_lens, sims, return_ranks=True)
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f" % rsum)
|
||
|
|
print("Average i2t Recall: %.1f" % ar)
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" % r)
|
||
|
|
print("Average t2i Recall: %.1f" % ari)
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" % ri)
|
||
|
|
for ele in sims[:10]:
|
||
|
|
inds = np.argsort(ele)[::-1]
|
||
|
|
print(inds[:10])
|
||
|
|
inds = np.argsort(sims[1])[::-1]
|
||
|
|
print(inds[:10])
|
||
|
|
else:
|
||
|
|
# 5fold cross-validation, only for MSCOCO
|
||
|
|
results = []
|
||
|
|
for i in range(5):
|
||
|
|
img_embs_shard = img_embs[i * 5000:(i + 1) * 5000:5]
|
||
|
|
img_means_shard = img_means[i * 5000:(i + 1) * 5000:5]
|
||
|
|
cap_embs_shard = cap_embs[i * 5000:(i + 1) * 5000]
|
||
|
|
cap_lens_shard = cap_lens[i * 5000:(i + 1) * 5000]
|
||
|
|
cap_means_shard = cap_means[i * 5000:(i + 1) * 5000]
|
||
|
|
start = time.time()
|
||
|
|
sims = shard_xattn(model, img_embs_shard, img_means_shard, cap_embs_shard, cap_lens_shard, cap_means_shard, opt, shard_size=128)
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
print("calculate similarity time:", end - start)
|
||
|
|
|
||
|
|
r, rt0 = i2t(img_embs_shard, cap_embs_shard, cap_lens_shard, sims, return_ranks=True)
|
||
|
|
print("Image to text: %.1f, %.1f, %.1f, %.1f, %.1f" % r)
|
||
|
|
ri, rti0 = t2i(img_embs_shard, cap_embs_shard, cap_lens_shard, sims, return_ranks=True)
|
||
|
|
print("Text to image: %.1f, %.1f, %.1f, %.1f, %.1f" % ri)
|
||
|
|
|
||
|
|
if i == 0:
|
||
|
|
rt, rti = rt0, rti0
|
||
|
|
ar = (r[0] + r[1] + r[2]) / 3
|
||
|
|
ari = (ri[0] + ri[1] + ri[2]) / 3
|
||
|
|
rsum = r[0] + r[1] + r[2] + ri[0] + ri[1] + ri[2]
|
||
|
|
print("rsum: %.1f ar: %.1f ari: %.1f" % (rsum, ar, ari))
|
||
|
|
results += [list(r) + list(ri) + [ar, ari, rsum]]
|
||
|
|
|
||
|
|
print("-----------------------------------")
|
||
|
|
print("Mean metrics: ")
|
||
|
|
mean_metrics = tuple(np.array(results).mean(axis=0).flatten())
|
||
|
|
print("rsum: %.1f" % (mean_metrics[10] * 6))
|
||
|
|
print("Average i2t Recall: %.1f" % mean_metrics[11])
|
||
|
|
print("Image to text: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[:5])
|
||
|
|
print("Average t2i Recall: %.1f" % mean_metrics[12])
|
||
|
|
print("Text to image: %.1f %.1f %.1f %.1f %.1f" %
|
||
|
|
mean_metrics[5:10])
|
||
|
|
|
||
|
|
torch.save({'rt': rt, 'rti': rti}, 'ranks.pth.tar')
|
||
|
|
|
||
|
|
def shard_xattn(model, images, img_means, captions, caplens, cap_means, opt, shard_size=128):
|
||
|
|
"""
|
||
|
|
Computer pairwise t2i image-caption distance with locality sharding
|
||
|
|
"""
|
||
|
|
n_im_shard = (len(images) - 1) // shard_size + 1
|
||
|
|
n_cap_shard = (len(captions) - 1) // shard_size + 1
|
||
|
|
|
||
|
|
d = np.zeros((len(images), len(captions)))
|
||
|
|
for i in range(n_im_shard):
|
||
|
|
im_start, im_end = shard_size * i, min(shard_size * (i + 1), len(images))
|
||
|
|
im = Variable(torch.from_numpy(images[im_start:im_end]), volatile=True).float().cuda()
|
||
|
|
im_m = Variable(torch.from_numpy(img_means[im_start:im_end]), volatile=True).float().cuda()
|
||
|
|
for j in range(n_cap_shard):
|
||
|
|
sys.stdout.write('\r>> shard_xattn_t2i batch (%d,%d)' % (i, j))
|
||
|
|
cap_start, cap_end = shard_size * j, min(shard_size * (j + 1), len(captions))
|
||
|
|
# im = Variable(torch.from_numpy(images[im_start:im_end]), volatile=True).float().cuda()
|
||
|
|
# im_m = Variable(torch.from_numpy(img_means[im_start:im_end]), volatile=True).float().cuda()
|
||
|
|
s_m = Variable(torch.from_numpy(cap_means[cap_start:cap_end]), volatile=True).float().cuda()
|
||
|
|
s = Variable(torch.from_numpy(captions[cap_start:cap_end]), volatile=True).float().cuda()
|
||
|
|
l = caplens[cap_start:cap_end]
|
||
|
|
with torch.no_grad():
|
||
|
|
sim = model.module.forward_sim(im, im_m, s, l, s_m)
|
||
|
|
# sim = model.xattn_score_t2i2(im, s, l)
|
||
|
|
d[im_start:im_end, cap_start:cap_end] = sim.data.cpu().numpy()
|
||
|
|
sys.stdout.write('\n')
|
||
|
|
return d
|
||
|
|
|
||
|
|
|
||
|
|
def i2t(images, captions, caplens, sims, npts=None, return_ranks=False):
|
||
|
|
"""
|
||
|
|
Images->Text (Image Annotation)
|
||
|
|
Images: (N, n_region, d) matrix of images
|
||
|
|
Captions: (5N, max_n_word, d) matrix of captions
|
||
|
|
CapLens: (5N) array of caption lengths
|
||
|
|
sims: (N, 5N) matrix of similarity im-cap
|
||
|
|
"""
|
||
|
|
npts = images.shape[0]
|
||
|
|
ranks = np.zeros(npts)
|
||
|
|
top1 = np.zeros(npts)
|
||
|
|
for index in range(npts):
|
||
|
|
inds = np.argsort(sims[index])[::-1]
|
||
|
|
# Score
|
||
|
|
rank = 1e20
|
||
|
|
for i in range(5 * index, 5 * index + 5, 1):
|
||
|
|
# print(inds, i, index, npts)
|
||
|
|
tmp = np.where(inds == i)[0][0]
|
||
|
|
if tmp < rank:
|
||
|
|
rank = tmp
|
||
|
|
ranks[index] = rank
|
||
|
|
top1[index] = inds[0]
|
||
|
|
|
||
|
|
# Compute metrics
|
||
|
|
r1 = 100.0 * len(np.where(ranks < 1)[0]) / len(ranks)
|
||
|
|
r5 = 100.0 * len(np.where(ranks < 5)[0]) / len(ranks)
|
||
|
|
r10 = 100.0 * len(np.where(ranks < 10)[0]) / len(ranks)
|
||
|
|
medr = np.floor(np.median(ranks)) + 1
|
||
|
|
meanr = ranks.mean() + 1
|
||
|
|
if return_ranks:
|
||
|
|
return (r1, r5, r10, medr, meanr), (ranks, top1)
|
||
|
|
else:
|
||
|
|
return (r1, r5, r10, medr, meanr)
|
||
|
|
|
||
|
|
|
||
|
|
def t2i(images, captions, caplens, sims, npts=None, return_ranks=False):
|
||
|
|
"""
|
||
|
|
Text->Images (Image Search)
|
||
|
|
Images: (N, n_region, d) matrix of images
|
||
|
|
Captions: (5N, max_n_word, d) matrix of captions
|
||
|
|
CapLens: (5N) array of caption lengths
|
||
|
|
sims: (N, 5N) matrix of similarity im-cap
|
||
|
|
"""
|
||
|
|
npts = images.shape[0]
|
||
|
|
ranks = np.zeros(5 * npts)
|
||
|
|
top1 = np.zeros(5 * npts)
|
||
|
|
|
||
|
|
# --> (5N(caption), N(image))
|
||
|
|
sims = sims.T
|
||
|
|
|
||
|
|
for index in range(npts):
|
||
|
|
for i in range(5):
|
||
|
|
inds = np.argsort(sims[5 * index + i])[::-1]
|
||
|
|
ranks[5 * index + i] = np.where(inds == index)[0][0]
|
||
|
|
top1[5 * index + i] = inds[0]
|
||
|
|
|
||
|
|
# Compute metrics
|
||
|
|
r1 = 100.0 * len(np.where(ranks < 1)[0]) / len(ranks)
|
||
|
|
r5 = 100.0 * len(np.where(ranks < 5)[0]) / len(ranks)
|
||
|
|
r10 = 100.0 * len(np.where(ranks < 10)[0]) / len(ranks)
|
||
|
|
medr = np.floor(np.median(ranks)) + 1
|
||
|
|
meanr = ranks.mean() + 1
|
||
|
|
if return_ranks:
|
||
|
|
return (r1, r5, r10, medr, meanr), (ranks, top1)
|
||
|
|
else:
|
||
|
|
return (r1, r5, r10, medr, meanr)
|
||
|
|
|
||
|
|
def i2t_shuffle(images, captions, caplens, sims, npts=None, return_ranks=False):
|
||
|
|
"""
|
||
|
|
Images->Text (Image Annotation)
|
||
|
|
Images: (N, n_region, d) matrix of images
|
||
|
|
Captions: (5N, max_n_word, d) matrix of captions
|
||
|
|
CapLens: (5N) array of caption lengths
|
||
|
|
sims: (N, 5N) matrix of similarity im-cap
|
||
|
|
"""
|
||
|
|
npts = images.shape[0]
|
||
|
|
ranks = np.zeros(npts)
|
||
|
|
top1 = np.zeros(npts)
|
||
|
|
for index in range(npts):
|
||
|
|
inds = np.argsort(sims[index])[::-1]
|
||
|
|
# Score
|
||
|
|
rank = 1e20
|
||
|
|
for i in range(20 * index, 20 * index + 20, 1):
|
||
|
|
# print(inds, i, index, npts)
|
||
|
|
tmp = np.where(inds == i)[0][0]
|
||
|
|
if tmp < rank:
|
||
|
|
rank = tmp
|
||
|
|
ranks[index] = rank
|
||
|
|
top1[index] = inds[0]
|
||
|
|
|
||
|
|
# Compute metrics
|
||
|
|
r1 = 100.0 * len(np.where(ranks < 1)[0]) / len(ranks)
|
||
|
|
r5 = 100.0 * len(np.where(ranks < 5)[0]) / len(ranks)
|
||
|
|
r10 = 100.0 * len(np.where(ranks < 10)[0]) / len(ranks)
|
||
|
|
medr = np.floor(np.median(ranks)) + 1
|
||
|
|
meanr = ranks.mean() + 1
|
||
|
|
if return_ranks:
|
||
|
|
return (r1, r5, r10, medr, meanr), (ranks, top1)
|
||
|
|
else:
|
||
|
|
return (r1, r5, r10, medr, meanr)
|
||
|
|
|
||
|
|
|
||
|
|
def t2i_shuffle(images, captions, caplens, sims, npts=None, return_ranks=False):
|
||
|
|
"""
|
||
|
|
Text->Images (Image Search)
|
||
|
|
Images: (N, n_region, d) matrix of images
|
||
|
|
Captions: (5N, max_n_word, d) matrix of captions
|
||
|
|
CapLens: (5N) array of caption lengths
|
||
|
|
sims: (N, 5N) matrix of similarity im-cap
|
||
|
|
"""
|
||
|
|
npts = images.shape[0]
|
||
|
|
ranks = np.zeros(20 * npts)
|
||
|
|
top1 = np.zeros(20 * npts)
|
||
|
|
|
||
|
|
# --> (5N(caption), N(image))
|
||
|
|
sims = sims.T
|
||
|
|
|
||
|
|
for index in range(npts):
|
||
|
|
for i in range(20):
|
||
|
|
inds = np.argsort(sims[20 * index + i])[::-1]
|
||
|
|
ranks[20 * index + i] = np.where(inds == index)[0][0]
|
||
|
|
top1[20 * index + i] = inds[0]
|
||
|
|
|
||
|
|
# Compute metrics
|
||
|
|
r1 = 100.0 * len(np.where(ranks < 1)[0]) / len(ranks)
|
||
|
|
r5 = 100.0 * len(np.where(ranks < 5)[0]) / len(ranks)
|
||
|
|
r10 = 100.0 * len(np.where(ranks < 10)[0]) / len(ranks)
|
||
|
|
medr = np.floor(np.median(ranks)) + 1
|
||
|
|
meanr = ranks.mean() + 1
|
||
|
|
if return_ranks:
|
||
|
|
return (r1, r5, r10, medr, meanr), (ranks, top1)
|
||
|
|
else:
|
||
|
|
return (r1, r5, r10, medr, meanr)
|
||
|
|
|
||
|
|
def i2t_vse(npts, sims, return_ranks=False, mode='coco'):
|
||
|
|
"""
|
||
|
|
Images->Text (Image Annotation)
|
||
|
|
Images: (N, n_region, d) matrix of images
|
||
|
|
Captions: (5N, max_n_word, d) matrix of captions
|
||
|
|
CapLens: (5N) array of caption lengths
|
||
|
|
sims: (N, 5N) matrix of similarity im-cap
|
||
|
|
"""
|
||
|
|
ranks = np.zeros(npts)
|
||
|
|
top1 = np.zeros(npts)
|
||
|
|
for index in range(npts):
|
||
|
|
inds = np.argsort(sims[index])[::-1]
|
||
|
|
if mode == 'coco':
|
||
|
|
rank = 1e20
|
||
|
|
for i in range(5 * index, 5 * index + 5, 1):
|
||
|
|
tmp = np.where(inds == i)[0][0]
|
||
|
|
if tmp < rank:
|
||
|
|
rank = tmp
|
||
|
|
ranks[index] = rank
|
||
|
|
top1[index] = inds[0]
|
||
|
|
else:
|
||
|
|
rank = np.where(inds == index)[0][0]
|
||
|
|
ranks[index] = rank
|
||
|
|
top1[index] = inds[0]
|
||
|
|
|
||
|
|
# Compute metrics
|
||
|
|
r1 = 100.0 * len(np.where(ranks < 1)[0]) / len(ranks)
|
||
|
|
r5 = 100.0 * len(np.where(ranks < 5)[0]) / len(ranks)
|
||
|
|
r10 = 100.0 * len(np.where(ranks < 10)[0]) / len(ranks)
|
||
|
|
medr = np.floor(np.median(ranks)) + 1
|
||
|
|
meanr = ranks.mean() + 1
|
||
|
|
|
||
|
|
if return_ranks:
|
||
|
|
return (r1, r5, r10, medr, meanr), (ranks, top1)
|
||
|
|
else:
|
||
|
|
return (r1, r5, r10, medr, meanr)
|
||
|
|
|
||
|
|
|
||
|
|
def t2i_vse(npts, sims, return_ranks=False, mode='coco'):
|
||
|
|
"""
|
||
|
|
Text->Images (Image Search)
|
||
|
|
Images: (N, n_region, d) matrix of images
|
||
|
|
Captions: (5N, max_n_word, d) matrix of captions
|
||
|
|
CapLens: (5N) array of caption lengths
|
||
|
|
sims: (N, 5N) matrix of similarity im-cap
|
||
|
|
"""
|
||
|
|
# npts = images.shape[0]
|
||
|
|
|
||
|
|
if mode == 'coco':
|
||
|
|
ranks = np.zeros(5 * npts)
|
||
|
|
top1 = np.zeros(5 * npts)
|
||
|
|
else:
|
||
|
|
ranks = np.zeros(npts)
|
||
|
|
top1 = np.zeros(npts)
|
||
|
|
|
||
|
|
# --> (5N(caption), N(image))
|
||
|
|
sims = sims.T
|
||
|
|
|
||
|
|
for index in range(npts):
|
||
|
|
if mode == 'coco':
|
||
|
|
for i in range(5):
|
||
|
|
inds = np.argsort(sims[5 * index + i])[::-1]
|
||
|
|
ranks[5 * index + i] = np.where(inds == index)[0][0]
|
||
|
|
top1[5 * index + i] = inds[0]
|
||
|
|
else:
|
||
|
|
inds = np.argsort(sims[index])[::-1]
|
||
|
|
ranks[index] = np.where(inds == index)[0][0]
|
||
|
|
top1[index] = inds[0]
|
||
|
|
|
||
|
|
# Compute metrics
|
||
|
|
r1 = 100.0 * len(np.where(ranks < 1)[0]) / len(ranks)
|
||
|
|
r5 = 100.0 * len(np.where(ranks < 5)[0]) / len(ranks)
|
||
|
|
r10 = 100.0 * len(np.where(ranks < 10)[0]) / len(ranks)
|
||
|
|
medr = np.floor(np.median(ranks)) + 1
|
||
|
|
meanr = ranks.mean() + 1
|
||
|
|
if return_ranks:
|
||
|
|
return (r1, r5, r10, medr, meanr), (ranks, top1)
|
||
|
|
else:
|
||
|
|
return (r1, r5, r10, medr, meanr)
|