From 773bbf57175e51fc3f138927407c05cd30f7d7eb Mon Sep 17 00:00:00 2001 From: Qiren Sun Date: Tue, 21 May 2019 10:27:05 -0400 Subject: [PATCH 1/5] Add files via upload --- local.py | 359 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ models.py | 238 ++++++++++++++++++++++++++++++++++++ utils.py | 147 ++++++++++++++++++++++ 3 files changed, 744 insertions(+) create mode 100644 local.py create mode 100644 models.py create mode 100644 utils.py diff --git a/local.py b/local.py new file mode 100644 index 0000000..9365427 --- /dev/null +++ b/local.py @@ -0,0 +1,359 @@ + +import os +import gc +import json +import progressbar +import visual_genome.utils as utils +from visual_genome.models import (Image, Object, Attribute, Relationship, + Graph, Synset) + + +def get_all_image_data(data_dir=None): + """ + Get Image ids from start_index to end_index. + """ + if data_dir is None: + data_dir = utils.get_data_dir() + dataFile = os.path.join(data_dir, 'image_data.json') + data = json.load(open(dataFile)) + return [utils.parse_image_data(image) for image in data] + + +def get_all_region_descriptions(data_dir=None): + """ + Get all region descriptions. + """ + if data_dir is None: + data_dir = utils.get_data_dir() + data_file = os.path.join(data_dir, 'region_descriptions.json') + image_data = get_all_image_data(data_dir) + image_map = {} + for d in image_data: + image_map[d.id] = d + images = json.load(open(data_file)) + output = [] + for image in images: + output.append(utils.parse_region_descriptions( + image['regions'], image_map[image['id']])) + return output + + + +def get_all_attributes(data_dir=None): + """ + Get all attributes. + """ + if data_dir is None: + data_dir = utils.get_data_dir() + data_file = os.path.join(data_dir, 'attributes.json') + image_data = get_all_image_data(data_dir) + image_map = {} + for d in image_data: + image_map[d.id] = d + images = json.load(open(data_file)) + output = [] + for image in images: + output.append(utils.parse_attributes( + image['attributes'], image_map[image['image_id']])) + return output + + +def get_all_relationships(data_dir=None): + """ + Get all relationships. + """ + if data_dir is None: + data_dir = utils.get_data_dir() + data_file = os.path.join(data_dir, 'relationships.json') + image_data = get_all_image_data(data_dir) + image_map = {} + for d in image_data: + image_map[d.id] = d + images = json.load(open(data_file)) + output = [] + for image in images: + output.append(utils.parse_relationships( + image['relationships'], image_map[image['image_id']])) + return output + +def get_all_qas(data_dir=None): + """ + Get all question answers. + """ + if data_dir is None: + data_dir = utils.get_data_dir() + data_file = os.path.join(data_dir, 'question_answers.json') + image_data = get_all_image_data(data_dir) + image_map = {} + for d in image_data: + image_map[d.id] = d + images = json.load(open(data_file)) + output = [] + for image in images: + output.append(utils.parse_QA(image['qas'], image_map)) + return output + + +# -------------------------------------------------------------------------------------------------- +# get_scene_graphs and sub-methods + + +def get_scene_graph(image_id, images='data/', + image_data_dir='data/by-id/', + synset_file='data/synsets.json'): + """ + Load a single scene graph from a .json file. + """ + if type(images) is str: + # Instead of a string, we can pass this dict as the argument `images` + images = {img.id: img for img in get_all_image_data(images)} + + fname = str(image_id) + '.json' + image = images[image_id] + data = json.load(open(image_data_dir + fname, 'r')) + + scene_graph = parse_graph_local(data, image) + scene_graph = init_synsets(scene_graph, synset_file) + return scene_graph + + +def get_scene_graphs(start_index=0, end_index=-1, + data_dir='data/', image_data_dir='data/by-id/', + min_rels=0, max_rels=100): + """ + Get scene graphs given locally stored .json files; + requires `save_scene_graphs_by_id`. + + start_index, end_index : get scene graphs listed by image, + from start_index through end_index + data_dir : directory with `image_data.json` and `synsets.json` + image_data_dir : directory of scene graph jsons saved by image id + (see `save_scene_graphs_by_id`) + min_rels, max_rels: only get scene graphs with at least / less + than this number of relationships + """ + images = {img.id: img for img in get_all_image_data(data_dir)} + scene_graphs = [] + + img_fnames = os.listdir(image_data_dir) + if (end_index < 1): + end_index = len(img_fnames) + + bar = progressbar.ProgressBar() + for fname in bar(img_fnames[start_index: end_index]): + image_id = int(fname.split('.')[0]) + scene_graph = get_scene_graph( + image_id, images, image_data_dir, data_dir + 'synsets.json') + n_rels = len(scene_graph.relationships) + if (min_rels <= n_rels <= max_rels): + scene_graphs.append(scene_graph) + + return scene_graphs + + +def map_object(object_map, obj): + """ + Use object ids as hashes to `visual_genome.models.Object` instances. + If item not in table, create new `Object`. Used when building + scene graphs from json. + """ + + oid = obj['object_id'] + obj['id'] = oid + del obj['object_id'] + + if oid in object_map: + object_ = object_map[oid] + + else: + if 'attributes' in obj: + attrs = obj['attributes'] + del obj['attributes'] + else: + attrs = [] + if 'w' in obj: + obj['width'] = obj['w'] + obj['height'] = obj['h'] + del obj['w'], obj['h'] + + object_ = Object(**obj) + + object_.attributes = attrs + object_map[oid] = object_ + + return object_map, object_ + + +global count_skips +count_skips = [0, 0] + + +def parse_graph_local(data, image, verbose=False): + """ + Modified version of `utils.ParseGraph`. + """ + global count_skips + objects = [] + object_map = {} + relationships = [] + attributes = [] + + for obj in data['objects']: + object_map, o_ = map_object(object_map, obj) + objects.append(o_) + for rel in data['relationships']: + if rel['subject_id'] in object_map and rel['object_id'] in object_map: + object_map, s = map_object( + object_map, {'object_id': rel['subject_id']}) + v = rel['predicate'] + object_map, o = map_object( + object_map, {'object_id': rel['object_id']}) + rid = rel['relationship_id'] + relationships.append(Relationship(rid, s, v, o, rel['synsets'])) + else: + # Skip this relationship if we don't have the subject and object in + # the object_map for this scene graph. Some data is missing in this + # way. + count_skips[0] += 1 + if 'attributes' in data: + for attr in data['attributes']: + a = attr['attribute'] + if a['object_id'] in object_map: + attributes.append(Attribute(attr['attribute_id'], + Object(a['object_id'], a['x'], + a['y'], a['w'], a['h'], + a['names'], a['synsets']), + a['attributes'], a['synsets'])) + else: + count_skips[1] += 1 + if verbose: + print('Skipped {} rels, {} attrs total'.format(*count_skips)) + return Graph(image, objects, relationships, attributes) + + +def init_synsets(scene_graph, synset_file): + """ + Convert synsets in a scene graph from strings to Synset objects. + """ + syn_data = json.load(open(synset_file, 'r')) + syn_class = {s['synset_name']: Synset( + s['synset_name'], s['synset_definition']) for s in syn_data} + + for obj in scene_graph.objects: + obj.synsets = [syn_class[sn] for sn in obj.synsets] + for rel in scene_graph.relationships: + rel.synset = [syn_class[sn] for sn in rel.synset] + for attr in scene_graph.attributes: + obj.synset = [syn_class[sn] for sn in attr.synset] + + return scene_graph + + +# -------------------------------------------------------------------------------------------------- +# This is a pre-processing step that only needs to be executed once. +# You can download .jsons segmented with these methods from: +# https://drive.google.com/file/d/0Bygumy5BKFtcQ1JrcFpyQWdaQWM + + +def save_scene_graphs_by_id(data_dir='data/', image_data_dir='data/by-id/'): + """ + Save a separate .json file for each image id in `image_data_dir`. + + Notes + ----- + - If we don't save .json's by id, `scene_graphs.json` is >6G in RAM + - Separated .json files are ~1.1G on disk + - Run `add_attrs_to_scene_graphs` before `parse_graph_local` will work + - Attributes are only present in objects, and do not have synset info + + Each output .json has the following keys: + - "id" + - "objects" + - "relationships" + """ + if not os.path.exists(image_data_dir): + os.mkdir(image_data_dir) + + all_data = json.load(open(os.path.join(data_dir, 'scene_graphs.json'))) + bar = progressbar.ProgressBar() + for sg_data in bar(all_data): + img_fname = str(sg_data['image_id']) + '.json' + with open(os.path.join(image_data_dir, img_fname), 'w') as f: + json.dump(sg_data, f) + + del all_data + gc.collect() # clear memory + + +def add_attrs_to_scene_graphs(data_dir='data/'): + """ + Add attributes to `scene_graph.json`, extracted from `attributes.json`. + + This also adds a unique id to each attribute, and separates individual + attibutes for each object (these are grouped in `attributes.json`). + """ + attr_data = json.load(open(os.path.join(data_dir, 'attributes.json'))) + with open(os.path.join(data_dir, 'scene_graphs.json'), 'r') as f: + sg_dict = {sg['image_id']: sg for sg in json.load(f)} + + id_count = 0 + bar = progressbar.ProgressBar() + for img_attrs in bar(attr_data): + attrs = [] + for attribute in img_attrs['attributes']: + a = img_attrs.copy() + del a['attributes'] + a['attribute'] = attribute + a['attribute_id'] = id_count + attrs.append(a) + id_count += 1 + iid = img_attrs['image_id'] + sg_dict[iid]['attributes'] = attrs + + with open(os.path.join(data_dir, 'scene_graphs.json'), 'w') as f: + json.dump(list(sg_dict.values()), f) + del attr_data, sg_dict + gc.collect() + + +# -------------------------------------------------------------------------------------------------- +# For info on VRD dataset, see: +# http://cs.stanford.edu/people/ranjaykrishna/vrd/ + +def get_scene_graphs_VRD(json_file='data/vrd/json/test.json'): + """ + Load VRD dataset into scene graph format. + """ + scene_graphs = [] + with open(json_file, 'r') as f: + D = json.load(f) + + scene_graphs = [parse_graph_VRD(d) for d in D] + return scene_graphs + + +def parse_graph_VRD(d): + image = Image(d['photo_id'], d['filename'], d[ + 'width'], d['height'], '', '') + + id2obj = {} + objs = [] + rels = [] + atrs = [] + + for i, o in enumerate(d['objects']): + b = o['bbox'] + obj = Object(i, b['x'], b['y'], b['w'], b['h'], o['names'], []) + id2obj[i] = obj + objs.append(obj) + + for j, a in enumerate(o['attributes']): + atrs.append(Attribute(j, obj, a['attribute'], [])) + + for i, r in enumerate(d['relationships']): + s = id2obj[r['objects'][0]] + o = id2obj[r['objects'][1]] + v = r['relationship'] + rels.append(Relationship(i, s, v, o, [])) + + return Graph(image, objs, rels, atrs) diff --git a/models.py b/models.py new file mode 100644 index 0000000..9ed4991 --- /dev/null +++ b/models.py @@ -0,0 +1,238 @@ +""" +Visual Genome Python API wrapper, models +""" + + +class Image: + """ + Image. + ID int + url hyperlink string + width int + height int + """ + + def __init__(self, id, url, width, height, coco_id, flickr_id): + self.id = id + self.url = url + self.width = width + self.height = height + self.coco_id = coco_id + self.flickr_id = flickr_id + + def __str__(self): + return 'id: %d, coco_id: %d, flickr_id: %d, width: %d, url: %s' \ + % (self.id, -1 + if self.coco_id is None + else self.coco_id, -1 + if self.flickr_id is None + else self.flickr_id, self.width, self.url) + + def __repr__(self): + return str(self) + + +class Region: + """ + Region. + image int + phrase string + x int + y int + width int + height int + """ + + def __init__(self, id, image, phrase, x, y, width, height): + self.id = id + self.image = image + self.phrase = phrase + self.x = x + self.y = y + self.width = width + self.height = height + + def __str__(self): + stat_str = 'id: {0}, x: {1}, y: {2}, width: {3},' \ + 'height: {4}, phrase: {5}, image: {6}' + return stat_str.format(self.id, self.x, self.y, + self.width, self.height, self.phrase, + self.image.id) + + def __repr__(self): + return str(self) + + +class Graph: + """ + Graphs contain objects, relationships and attributes + image Image + bboxes Object array + relationships Relationship array + attributes Attribute array + """ + + def __init__(self, image, objects, relationships, attributes): + self.image = image + self.objects = objects + self.relationships = relationships + self.attributes = attributes + + +class Object: + """ + Objects. + id int + x int + y int + width int + height int + names string array + synsets Synset array + """ + + def __init__(self, id, x, y, width, height, names, synsets): + self.id = id + self.x = x + self.y = y + self.width = width + self.height = height + self.names = names + self.synsets = synsets + + def __str__(self): + name = self.names[0] if len(self.names) != 0 else 'None' + return '%s' % (name) + + def __repr__(self): + return str(self) + + +class Relationship: + """ + Relationships. Ex, 'man - jumping over - fire hydrant'. + subject int + predicate string + object int + phrase string + rel_canon Synset + image int + """ + + def __init__(self, id, image, subject, predicate, object, synsets): + self.id = id + self.image = image + self.subject = subject + self.predicate = predicate + self.object = object + self.synsets = synsets + if self.object.get('names')!=None: + object1=''.join(self.object.get('names')) + else: + object1=self.object.get('name') + if self.subject.get('names')!=None: + subject=''.join(self.subject.get('names')) + else: + subject=self.subject.get('name') + #predicate=self.predicate + self.phrase=subject+' '+self.predicate.lower()+' '+object1 + + + + def __str__(self): + stat_str = 'id: {0}, subject: {1}, object: {2}, predicate: {3},' \ + ' phrase: {4}, image: {5}' + return stat_str.format(self.id, self.subject,self.predicate, self.object,self.phrase,self.image.id) + + + def __repr__(self): + return str(self) + + +class Attribute: + """ + Attributes. Ex, 'man - old'. + subject Object + attribute string + synset Synset + """ + + def __init__(self, id, image,subject,w,h,x,y, attribute, synset): + self.id = id + self.image=image + self.subject = ''.join(subject) + self.attribute = ', '.join(attribute) + self.synset = synset + + def __str__(self): + return "%d: %s is %s" % (self.id, self.subject, self.attribute) + + def __repr__(self): + return str(self) + + +class QA: + """ + Question Answer Pairs. + ID int + image int + question string + answer string + q_objects QAObject array + a_objects QAObject array + """ + + def __init__(self, id, image, question, answer, + question_objects, answer_objects): + self.id = id + self.image = image + self.question = question + self.answer = answer + self.q_objects = question_objects + self.a_objects = answer_objects + + def __str__(self): + return 'id: %d, image: %d, question: %s, answer: %s' \ + % (self.id, self.image.id, self.question, self.answer) + + def __repr__(self): + return str(self) + + +class QAObject: + """ + Question Answer Objects are localized in the image and refer to a part + of the question text or the answer text. + start_idx int + end_idx int + name string + synset_name string + synset_definition string + """ + + def __init__(self, start_idx, end_idx, name, synset): + self.start_idx = start_idx + self.end_idx = end_idx + self.name = name + self.synset = synset + + def __repr__(self): + return str(self) + + +class Synset: + """ + Wordnet Synsets. + name string + definition string + """ + + def __init__(self, name, definition): + self.name = name + self.definition = definition + + def __str__(self): + return '{} - {}'.format(self.name, self.definition) + + def __repr__(self): + return str(self) diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..f45ff40 --- /dev/null +++ b/utils.py @@ -0,0 +1,147 @@ + +import json +import requests +from os.path import dirname, realpath, join +from visual_genome.models import (Image, Object, Attribute, Relationship, + Region, Graph, QA, QAObject, Synset) + + +def get_data_dir(): + """ + Get the local directory where the Visual Genome data is locally stored. + """ + data_dir = join(dirname(realpath('__file__')), 'data') + return data_dir + + +def retrieve_data(request): + """ + Helper Method used to get all data from request string. + """ + url = 'http://visualgenome.org' + data = requests.get(url + request).json() + # connection = httplib.HTTPSConnection("visualgenome.org", '443') + # connection.request("GET", request) + # response = connection.getresponse() + # jsonString = response.read() + # data = json.loads(json_string) + return data + + +def parse_synset(canon): + """ + Helper to Extract Synset from canon object. + """ + if len(canon) == 0: + return None + return Synset(canon[0]['synset_name'], canon[0]['synset_definition']) + + +def parse_graph(data, image): + """ + Helper to parse a Graph object from API data. + """ + objects = [] + object_map = {} + relationships = [] + attributes = [] + # Create the Objects + for obj in data['bounding_boxes']: + names = [] + synsets = [] + for bbx_obj in obj['boxed_objects']: + names.append(bbx_obj['name']) + synsets.append(parse_synset(bbx_obj['object_canon'])) + object_ = Object(obj['id'], obj['x'], obj['y'], obj[ + 'width'], obj['height'], names, synsets) + object_map[obj['id']] = object_ + objects.append(object_) + # Create the Relationships + for rel in data['relationships']: + relationships.append(Relationship(rel['id'], + object_map[rel['subject']], + rel['predicate'], + object_map[rel['object']], + parse_synset( + rel['relationship_canon']))) + # Create the Attributes + for atr in data['attributes']: + attributes.append(Attribute(atr['id'], object_map[atr['subject']], + atr['attribute'], + parse_synset(atr['attribute_canon']))) + return Graph(image, objects, relationships, attributes) + + +def parse_image_data(data): + """ + Helper to parse the image data for one image. + """ + img_id = data['id'] if 'id' in data else data['image_id'] + url = data['url'] + width = data['width'] + height = data['height'] + coco_id = data['coco_id'] + flickr_id = data['flickr_id'] + image = Image(img_id, url, width, height, coco_id, flickr_id) + return image + + +def parse_region_descriptions(data, image): + """ + Helper to parse region descriptions. + """ + regions = [] + if 'region_id' in data[0]: + region_id_key = 'region_id' + else: + region_id_key = 'id' + for info in data: + regions.append(Region(info[region_id_key], image, info['phrase'], + info['x'], info['y'], info['width'], + info['height'])) + return regions + + + +def parse_relationships(data,image): + """ + Helper to parse relationships. + """ + relationships = [] + for info in data: + relationships.append(Relationship(info['relationship_id'], image, info['subject'], + info['predicate'], info['object'],info['synsets'])) + return relationships + +def parse_attributes(data,image): + """ + Helper to parse attributes. + """ + attributes = [] + for info in data: + attributes.append(Attribute(info['object_id'], image, info['names'],info['w'],info['h'], + info['x'],info['y'],info['attributes'] if 'attributes' in info else '',info['synsets'])) + return attributes + + +def parse_QA(data, image_map): + """ + Helper to parse a list of question answers. + """ + qas = [] + for info in data: + qos = [] + aos = [] + if 'question_objects' in info: + for qo in info['question_objects']: + synset = Synset(qo['synset_name'], qo['synset_definition']) + qos.append(QAObject(qo['entity_idx_start'], qo[ + 'entity_idx_end'], qo['entity_name'], synset)) + if 'answer_objects' in info: + for ao in info['answer_objects']: + synset = Synset(ao['synset_name'], ao['synset_definition']) + aos.append(QAObject(ao['entity_idx_start'], ao[ + 'entity_idx_end'], ao['entity_name'], synset)) + qas.append(QA(info['qa_id'], image_map[info['image_id']], + info['question'], info['answer'], qos, aos)) + return qas From f2ea21414a787185e49f01bebd4d6a55e0cec744 Mon Sep 17 00:00:00 2001 From: Qiren Sun Date: Tue, 21 May 2019 10:46:26 -0400 Subject: [PATCH 2/5] Add files via upload --- models.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ utils.py | 4 ++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/models.py b/models.py index 9ed4991..740bfa5 100644 --- a/models.py +++ b/models.py @@ -109,6 +109,29 @@ def __repr__(self): class Relationship: + """ + Relationships. Ex, 'man - jumping over - fire hydrant'. + subject int + predicate string + object int + rel_canon Synset + """ + + def __init__(self, id, subject, predicate, object, synset): + self.id = id + self.subject = subject + self.predicate = predicate + self.object = object + self.synset = synset + + def __str__(self): + return "{0}: {1} {2} {3}".format(self.id, self.subject, + self.predicate, self.object) + + def __repr__(self): + return str(self) + +class Relationships: """ Relationships. Ex, 'man - jumping over - fire hydrant'. subject int @@ -157,6 +180,27 @@ class Attribute: synset Synset """ + def __init__(self, id, subject, attribute, synset): + self.id = id + self.subject = subject + self.attribute = attribute + self.synset = synset + + def __str__(self): + return "%d: %s is %s" % (self.id, self.subject, self.attribute) + + def __repr__(self): + return str(self) + + +class Attributes: + """ + Attributes. Ex, 'man - old'. + subject Object + attribute string + synset Synset + """ + def __init__(self, id, image,subject,w,h,x,y, attribute, synset): self.id = id self.image=image diff --git a/utils.py b/utils.py index f45ff40..bf606fe 100644 --- a/utils.py +++ b/utils.py @@ -109,7 +109,7 @@ def parse_relationships(data,image): """ relationships = [] for info in data: - relationships.append(Relationship(info['relationship_id'], image, info['subject'], + relationships.append(Relationships(info['relationship_id'], image, info['subject'], info['predicate'], info['object'],info['synsets'])) return relationships @@ -119,7 +119,7 @@ def parse_attributes(data,image): """ attributes = [] for info in data: - attributes.append(Attribute(info['object_id'], image, info['names'],info['w'],info['h'], + attributes.append(Attributes(info['object_id'], image, info['names'],info['w'],info['h'], info['x'],info['y'],info['attributes'] if 'attributes' in info else '',info['synsets'])) return attributes From bb5dd8722b32bdafc70f85d369bfe0711cabdb4c Mon Sep 17 00:00:00 2001 From: Qiren Sun Date: Tue, 21 May 2019 10:48:12 -0400 Subject: [PATCH 3/5] Add files via upload From 02e6c64ec082b268cc46918ee6136bc5354bad90 Mon Sep 17 00:00:00 2001 From: Qiren Sun Date: Tue, 21 May 2019 10:48:53 -0400 Subject: [PATCH 4/5] Add files via upload From 4295b05e7818eecc8dd5f725595ce25328399958 Mon Sep 17 00:00:00 2001 From: Qiren Sun Date: Wed, 22 May 2019 09:29:34 -0400 Subject: [PATCH 5/5] Add files via upload --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index bf606fe..b7d4fa8 100644 --- a/utils.py +++ b/utils.py @@ -3,7 +3,7 @@ import requests from os.path import dirname, realpath, join from visual_genome.models import (Image, Object, Attribute, Relationship, - Region, Graph, QA, QAObject, Synset) + Region, Graph, QA, QAObject, Synset,Attributes,Relationships) def get_data_dir():