From: Adam Nemet Date: Wed, 19 Jul 2017 22:04:59 +0000 (+0000) Subject: [opt-viewer] Reduce memory consumption by another 20-25% X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=b2613733b11115c14da51a0cfaf50c27a59cdbd9;p=llvm [opt-viewer] Reduce memory consumption by another 20-25% The Args field of the remark which consists of a list of mappings in YAML is translated into a list of (small) dicts on Python. An empty dict is 280 bytes on my system so we can save memory by using a tuple of tuples instead. Making a tuple of tuples rather than a list of tuples allows Args to be shared with the key of the remark. This is actually an even greater saving. (Keys are alive throughout the entire run in all_remarks.) Here are a few opt-stats runs with different input sizes while measuring heap usage with heapy. Avg remark size is simply estimated as heap-size / # of remarks: | # of files | 60 | 114 | 308 | 605 | 1370 | | # of remarks | 20K | 37K | 146K | 180K | 640K | | total file size (MB) | 22 | 51 | 219 | 202 | 1034 | |------------------------+------+------+------+------+------| | Avg remark size before | 4339 | 4792 | 4761 | 4096 | 4607 | | Avg remark size after | 3446 | 3641 | 3567 | 3146 | 3347 | | Rate | 0.79 | 0.76 | 0.75 | 0.77 | 0.73 | Differential Revision: https://reviews.llvm.org/D35611 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@308538 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/tools/opt-viewer/opt-diff.py b/tools/opt-viewer/opt-diff.py index f39432c8bc9..32d6e5afd4c 100755 --- a/tools/opt-viewer/opt-diff.py +++ b/tools/opt-viewer/opt-diff.py @@ -60,5 +60,10 @@ if __name__ == '__main__': r.Added = True for r in removed: r.Added = False + + result = added | removed + for r in result: + r.recover_yaml_structure() + with open(args.output, 'w') as stream: - yaml.dump_all(added | removed, stream) + yaml.dump_all(result, stream) diff --git a/tools/opt-viewer/opt-viewer.py b/tools/opt-viewer/opt-viewer.py index 69bcaedb766..b5c7c09547a 100755 --- a/tools/opt-viewer/opt-viewer.py +++ b/tools/opt-viewer/opt-viewer.py @@ -82,7 +82,8 @@ class SourceFileRenderer: inlining_context = r.DemangledFunctionName dl = context.caller_loc.get(r.Function) if dl: - link = optrecord.make_link(dl['File'], dl['Line'] - 2) + dl_dict = dict(list(dl)) + link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2) inlining_context = "{r.DemangledFunctionName}".format(**locals()) # Column is the number of characters *including* tabs, keep those and @@ -176,10 +177,11 @@ def map_remarks(all_remarks): for remark in optrecord.itervalues(all_remarks): if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined": for arg in remark.Args: - caller = arg.get('Caller') + arg_dict = dict(list(arg)) + caller = arg_dict.get('Caller') if caller: try: - context.caller_loc[caller] = arg['DebugLoc'] + context.caller_loc[caller] = arg_dict['DebugLoc'] except KeyError: pass diff --git a/tools/opt-viewer/optrecord.py b/tools/opt-viewer/optrecord.py index adc2016017c..3744c68af60 100644 --- a/tools/opt-viewer/optrecord.py +++ b/tools/opt-viewer/optrecord.py @@ -60,14 +60,19 @@ class Remark(yaml.YAMLObject): # Work-around for http://pyyaml.org/ticket/154. yaml_loader = Loader - def _intern_strings(self): + # Intern all strings since we have lot of duplication across filenames, + # remark text. + # + # Change Args from a list of dicts to a tuple of tuples. This saves + # memory in two ways. One, a small tuple is significantly smaller than a + # small dict. Two, using tuple instead of list allows Args to be directly + # used as part of the key (in Python only immutable types are hashable). + def _reduce_memory(self): self.Pass = intern(self.Pass) self.Name = intern(self.Name) self.Function = intern(self.Function) - # Intern key and value if string and recurse if value is a dictionary. - # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}] - def _intern_dict(old_dict): + def _reduce_memory_dict(old_dict): new_dict = dict() for (k, v) in old_dict.iteritems(): if type(k) is str: @@ -76,18 +81,33 @@ class Remark(yaml.YAMLObject): if type(v) is str: v = intern(v) elif type(v) is dict: - v = _intern_dict(v) + # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}] + v = _reduce_memory_dict(v) new_dict[k] = v - return new_dict + return tuple(new_dict.items()) - self.Args = [_intern_dict(arg_dict) for arg_dict in self.Args] + self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args]) + + # The inverse operation of the dictonary-related memory optimization in + # _reduce_memory_dict. E.g. + # (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}] + def recover_yaml_structure(self): + def tuple_to_dict(t): + d = dict() + for (k, v) in t: + if type(v) is tuple: + v = tuple_to_dict(v) + d[k] = v + return d + + self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args] def canonicalize(self): if not hasattr(self, 'Hotness'): self.Hotness = 0 if not hasattr(self, 'Args'): self.Args = [] - self._intern_strings() + self._reduce_memory() @property def File(self): @@ -114,7 +134,7 @@ class Remark(yaml.YAMLObject): return make_link(self.File, self.Line) def getArgString(self, mapping): - mapping = mapping.copy() + mapping = dict(list(mapping)) dl = mapping.get('DebugLoc') if dl: del mapping['DebugLoc'] @@ -126,8 +146,9 @@ class Remark(yaml.YAMLObject): value = cgi.escape(demangle(value)) if dl and key != 'Caller': + dl_dict = dict(list(dl)) return "{}".format( - make_link(dl['File'], dl['Line']), value) + make_link(dl_dict['File'], dl_dict['Line']), value) else: return value @@ -158,13 +179,8 @@ class Remark(yaml.YAMLObject): @property def key(self): - k = (self.__class__, self.PassWithDiffPrefix, self.Name, self.File, self.Line, self.Column, self.Function) - for arg in self.Args: - for (key, value) in iteritems(arg): - if type(value) is dict: - value = tuple(value.items()) - k += (key, value) - return k + return (self.__class__, self.PassWithDiffPrefix, self.Name, self.File, + self.Line, self.Column, self.Function, self.Args) def __hash__(self): return hash(self.key)