edglazer · July 1, 2016 18:49
diff --git a/tribe-weight-links-count.gfm b/tribe-weight-links-count.gfm
 Timer unit: 1e-06 s

 Total time: 1401.2 s
 File: tribe/extract.py
 Function: extract_graph at line 105

 Line #      Hits         Time  Per Hit   % Time  Line Contents
 ==============================================================
   105                                               @profile
   106                                               def extract_graph(self):
   107                                                   """
   108                                                   Extracts a Graph where the nodes are EmailAddress
   109                                                   """
   110
   111         1            3      3.0      0.0          def relationships(email):
   112                                                       """
   113                                                       Inner function that constructs email relationships
   114                                                       """
   115                                                       people = [email.sender,]
   116                                                       people.extend(email.recipients)
   117                                                       people.extend(email.copied)
   118
   119                                                       people = filter(lambda p: p is not None, people)            # Filter out any None addresses
   120                                                       people = set(addr.email for addr in people if addr.email)   # Obtain only unique people
   121                                                       people = sorted(people)                                     # Sort lexicographically for combinations
   122
   123                                                       for combo in combinations(people, 2):
   124                                                           yield combo
   125
   126
   127                                                   # Keep track of all the email to email links
   128         1           18     18.0      0.0          links = FreqDist()
   129
   130                                                   # Iterate over all the extracted emails
   131                                                   # Catch exceptions, if any, and move forward
   132                                                   # NOTE: This will allow the progress bar to work
   133                                                   # NOTE: This will build the graph data structure in memory
   134    114956   1159836727  10089.4     82.8          for email in self.extract():
   135    114955       136897      1.2      0.0              try:
   136   4870025      7657826      1.6      0.5                  for combo in relationships(email):
   137   4755070      7686760      1.6      0.5                      links[combo] += 1
   138                                                       except Exception as e:
   139                                                           self.errors[e] += 1
   140                                                           continue
   141
   142                                                   # Construct the networkx graph and add edges
   143         1           66     66.0      0.0          G = nx.Graph(name="Email Network", mbox=self.path, extracted=strfnow())
   144      4173        67408     16.2      0.0          for link in links.keys():
   145      4173    225812988  54112.9     16.1              G.add_edge(*link, weight=links.freq(link))
   146
   147                                                   # Return the generated graph
   148                                                   return G

 ➜  tribe git:(master) ✗  pip install -U memory_profiler
 Requirement already up-to-date: memory_profiler in /usr/local/lib/python2.7/site-packages
 ➜  tribe git:(master) ✗ pip install psutil
 Requirement already satisfied (use --upgrade to upgrade): psutil in /usr/local/lib/python2.7/site-packages
 ➜  tribe git:(master) ✗ pip install -U psutil
 Requirement already up-to-date: psutil in /usr/local/lib/python2.7/site-packages
 ➜  tribe git:(master) ✗ kernprof --help
 Usage: kernprof [-s setupfile] [-o output_file_path] scriptfile [arg] ...

 Options:
  --version             show program's version number and exit
  -h, --help            show this help message and exit
  -l, --line-by-line    Use the line-by-line profiler from the line_profiler
                        module instead of Profile. Implies --builtin.
  -b, --builtin         Put 'profile' in the builtins. Use 'profile.enable()'
                        and 'profile.disable()' in your code to turn it on and
                        off, or '@profile' to decorate a single function, or
                        'with profile:' to profile a single section of code.
  -o OUTFILE, --outfile=OUTFILE
                        Save stats to <outfile>
  -s SETUP, --setup=SETUP
                        Code to execute before the code to profile
  -v, --view            View the results of the profile in addition to saving
                        it.
 ➜  tribe git:(master) ✗ python -m memory_profiler tribe-admin.py extract -w myemails.graphml allmail.mbox
 Starting Graph extraction, a long running process
 Initializing MBox iteration on allmail.mbox (8.8GiB)
 ^CFilename: tribe/extract.py seconds | Parsed: 97431 emails

 Line #    Mem usage    Increment   Line Contents
 ================================================
   105   70.691 MiB    0.000 MiB       @profile
   106                                 def extract_graph(self):
   107                                     """
   108                                     Extracts a Graph where the nodes are EmailAddress
   109                                     """
   110
   111  490.387 MiB  419.695 MiB           def relationships(email):
   112                                         """
   113                                         Inner function that constructs email relationships
   114                                         """
   115  490.387 MiB    0.000 MiB               people = [email.sender,]
   116  490.387 MiB    0.000 MiB               people.extend(email.recipients)
   117  490.387 MiB    0.000 MiB               people.extend(email.copied)
   118
   119  490.387 MiB    0.000 MiB               people = filter(lambda p: p is not None, people)            # Filter out any None addresses
   120  490.387 MiB    0.000 MiB               people = set(addr.email for addr in people if addr.email)   # Obtain only unique people
   121  490.387 MiB    0.000 MiB               people = sorted(people)                                     # Sort lexicographically for combinations
   122
   123  490.387 MiB    0.000 MiB               for combo in combinations(people, 2):
   124  490.387 MiB    0.000 MiB                   yield combo
   125
   126
   127                                     # Keep track of all the email to email links
   128   70.691 MiB -419.695 MiB           links = FreqDist()
   129
   130                                     # Iterate over all the extracted emails
   131                                     # Catch exceptions, if any, and move forward
   132                                     # NOTE: This will allow the progress bar to work
   133                                     # NOTE: This will build the graph data structure in memory
   134  490.387 MiB  419.695 MiB           for email in self.extract():
   135  490.387 MiB    0.000 MiB               try:
   136  490.387 MiB    0.000 MiB                   for combo in relationships(email):
   137  490.387 MiB    0.000 MiB                       links[combo] += 1
   138                                         except Exception as e:
   139                                             self.errors[e] += 1
   140                                             continue
   141
   142                                     # Construct the networkx graph and add edges
   143                                     G = nx.Graph(name="Email Network", mbox=self.path, extracted=strfnow())
   144                                     for link in links.keys():
   145                                         G.add_edge(*link, weight=links.freq(link))
   146
   147                                     # Return the generated graph
   148                                     return G


 Traceback (most recent call last):
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py", line 162, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py", line 72, in _run_code
    exec code in run_globals
  File "/usr/local/lib/python2.7/site-packages/memory_profiler.py", line 982, in <module>
    exec_with_profiler(script_filename, prof)
  File "/usr/local/lib/python2.7/site-packages/memory_profiler.py", line 917, in exec_with_profiler
    execfile(filename, ns, ns)
  File "tribe-admin.py", line 179, in <module>
    main(*sys.argv[1:])
  File "tribe-admin.py", line 173, in main
    msg = args.func(args)             # Call the default function
  File "tribe-admin.py", line 91, in extract
    errors, seconds = timed_inner(args.mbox[0], args.write)
  File "tribe/utils.py", line 118, in wrapper
    result = func(*args, **kwargs)
  File "tribe-admin.py", line 86, in timed_inner
    G = reader.extract_graph()
  File "/usr/local/lib/python2.7/site-packages/memory_profiler.py", line 498, in f
    return func(*args, **kwds)
  File "tribe/extract.py", line 134, in extract_graph
    for email in self.extract():
  File "tribe/extract.py", line 97, in extract
    for msg in self:
  File "tribe/extract.py", line 177, in __iter__
    for msg in super(ConsoleMBoxReader, self).__iter__():
  File "tribe/extract.py", line 46, in __iter__
    for msg in self.mbox:
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/mailbox.py", line 108, in itervalues
    value = self[key]
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/mailbox.py", line 80, in __getitem__
    return self.get_message(key)
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/mailbox.py", line 775, in get_message
    msg = self._message_factory(string.replace(os.linesep, '\n'))
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/mailbox.py", line 1598, in __init__
    Message.__init__(self, message)
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/mailbox.py", line 1459, in __init__
    self._become_message(email.message_from_string(message))
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/__init__.py", line 57, in message_from_string
    return Parser(*args, **kws).parsestr(s)
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/parser.py", line 82, in parsestr
    return self.parse(StringIO(text), headersonly=headersonly)
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/parser.py", line 71, in parse
    feedparser.feed(data)
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/feedparser.py", line 178, in feed
    self._call_parse()
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/feedparser.py", line 182, in _call_parse
    self._parse()
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/feedparser.py", line 373, in _parsegen
    for retval in self._parsegen():
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/feedparser.py", line 446, in _parsegen
    if line is NeedMoreData:
  File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/feedparser.py", line 446, in _parsegen
    if line is NeedMoreData:
  File "/usr/local/lib/python2.7/site-packages/memory_profiler.py", line 537, in trace_memory_usage
    def trace_memory_usage(self, frame, event, arg):
 KeyboardInterrupt
 Exception KeyboardInterrupt in <module 'threading' from '/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.pyc'> ignored
 ^X%                                                                                                                                                                                 ➜  tribe git:(master) ✗
 ➜  tribe git:(master) ✗ kernprof -l tribe-admin.py extract -w myemails.graphml allmail.mbox
 Starting Graph extraction, a long running process
 Initializing MBox iteration on allmail.mbox (8.8GiB)
 ^CWrote profile results to tribe-admin.py.lprof.
 Elapsed: 14 minutes 41 seconds | initializing ...

 Session Restored

 Last login: Fri Jul  1 11:44:11 on console
 You have new mail.
 ➜  tribe git:(master) ✗ kernprof -l tribe-admin.py extract -w myemails.graphml allmail.mbox
 Starting Graph extraction, a long running process
 Initializing MBox iteration on allmail.mbox (8.8GiB)
 Elapsed: 18 minutes 35 seconds | Parsed: 114856 emails
 GraphML written out to myemails.graphml

 No errors encountered in processing

 Graph extraction took 19 minutes 44 seconds
 Wrote profile results to tribe-admin.py.lprof
 ➜  tribe git:(master) ✗ python -m line_profiler tribe-admin.py.lprof
 Timer unit: 1e-06 s

 Total time: 1119.33 s
 File: tribe/extract.py
 Function: extract_graph at line 105

 Line #      Hits         Time  Per Hit   % Time  Line Contents
 ==============================================================
   105                                               @profile
   106                                               def extract_graph(self):
   107                                                   """
   108                                                   Extracts a Graph where the nodes are EmailAddress
   109                                                   """
   110
   111         1           13     13.0      0.0          def relationships(email):
   112                                                       """
   113                                                       Inner function that constructs email relationships
   114                                                       """
   115                                                       people = [email.sender,]
   116                                                       people.extend(email.recipients)
   117                                                       people.extend(email.copied)
   118
   119                                                       people = filter(lambda p: p is not None, people)            # Filter out any None addresses
   120                                                       people = set(addr.email for addr in people if addr.email)   # Obtain only unique people
   121                                                       people = sorted(people)                                     # Sort lexicographically for combinations
   122
   123                                                       for combo in combinations(people, 2):
   124                                                           yield combo
   125
   126
   127                                                   # Keep track of all the email to email links
   128         1            8      8.0      0.0          links = FreqDist()
   129
   130                                                   # Iterate over all the extracted emails
   131                                                   # Catch exceptions, if any, and move forward
   132                                                   # NOTE: This will allow the progress bar to work
   133                                                   # NOTE: This will build the graph data structure in memory
   134    114956   1094585725   9521.8     97.8          for email in self.extract():
   135    114955       116953      1.0      0.0              try:
   136   4870025      7024773      1.4      0.6                  for combo in relationships(email):
   137   4755070      7022225      1.5      0.6                      links[combo] += 1
   138                                                       except Exception as e:
   139                                                           self.errors[e] += 1
   140                                                           continue
   141
   142                                                   # Construct the networkx graph and add edges
   143         1           68     68.0      0.0          G = nx.Graph(name="Email Network", mbox=self.path, extracted=strfnow())
   144   1837532      1518627      0.8      0.1          for link in links.keys():
   145                                                       G.add_edge(*link, weight=links.freq(link))  # took out Changed per suggestion of @bbengfort, as calcultion may be too strenuous
   146   1837531      9065502      4.9      0.8  #            G.add_edge(*link, weight=links[link])
   147
   148                                                   # Return the generated graph
   149         1            3      3.0      0.0          return G
	Timer unit: 1e-06 s

	Total time: 1401.2 s
	File: tribe/extract.py
	Function: extract_graph at line 105

	Line # Hits Time Per Hit % Time Line Contents
	==============================================================
	105 @profile
	106 def extract_graph(self):
	107 """
	108 Extracts a Graph where the nodes are EmailAddress
	109 """
	110
	111 1 3 3.0 0.0 def relationships(email):
	112 """
	113 Inner function that constructs email relationships
	114 """
	115 people = [email.sender,]
	116 people.extend(email.recipients)
	117 people.extend(email.copied)
	118
	119 people = filter(lambda p: p is not None, people) # Filter out any None addresses
	120 people = set(addr.email for addr in people if addr.email) # Obtain only unique people
	121 people = sorted(people) # Sort lexicographically for combinations
	122
	123 for combo in combinations(people, 2):
	124 yield combo
	125
	126
	127 # Keep track of all the email to email links
	128 1 18 18.0 0.0 links = FreqDist()
	129
	130 # Iterate over all the extracted emails
	131 # Catch exceptions, if any, and move forward
	132 # NOTE: This will allow the progress bar to work
	133 # NOTE: This will build the graph data structure in memory
	134 114956 1159836727 10089.4 82.8 for email in self.extract():
	135 114955 136897 1.2 0.0 try:
	136 4870025 7657826 1.6 0.5 for combo in relationships(email):
	137 4755070 7686760 1.6 0.5 links[combo] += 1
	138 except Exception as e:
	139 self.errors[e] += 1
	140 continue
	141
	142 # Construct the networkx graph and add edges
	143 1 66 66.0 0.0 G = nx.Graph(name="Email Network", mbox=self.path, extracted=strfnow())
	144 4173 67408 16.2 0.0 for link in links.keys():
	145 4173 225812988 54112.9 16.1 G.add_edge(*link, weight=links.freq(link))
	146
	147 # Return the generated graph
	148 return G

	➜ tribe git:(master) ✗ pip install -U memory_profiler
	Requirement already up-to-date: memory_profiler in /usr/local/lib/python2.7/site-packages
	➜ tribe git:(master) ✗ pip install psutil
	Requirement already satisfied (use --upgrade to upgrade): psutil in /usr/local/lib/python2.7/site-packages
	➜ tribe git:(master) ✗ pip install -U psutil
	Requirement already up-to-date: psutil in /usr/local/lib/python2.7/site-packages
	➜ tribe git:(master) ✗ kernprof --help
	Usage: kernprof [-s setupfile] [-o output_file_path] scriptfile [arg] ...

	Options:
	--version show program's version number and exit
	-h, --help show this help message and exit
	-l, --line-by-line Use the line-by-line profiler from the line_profiler
	module instead of Profile. Implies --builtin.
	-b, --builtin Put 'profile' in the builtins. Use 'profile.enable()'
	and 'profile.disable()' in your code to turn it on and
	off, or '@profile' to decorate a single function, or
	'with profile:' to profile a single section of code.
	-o OUTFILE, --outfile=OUTFILE
	Save stats to <outfile>
	-s SETUP, --setup=SETUP
	Code to execute before the code to profile
	-v, --view View the results of the profile in addition to saving
	it.
	➜ tribe git:(master) ✗ python -m memory_profiler tribe-admin.py extract -w myemails.graphml allmail.mbox
	Starting Graph extraction, a long running process
	Initializing MBox iteration on allmail.mbox (8.8GiB)
	^CFilename: tribe/extract.py seconds \| Parsed: 97431 emails

	Line # Mem usage Increment Line Contents
	================================================
	105 70.691 MiB 0.000 MiB @profile
	106 def extract_graph(self):
	107 """
	108 Extracts a Graph where the nodes are EmailAddress
	109 """
	110
	111 490.387 MiB 419.695 MiB def relationships(email):
	112 """
	113 Inner function that constructs email relationships
	114 """
	115 490.387 MiB 0.000 MiB people = [email.sender,]
	116 490.387 MiB 0.000 MiB people.extend(email.recipients)
	117 490.387 MiB 0.000 MiB people.extend(email.copied)
	118
	119 490.387 MiB 0.000 MiB people = filter(lambda p: p is not None, people) # Filter out any None addresses
	120 490.387 MiB 0.000 MiB people = set(addr.email for addr in people if addr.email) # Obtain only unique people
	121 490.387 MiB 0.000 MiB people = sorted(people) # Sort lexicographically for combinations
	122
	123 490.387 MiB 0.000 MiB for combo in combinations(people, 2):
	124 490.387 MiB 0.000 MiB yield combo
	125
	126
	127 # Keep track of all the email to email links
	128 70.691 MiB -419.695 MiB links = FreqDist()
	129
	130 # Iterate over all the extracted emails
	131 # Catch exceptions, if any, and move forward
	132 # NOTE: This will allow the progress bar to work
	133 # NOTE: This will build the graph data structure in memory
	134 490.387 MiB 419.695 MiB for email in self.extract():
	135 490.387 MiB 0.000 MiB try:
	136 490.387 MiB 0.000 MiB for combo in relationships(email):
	137 490.387 MiB 0.000 MiB links[combo] += 1
	138 except Exception as e:
	139 self.errors[e] += 1
	140 continue
	141
	142 # Construct the networkx graph and add edges
	143 G = nx.Graph(name="Email Network", mbox=self.path, extracted=strfnow())
	144 for link in links.keys():
	145 G.add_edge(*link, weight=links.freq(link))
	146
	147 # Return the generated graph
	148 return G


	Traceback (most recent call last):
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py", line 162, in _run_module_as_main
	"__main__", fname, loader, pkg_name)
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py", line 72, in _run_code
	exec code in run_globals
	File "/usr/local/lib/python2.7/site-packages/memory_profiler.py", line 982, in <module>
	exec_with_profiler(script_filename, prof)
	File "/usr/local/lib/python2.7/site-packages/memory_profiler.py", line 917, in exec_with_profiler
	execfile(filename, ns, ns)
	File "tribe-admin.py", line 179, in <module>
	main(*sys.argv[1:])
	File "tribe-admin.py", line 173, in main
	msg = args.func(args) # Call the default function
	File "tribe-admin.py", line 91, in extract
	errors, seconds = timed_inner(args.mbox[0], args.write)
	File "tribe/utils.py", line 118, in wrapper
	result = func(args, *kwargs)
	File "tribe-admin.py", line 86, in timed_inner
	G = reader.extract_graph()
	File "/usr/local/lib/python2.7/site-packages/memory_profiler.py", line 498, in f
	return func(args, *kwds)
	File "tribe/extract.py", line 134, in extract_graph
	for email in self.extract():
	File "tribe/extract.py", line 97, in extract
	for msg in self:
	File "tribe/extract.py", line 177, in __iter__
	for msg in super(ConsoleMBoxReader, self).__iter__():
	File "tribe/extract.py", line 46, in __iter__
	for msg in self.mbox:
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/mailbox.py", line 108, in itervalues
	value = self[key]
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/mailbox.py", line 80, in __getitem__
	return self.get_message(key)
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/mailbox.py", line 775, in get_message
	msg = self._message_factory(string.replace(os.linesep, '\n'))
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/mailbox.py", line 1598, in __init__
	Message.__init__(self, message)
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/mailbox.py", line 1459, in __init__
	self._become_message(email.message_from_string(message))
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/__init__.py", line 57, in message_from_string
	return Parser(args, *kws).parsestr(s)
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/parser.py", line 82, in parsestr
	return self.parse(StringIO(text), headersonly=headersonly)
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/parser.py", line 71, in parse
	feedparser.feed(data)
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/feedparser.py", line 178, in feed
	self._call_parse()
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/feedparser.py", line 182, in _call_parse
	self._parse()
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/feedparser.py", line 373, in _parsegen
	for retval in self._parsegen():
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/feedparser.py", line 446, in _parsegen
	if line is NeedMoreData:
	File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/email/feedparser.py", line 446, in _parsegen
	if line is NeedMoreData:
	File "/usr/local/lib/python2.7/site-packages/memory_profiler.py", line 537, in trace_memory_usage
	def trace_memory_usage(self, frame, event, arg):
	KeyboardInterrupt
	Exception KeyboardInterrupt in <module 'threading' from '/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.pyc'> ignored
	^X% ➜ tribe git:(master) ✗
	➜ tribe git:(master) ✗ kernprof -l tribe-admin.py extract -w myemails.graphml allmail.mbox
	Starting Graph extraction, a long running process
	Initializing MBox iteration on allmail.mbox (8.8GiB)
	^CWrote profile results to tribe-admin.py.lprof.
	Elapsed: 14 minutes 41 seconds \| initializing ...

	Session Restored

	Last login: Fri Jul 1 11:44:11 on console
	You have new mail.
	➜ tribe git:(master) ✗ kernprof -l tribe-admin.py extract -w myemails.graphml allmail.mbox
	Starting Graph extraction, a long running process
	Initializing MBox iteration on allmail.mbox (8.8GiB)
	Elapsed: 18 minutes 35 seconds \| Parsed: 114856 emails
	GraphML written out to myemails.graphml

	No errors encountered in processing

	Graph extraction took 19 minutes 44 seconds
	Wrote profile results to tribe-admin.py.lprof
	➜ tribe git:(master) ✗ python -m line_profiler tribe-admin.py.lprof
	Timer unit: 1e-06 s

	Total time: 1119.33 s
	File: tribe/extract.py
	Function: extract_graph at line 105

	Line # Hits Time Per Hit % Time Line Contents
	==============================================================
	105 @profile
	106 def extract_graph(self):
	107 """
	108 Extracts a Graph where the nodes are EmailAddress
	109 """
	110
	111 1 13 13.0 0.0 def relationships(email):
	112 """
	113 Inner function that constructs email relationships
	114 """
	115 people = [email.sender,]
	116 people.extend(email.recipients)
	117 people.extend(email.copied)
	118
	119 people = filter(lambda p: p is not None, people) # Filter out any None addresses
	120 people = set(addr.email for addr in people if addr.email) # Obtain only unique people
	121 people = sorted(people) # Sort lexicographically for combinations
	122
	123 for combo in combinations(people, 2):
	124 yield combo
	125
	126
	127 # Keep track of all the email to email links
	128 1 8 8.0 0.0 links = FreqDist()
	129
	130 # Iterate over all the extracted emails
	131 # Catch exceptions, if any, and move forward
	132 # NOTE: This will allow the progress bar to work
	133 # NOTE: This will build the graph data structure in memory
	134 114956 1094585725 9521.8 97.8 for email in self.extract():
	135 114955 116953 1.0 0.0 try:
	136 4870025 7024773 1.4 0.6 for combo in relationships(email):
	137 4755070 7022225 1.5 0.6 links[combo] += 1
	138 except Exception as e:
	139 self.errors[e] += 1
	140 continue
	141
	142 # Construct the networkx graph and add edges
	143 1 68 68.0 0.0 G = nx.Graph(name="Email Network", mbox=self.path, extracted=strfnow())
	144 1837532 1518627 0.8 0.1 for link in links.keys():
	145 G.add_edge(*link, weight=links.freq(link)) # took out Changed per suggestion of @bbengfort, as calcultion may be too strenuous
	146 1837531 9065502 4.9 0.8 # G.add_edge(*link, weight=links[link])
	147
	148 # Return the generated graph
	149 1 3 3.0 0.0 return G