Skip to content

Instantly share code, notes, and snippets.

@irskep
Last active August 29, 2015 13:58
Show Gist options
  • Save irskep/10024667 to your computer and use it in GitHub Desktop.
Save irskep/10024667 to your computer and use it in GitHub Desktop.
def run_job(self):
"""Run the all steps of the job, logging errors (and debugging output
if :option:`--verbose` is specified) to STDERR and streaming the
output to STDOUT.
Called from :py:meth:`run`. You'd probably only want to call this
directly from automated tests.
"""
self.set_up_logging(quiet=self.options.quiet,
verbose=self.options.verbose,
stream=self.stderr)
with self.make_runner() as runner:
runner.run()
if self.post_process is overridden by subclass:
self.post_process(runner, runner.stream_output())
elif not self.options.no_output:
for line in runner.stream_output():
self.stdout.write(line)
self.stdout.flush()
class StevesJobSteveJobs(MRJob):
def post_process(self, runner, lines):
db = connect_to_sqlalchemy()
for line in lines:
parsed_line = self.parse_output_line(line)
db.add(MyThing(id=line['id'], value=line['value']))
db.commit()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment