zipping the pigeon

gistfile1.py

### pigeon_push.py

# (...)

class Command(BaseCommand):
    """Push items in the ItemToPush queue."""
    help = __doc__

    def handle(self, *args, **options):

        rules = {}

        # First, ident/init all the configurations involved in this push

        for row in item_to_push_queue():
            rule_name = row.rule_name
            rule = REGISTRY[rule_name]
            if rule_name not in rules.keys():
                rule.initialize_push()
                rules['rule_name'] = rule

        # Then, iterate through the items to push

        for row in item_to_push_queue():
            logger.debug(u'processing row id=%s, rule_name=%s' %
                                                        (row.pk, row.rule_name))
            row.status = ItemToPush.STATUS.IN_PROGRESS
            row.save()

            rule_name = row.rule_name
            rule = rules[rule_name]
            rule.push_one(row)

        # Then, finalize the push for each relevant rule

        for rule in rules:
            rule.finalize_push()

#########################################################################

### configuration.py

class DefaultConfiguration:

# (...)

    def push_one(self, row):
        rule_name = self.name

        instance = row.content_object

        output_files = []

        # Build output
        try:
            output = self.output(instance)
        except Exception, e:
            message = u"Exception during output generation. "
            message += u'Exception ``%s`` raised: %s ' % (
                            e.__class__.__name__, e.message)
            row.status = ItemToPush.STATUS.OUTPUT_GENERATION_ERROR
            row.message = message
            row.save()
            logger.error(message, exc_info=True)
            raise

        # Validate output
        validation = True
        for validator in self.validators:
            try:
                validator(output)
                logger.debug('validation ``%s`` passed successfully'
                                                       % validator.__name__)
            except Exception, e:
                validation = False
                message = u"Validation ``%s`` failed ! " % validator.__name__
                message += u'Catched exception %s : %s' % (
                            e.__class__.__name__, e.message)
                row.status = ItemToPush.STATUS.VALIDATION_ERROR
                if row.message != None:
                    row.message += '\n' + message
                else:
                    row.message = message
                logger.error(message, exc_info=True)
                row.save()

        if not validation:  # if one validator did not pass we
                            # do no want to send the file
            logger.debug('the output was not validated')
            raise
        
        output_filename = self.get_output_filename(instance)
        
        # Get target directory
        target_directory = None
        try:
            target_directory = self.get_directory(instance)
        except Exception, e:
            message = u"Error during ``get_directory``. "
            message += u"%s: %s" % (
                            e.__class__.__name__, e.message)
            row = ItemToPush(rule_name=rule_name,
                             content_object=instance)
            row.status = ItemToPush.STATUS.GET_DIRECTORY_ERROR
            row.message = message
            row.save()
            logger.error(message, exc_info=True)
            raise

        # Build output file path for archiving
        output_directory = settings.CARRIER_PIGEON_OUTPUT_DIRECTORY
        output_directory += '/%s/' % rule_name
        output_directory += '/%s/' % target_directory

        # Create output directory if necessary
        if not os.path.exists(output_directory):
            os.makedirs(output_directory)
        output_path = '%s/%s' % (output_directory, output_filename)

        # Write output file
        f = open(output_path, 'w')
        f.write(output)
        f.close()
        output_files.append(output_path)

        return output_files

# (...)

class ArchivePusherConfiguration(DefaultConfiguration):
    """
    Configurations inheriting this class will first archive all their files
    into a ZIP archive and _then_ send this archive onto the destination server.
    """

    rows = []

    def get_items_to_push(self):
        now = datetime.now()
        yesterday = now - datetime.timedelta(days=1)
        articles = Article.objects.filter(publication_date__gt=yesterday)
        for article in articles:
            add_item_to_push(article, self.name)

    def initialize_push(self):
        # Nothing here at the moment
        pass

    def push_one(self, row):
        files = super(FilesPusherConfiguration, self).push_one(self, row)
        self.rows.append(row)

    def finalize_push(self):

        # Build the ZIP archive
        directory_name = self.get_export_root_directory()
        archive_name = self.get_archive_name()
        zipdir(directory_name, archive_name)

        # Send it
        sent = False
        max_ = settings.CARRIER_PIGEON_MAX_PUSH_ATTEMPTS
        for push_attempt_num in xrange(max_):
            logger.debug('push attempt %s' % push_attempt_num)

            if send(archive_name, target_url, output_path):

                # Mark all the rows as sent 
                logger.debug('succeeded')
                sent = True
                for row in self.rows:
                    row.status = ItemToPush.STATUS.PUSHED
                    row.save()
                continue

        if sent:
            # Cleanup...

        else:
            logger.error(u'Send failed for "%s" after %d attempts' % (
                archive_name, max_))

    def get_export_root_directory(self):
        """ Build the name of the directory to zip. Override me! """

        return '%s/%s' % (
            settings.CARRIER_PIGEON_OUTPUT_DIRECTORY,
            self.name(),
        )

    def get_archive_name(self):
        """ Build the filename of the zip archive to create.  Override me! """

        now = datetime.datetime.now()
        return '%s/%s_%s.zip' % (
            settings.CARRIER_PIGEON_OUTPUT_DIRECTORY,
            self.name(),
            now.strftime(settings.CARRIER_PIGEON_TIMESTAMP_FORMAT)
        )