An incomplete account of how I used DSE to be happy

converters.py

class MyConverter(Converter):
    """
    Convert .txt files extracted from SAP.
    
    """
    
    COLUMNS = ('date', 'account', 'value', 'user', 'text')
    DELIMITER = '|'
    name =_('TIM SAP - Layout /FCORREA')

    def work_through_file(self, f):
        # grab a list of current accounts to know
        # those which will need to be added
        self.temp_table = 'temp_pannel_entry'
        self.accounts = dict(Account.objects.values_list('pk', 'pk'))
        self.accounts_queue = self.accounts
        dse.patch_model(Account)
        dse.patch_model(Entry, db_table=self.temp_table,
                        sql="""
                        CREATE TEMPORARY TABLE "%(temp)s" (
                            "id" serial NOT NULL PRIMARY KEY,
                            "account_id" varchar(255),
                            "value" numeric(20, 2),
                            "date" date,
                            "text" varchar(255)
                        );
                        """ % {'temp': self.temp_table}
                        )
        
        # we use DSE to insert entries in chunks and speed up the import.
        # using a DSE-delayed Entry, inserts will only happen when the
        # loop is done or after each 1000 iterations.
        with Entry.delayed as model:
            for line in f:
                self.pos = self.file.tell()
                self.add_entry(model, line)
                self.update_cache(self.pos)
            
            model.flush()
            
            # after flushing the insert, fetch aggregate rows to
            # insert them into the permanent database.
            c = model.cursor
            c.execute("""
                INSERT INTO %(db_table)s
                    ("account_id", "value", "date", "date_created",
                     "date_edited", "text", "status", "upload_id",
                     "created_by_id", "last_edited_by_id")
                 
                SELECT
                    account_id,
                    SUM(value) as value,
                    date_trunc('month', date) as date,
                    now() as date_created,
                    now() as date_edited,
                    text,
                    '1' as status,
                    %(upload_pk)d as upload_id,
                    %(user_id)s as created_by_id,
                    %(user_id)s as last_edited_by_id
                    
                    
                    FROM "temp_pannel_entry"
                    
                    GROUP BY text, account_id, date_trunc('month', date)
                    ORDER BY text;
                """ % {'db_table': Entry._meta.db_table, 'user_id': self.user_pk, 'upload_pk': self.upload.pk})
                
            c.execute("""COMMIT;""")
                
    def add_entry(self, model, line):
        date = self.fix_date(line['date'])

        if self.accounts.get(line['account'], None):
            account = Account.objects.get(code=line['account'])
            
        else:
            account = Account.objects.create(code=line['account'],
                                             created_by_id=self.user_pk)
            
            self.accounts[line['account']] = line['account']
            
        line.pop('account', None)
        line.pop('task', None)
        value = Decimal(line['value'])
        account_pk = account.pk
        line.update({
                     'account_id': account_pk,
                     'date': date,
                     'value': value, 
                     'text': line['text'],
                     })
        
        # DSE insert
        model.insert(dict(line))

models.py

class BasePannelModel(models.Model):
    created_by = models.ForeignKey(User, blank=True, null=True,
                                   related_name='created_%(class)s')
    
    last_edited_by = models.ForeignKey(User, blank=True, null=True,
                                       related_name='edited_%(class)s')
    
    date_created = models.DateTimeField(blank=True, null=True)
    
    date_edited = models.DateTimeField(blank=True, null=True)
    
    DEFAULT_STATUS = 1
    STATUS_CHOICES = (
        (0, _('Inactive')),    
        (1, _('Active')),                 
        (9, _('Other')),
    )

    status = models.IntegerField(choices=STATUS_CHOICES, blank=True)

    class Meta:
        abstract = True

class Account(BasePannelModel):
    code = models.CharField(max_length=255, primary_key=True,
                            verbose_name=_('Account Code'))
    
    name = models.CharField(max_length=512,
                            blank=True, null=True,
                            verbose_name=_('Name'))

class Entry(PermalinkMixin, BasePannelModel):
    account = models.ForeignKey('Account',
                                verbose_name=_('Account code'))
    
    value = CurrencyField(max_digits=20, decimal_places=2,
                          verbose_name=_('Value'))
    
    text = models.CharField(max_length=512, verbose_name=_('Text'))
    
    date = models.DateField(verbose_name=_('Date'))