rectalogic

#!/bin/bash

# Copies an entire mongodb collection into an RDS table, skipping existing rows.

MONGOHOST=""
RDS=""
TABLE=""
DB="cureatr"
COLLECTION=""

rectalogic

import gc
from threading import Thread
import boto3
import botocore


def client():
    boto3.Session().client("s3")

rectalogic

def stream(q):
    def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
        cursor.scrollable = True
    with sql.rds_engine.engine.connect() as conn:
        streamconn = conn.execution_options(stream_results=True)
        sa.event.listen(streamconn, "before_cursor_execute", before_cursor_execute)
        results = streamconn.execute(q)
        print "rowcount", results.rowcount
        r = conn.execute("MOVE FORWARD ALL FROM {}".format(results.cursor.name))
        r = conn.execute("MOVE BACKWARD ALL FROM {}".format(results.cursor.name))

rectalogic

>>> import sqlalchemy as sa
>>> from cureatr.model import sql
>>> t = sa.Table(
...     "cureatr.user_messages", sql.rds_metadata,
...     sa.Column("_id", sql.ObjectIdType, primary_key=True),
...     sa.Column("document", sa.dialects.postgresql.JSONB))
>>> message_id = t.c.document['message_id'].label("message_id")
>>> iid = t.c.document['iid'].label("iid")
>>> q = sa.select([message_id, iid])
>>> r = sql.rds_engine.engine.execute(q)

rectalogic

>>> import contextlib
>>> import struct
>>> import bson
>>> import mock
>>>
>>> @contextlib.contextmanager
... def timed_oid(time):
...     increment = [0]
...     RealObjectId = bson.ObjectId
...     def TimedObjectId(oid=None):

rectalogic

cureatr_mongodb=> select not '{"tags": ["foo"]}'::jsonb <@ '{"tags": []}';
 ?column?
----------
 t
(1 row)

cureatr_mongodb=> select not '{"tags": []}'::jsonb <@ '{"tags": []}';
 ?column?
----------
 f

rectalogic

>>> import sqlalchemy as sa
>>> metadata = sa.MetaData()
>>> table = sa.Table("foo", metadata, sa.Column("id", sa.String))
>>> index = sa.Index("foo_idx", table.c.id, postgresql_concurrently=True)
>>> engine = sa.create_engine("postgresql://cureatr@localhost/cureatr_mongodb?sslmode=require")
>>> table.create(engine)
Traceback (most recent call last):
  File "<console>", line 1, in <module>
  File "/virtualenv/local/lib/python2.7/site-packages/sqlalchemy/sql/schema.py", line 742, in create
    checkfirst=checkfirst)

rectalogic

>>> INDEX_RE = re.compile(r"CREATE( | UNIQUE )INDEX( | CONCURRENTLY)")
>>> INDEX_RE.match("CREATE UNIQUE INDEX CONCURRENTLY foo").group(0)
'CREATE UNIQUE INDEX '
>>> INDEX_RE.match("CREATE UNIQUE INDEX CONCURRENTLY foo").group(1)
' UNIQUE '
>>> INDEX_RE.match("CREATE UNIQUE INDEX CONCURRENTLY foo").group(2)
' '

rectalogic

create table json_test (
id SERIAL,
assay1_ic50 FLOAT,
assay2_ic50 FLOAT,
data JSONB
);

DO
$do$
DECLARE

rectalogic

>>> class Foo(object):
...     __slots__ = ("e",)
...     e = None
...
>>> f = Foo()
>>> f.e = "x"
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
AttributeError: 'Foo' object attribute 'e' is read-only
>>> f.e