Skip to content

Instantly share code, notes, and snippets.

@jvilledieu
Last active January 4, 2017 09:42
Show Gist options
  • Save jvilledieu/9aad4b8759be0162e6ec to your computer and use it in GitHub Desktop.
Save jvilledieu/9aad4b8759be0162e6ec to your computer and use it in GitHub Desktop.
Importing the FEC data available here: http://www.fec.gov/finance/disclosure/ftpdet.shtml#a2015_2016
//-----------------------
//Import the committees
//-----------------------
CREATE CONSTRAINT ON (e:COMMITTEE) ASSERT e.CMTE_ID IS UNIQUE;
USING PERIODIC COMMIT 2000
LOAD CSV WITH HEADERS FROM "file:c:/cmv1.csv" AS line
FIELDTERMINATOR '|'
MERGE (:COMMITTEE {
CMTE_ID: line.CMTE_ID,
CMTE_NM: line.CMTE_NM,
TRES_NM: line.TRES_NM,
CMTE_ST1: line.CMTE_ST1,
CMTE_ST2: line.CMTE_ST2,
CMTE_CITY: line.CMTE_CITY,
CMTE_ST: line.CMTE_ST,
CMTE_ZIP: line.CMTE_ZIP,
CMTE_DSGN: line.CMTE_DSGN,
CMTE_TP: line.CMTE_TP,
CMTE_PTY_AFFILIATION: line.CMTE_PTY_AFFILIATION,
CMTE_FILING_FREQ: line.CMTE_FILING_FREQ,
ORG_TP: line.ORG_TP,
CONNECTED_ORG_NM: line.CONNECTED_ORG_NM,
CAND_ID: line.CAND_ID
});
//-----------------------
//Import the candidates
//-----------------------
CREATE CONSTRAINT ON (e:CANDIDATE) ASSERT e.CAND_ID IS UNIQUE;
USING PERIODIC COMMIT 2000
LOAD CSV WITH HEADERS FROM "file:c:/cnv1.csv" AS line
FIELDTERMINATOR '|'
MERGE (:CANDIDATE {
CAND_ID: line.CAND_ID,
CAND_NAME: line.CAND_NAME,
CAND_PTY_AFFILIATION: line.CAND_PTY_AFFILIATION,
CAND_ELECTION_YR: line.CAND_ELECTION_YR,
CAND_OFFICE_ST: line.CAND_OFFICE_ST, CAND_OFFICE: line.CAND_OFFICE,
CAND_OFFICE_DISTRICT: line.CAND_OFFICE_DISTRICT, CAND_ICI: line.CAND_ICI,
CAND_STATUS: line.CAND_STATUS,
CAND_PCC: line.CAND_PCC,
CAND_ST1: line.CAND_ST1,
CAND_ST2: line.CAND_ST2,
CAND_CITY: line.CAND_CITY,
CAND_ST: line.CAND_ST,
CAND_ZIP: line.CAND_ZIP
});
//-----------------------
//Import the individuals
//-----------------------
CREATE CONSTRAINT ON (e:INDIVIDUAL) ASSERT e.IND_ID IS UNIQUE;
USING PERIODIC COMMIT 2000
LOAD CSV WITH HEADERS FROM "file:c:/itcontv1.csv" AS line
FIELDTERMINATOR '|'
WITH line
WHERE line.ENTITY_TP = 'IND'
MERGE (a:INDIVIDUAL {IND_ID: line.NAME + line.EMPLOYER})
ON CREATE SET
a.ENTITY_TP= line.ENTITY_TP,
a.NAME= line.NAME,
a.CITY= line.CITY,
a.STATE= line.STATE,
a.ZIP_CODE= line.ZIP_CODE,
a.EMPLOYER= line.EMPLOYER,
a.OCCUPATION= line.OCCUPATION;
//-----------------------
//Import the companies
//-----------------------
CREATE CONSTRAINT ON (e:COMPANIES) ASSERT e.EMPLOYER IS UNIQUE;
USING PERIODIC COMMIT 2000
LOAD CSV WITH HEADERS FROM "file:c:/itcontv1.csv" AS line
FIELDTERMINATOR '|'
WITH line
WHERE line.EMPLOYER <> ''
MERGE (:COMPANY {EMPLOYER: line.EMPLOYER});
//-----------------------
//Create the parties
//-----------------------
CREATE CONSTRAINT ON (e:PARTY) ASSERT e.PARTY_NAME IS UNIQUE;
USING PERIODIC COMMIT 2000
LOAD CSV WITH HEADERS FROM "file:c:/cmv1.csv" AS line
FIELDTERMINATOR '|'
WITH line
WHERE line.CMTE_PTY_AFFILIATION IS NOT NULL
MERGE (:PARTY {PARTY_NAME: line.CMTE_PTY_AFFILIATION});
USING PERIODIC COMMIT 2000
LOAD CSV WITH HEADERS FROM "file:c:/cnv1.csv" AS line
FIELDTERMINATOR '|'
WITH line
WHERE line.CAND_PTY_AFFILIATION IS NOT NULL
MERGE (:PARTY {PARTY_NAME: line.CAND_PTY_AFFILIATION});
//-----------------------
//Import the contributions to candidates from committees
//-----------------------
USING PERIODIC COMMIT 2000
LOAD CSV WITH HEADERS FROM "file:c:/pas2v1.csv" AS line
FIELDTERMINATOR '|'
MATCH (a:COMMITTEE {CMTE_ID: line.CMTE_ID})
MATCH (b:CANDIDATE {CAND_ID: line.CAND_ID})
CREATE (a)-[:HAS_CONTRIBUTED {
AMNDT_IND: line.AMNDT_IND,
RPT_TP: line.RPT_TP,
TRANSACTION_PGI: line.TRANSACTION_PGI,
IMAGE_NUM: line.IMAGE_NUM,
TRANSACTION_TP: line.TRANSACTION_TP,
ENTITY_TP: line.ENTITY_TP,
NAME: line.NAME,
CITY: line.CITY,
STATE: line.STATE,
ZIP_CODE: line.ZIP_CODE,
EMPLOYER: line.EMPLOYER,
OCCUPATION: line.OCCUPATION,
TRANSACTION_DT: line.TRANSACTION_DT,
TRANSACTION_AMT: toInt(line.TRANSACTION_AMT),
OTHER_ID: line.OTHER_ID,
TRAN_ID: line.TRAN_ID,
FILE_NUM: line.FILE_NUM,
MEMO_CD: line.MEMO_CD,
MEMO_TEXT: line.MEMO_TEXT,
SUB_ID: line.SUB_ID
}]->(b);
//-----------------------
//Import the links between candidates and committees
//-----------------------
USING PERIODIC COMMIT 2000
LOAD CSV WITH HEADERS FROM "file:c:/cclv1.csv" AS line
FIELDTERMINATOR '|'
MATCH (a:COMMITTEE {CMTE_ID: line.CMTE_ID})
MATCH (b:CANDIDATE {CAND_ID: line.CAND_ID})
CREATE (a)-[:HAS_RELATIONSHIP {
CAND_ELECTION_YR: line.CAND_ELECTION_YR,
FEC_ELECTION_YR: line.FEC_ELECTION_YR,
CMTE_TP: line.CMTE_TP,
IMAGE_NUM: line.IMAGE_NUM,
TRANSACTION_TP: line.TRANSACTION_TP,
CMTE_DSGN: line.CMTE_DSGN,
LINKAGE_ID: line.LINKAGE_ID
}]->(b);
//-----------------------
//Import all transactions between the committees
//-----------------------
USING PERIODIC COMMIT 2000
LOAD CSV WITH HEADERS FROM "file:c:/othv1.csv" AS line
FIELDTERMINATOR '|'
MATCH (a:COMMITTEE {CMTE_ID: line.CMTE_ID})
MATCH (b:COMMITTEE {CMTE_ID: line.OTHER_ID})
CREATE (a)-[:HAS_RELATIONSHIP {
AMNDT_IND: line.AMNDT_IND,
RPT_TP: line.RPT_TP,
TRANSACTION_PGI: line.TRANSACTION_PGI,
IMAGE_NUM: line.IMAGE_NUM,
TRANSACTION_TP: line.TRANSACTION_TP,
ENTITY_TP: line.ENTITY_TP,
NAME: line.NAME,
CITY: line.CITY,
STATE: line.STATE,
ZIP_CODE: line.ZIP_CODE,
EMPLOYER: line.EMPLOYER,
OCCUPATION: line.OCCUPATION,
TRANSACTION_DT: line.TRANSACTION_DT,
TRANSACTION_AMT: toInt(line.TRANSACTION_AMT),
TRAN_ID: line.TRAN_ID,
FILE_NUM: line.FILE_NUM,
MEMO_CD: line.MEMO_CD,
MEMO_TEXT: line.MEMO_TEXT,
SUB_ID: line.SUB_ID
}]->(b);
//-----------------------
//Import the contributions from individuals
//-----------------------
USING PERIODIC COMMIT 2000
LOAD CSV WITH HEADERS FROM "file:c:/itcontv1.csv" AS line
FIELDTERMINATOR '|'
MATCH (a:INDIVIDUAL {IND_ID: line.NAME + line.EMPLOYER})
MATCH (b:COMMITTEE {CMTE_ID: line.CMTE_ID})
CREATE (a)-[:HAS_CONTRIBUTED {
AMNDT_IND: line.AMNDT_IND,
RPT_TP: line.RPT_TP,
TRANSACTION_PGI: line.TRANSACTION_PGI,
IMAGE_NUM: line.IMAGE_NUM,
TRANSACTION_TP: line.TRANSACTION_TP,
TRANSACTION_DT: line.TRANSACTION_DT,
TRANSACTION_AMT: toInt(line.TRANSACTION_AMT),
TRAN_ID: line.TRAN_ID,
FILE_NUM: line.FILE_NUM,
MEMO_CD: line.MEMO_CD,
MEMO_TEXT: line.MEMO_TEXT,
SUB_ID: line.SUB_ID
}]->(b);
//-----------------------
//Create links from individuals and their employers
//-----------------------
MATCH (a:INDIVIDUAL)
MATCH (b:COMPANY {EMPLOYER: a.EMPLOYER})
CREATE (a)-[:IS_EMPLOYED {OCCUPATION: a.OCCUPATION}]->(b);
//-----------------------
//Import the contributions from candidates to committees
//-----------------------
USING PERIODIC COMMIT 2000
LOAD CSV WITH HEADERS FROM "file:c:/itcontv1.csv" AS line
FIELDTERMINATOR '|'
WITH line
WHERE line.ENTITY_TP = "CAN"
MATCH (a:CANDIDATE {CAND_ID: line.OTHER_ID})
MATCH (b:COMMITTEE {CMTE_ID: line.CMTE_ID})
CREATE (a)-[:HAS_CONTRIBUTED {
AMNDT_IND: line.AMNDT_IND,
RPT_TP: line.RPT_TP,
TRANSACTION_PGI: line.TRANSACTION_PGI,
IMAGE_NUM: line.IMAGE_NUM,
TRANSACTION_TP: line.TRANSACTION_TP,
TRANSACTION_DT: line.TRANSACTION_DT,
TRANSACTION_AMT: toInt(line.TRANSACTION_AMT),
TRAN_ID: line.TRAN_ID,
FILE_NUM: line.FILE_NUM,
MEMO_CD: line.MEMO_CD,
MEMO_TEXT: line.MEMO_TEXT,
SUB_ID: line.SUB_ID
}]->(b);
//-----------------------
//Import the party affiliation of committees
//-----------------------
MATCH (a:COMMITTEE)
WHERE a.CMTE_PTY_AFFILIATION IS NOT NULL
MATCH (b:PARTY {PARTY_NAME: a.CMTE_PTY_AFFILIATION })
MERGE (a)-[:AFFILIATED_TO]->(b);
//-----------------------
//Import the party affiliation of candidates
//-----------------------
MATCH (a:CANDIDATE)
WHERE a.CAND_PTY_AFFILIATION <> ''
MATCH (b:PARTY {PARTY_NAME: a.CAND_PTY_AFFILIATION })
MERGE (a)-[:AFFILIATED_TO]->(b);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment