Created
July 10, 2015 04:04
-
-
Save glennsarti/ea09e9b346c842ff5d45 to your computer and use it in GitHub Desktop.
These two scripts can be used to convert the data from the Star Wars API (http://swapi.co) into an import file for Neo4j. These scripts download the JSON data from the API and converts it into Cypher statements. It's crude and probably not the best way to do it, but, it works!. Firstly download the JSON data using GetSWAPI.ps1, and then run Conv…
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
$ErrorActionPreference = 'Stop' | |
$VerbosePreference = 'SilentlyContinue' | |
Function Safe-CypherString($value) { | |
$value = $value.Replace('\','\\').Replace("`n",'\n').Replace("`r",'\r').Replace("'","\'") | |
Write-Output $value | |
} | |
Function Convert-JSONToCypher($dataObject) { | |
Write-Verbose "Processing $($dataObject.url)" | |
if ($matches -ne $null) { $matches.Clear() } | |
if (($dataobject.Url) -match '\/([a-zA-Z]+)\/\d+\/$') { | |
switch ($matches[1].ToLower()){ | |
'films' { $nodeLabel = 'Film' } | |
'people' { $nodeLabel = 'Person' } | |
'planets' { $nodeLabel = 'Planet' } | |
'species' { $nodeLabel = 'Species' } | |
'starships' { $nodeLabel = 'Starship' } | |
'vehicles' { $nodeLabel = 'Vehicle' } | |
default { Throw "Unknown URL Type $($matches[1])" } | |
} | |
} | |
$cypher = '' | |
$cypherSet = '' | |
$relationships = @() | |
Get-Member -InputObject $dataObject -MemberType NoteProperty | % { | |
$thisProperty = $_ | |
Write-Verbose "Processing property $($thisProperty.Name)" | |
$thisPropertyValue = $thisResult."$($thisProperty.Name)" | |
if (($thisProperty.Name -ne 'created') -and ($thisProperty.Name -ne 'edited') -and ($thisPropertyValue -ne $null) ) | |
{ | |
switch ($thisPropertyValue.GetType().ToString()) | |
{ | |
'System.Object[]' { | |
$relName = '' | |
switch ($thisProperty.Name) { | |
'films' { $relName = 'FILM' } | |
'species' { $relName = 'SPECIES' } | |
'starships' { $relName = 'STARSHIP' } | |
'vehicles' { $relName = 'VEHICLE' } | |
'planets' { $relName = 'PLANET' } | |
'people' { $relName = 'PERSON' } | |
'pilots' { $relName = 'PILOT' } | |
'residents' { $relName = 'RESIDENT' } | |
'characters' { $relName = 'CHARACTER' } | |
default { Throw "Unknown array name of $($thisProperty.Name)" } | |
} | |
$thisPropertyValue | % { | |
$relationships += ($relName + "|" + $_) | |
} | |
} | |
'System.String' { | |
if (($thisPropertyValue.IndexOf('http') -ne -1) -and ($thisProperty.Name -ne 'url')) | |
{ | |
$relName = '' | |
switch ($thisProperty.Name) { | |
'homeworld' { $relName = 'HOMEWORLD' } | |
default { Throw "Unknown single reference name of $($thisProperty.Name)" } | |
} | |
$relationships += ($relName + "|" + $thisPropertyValue) | |
} | |
else | |
{ | |
$cypherSet += ",$($thisProperty.Name):'$(Safe-CypherString $thisPropertyValue)'" | |
} | |
} | |
'System.Int32' { | |
$cypherSet += ",$($thisProperty.Name):$thisPropertyValue" | |
} | |
default { Throw "Unknown object type $($thisPropertyValue.GetType().ToString())" } | |
} | |
} | |
} | |
$cypherSet = 'n = { ' + $cypherSet.SubString(1,$cypherSet.Length - 1) + '}' | |
$cypher = "MERGE (n {url:'$($dataobject.Url)'}) SET n:$($nodeLabel), $cypherSet;`n`r" | |
$relText = '' | |
$index = [int]0 | |
$relationships | % { | |
$relName = ($_.Split('|')[0]) | |
$url = ($_.Split('|')[1]) | |
$relText += "MERGE (n$($index.ToString()) {url:'$($url)'})`n`rMERGE (n)-[:$($relName)]->(n$($index.ToString()))`n`r" | |
$index++ | |
} | |
if ($relText -ne '') { | |
$cypher += "MATCH (n {url:'$($dataObject.url)'})`n`r" + $relText + ';' | |
} | |
Write-Output $cypher | |
} | |
@" | |
// Delete Everything | |
MATCH ()-[r]-() DELETE r; | |
MATCH (n) DELETE n; | |
"@ | Out-File -FilePath "$($PSScriptRoot)\StarWars.cypher" -Encoding "ASCII" -Force -Confirm:$false | |
Get-ChildItem -Path $PSScriptRoot -Filter "*.json" | % { | |
$thisFile = $_ | |
Write-Verbose "Reading $($_.Fullname)" | |
Write-Output "// Exported from $($thisFile.Name)" | |
$dataFile = ConvertFrom-JSon -InputObject ([IO.File]::ReadAllText($thisFile.Fullname)) | |
$dataFile.results | % { | |
$thisResult = $_ | |
Write-Output (Convert-JSONToCypher $thisResult) | |
} | |
} | Out-File -FilePath "$($PSScriptRoot)\StarWars.cypher" -Encoding "ASCII" -Append -NoClobber -Force -Confirm:$false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
$rootURL = 'http://swapi.co:80/api' | |
$resources = @('films','people','starships','vehicles','species','planets') | |
Get-ChildItem -Path $PSScriptRoot -Filter "*.json" | Remove-Item -Force -Confirm:$false | Out-Null | |
$resources | % { | |
$resource = $_ | |
$page = [int]1 | |
try | |
{ | |
do | |
{ | |
if ($page -eq 1) { $url = "$rootURL/$resource/" } else { $url = "$rootURL/$resource/?page=$page" } | |
Write-Host $url | |
Invoke-RestMethod -Uri $url -Method GET -OutFile "$PSScriptRoot\$($resource)_$($page).JSON" -ErrorAction 'Stop' | |
$page++ | |
} until ($page -gt 50) | |
} | |
catch | |
{ # Ignore errors. It's a hack but hey... | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment