Created
October 29, 2015 16:43
-
-
Save fson/a2bea185536daaaab860 to your computer and use it in GitHub Desktop.
Import data from the Movie Database into Reindex
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import fetch from 'node-fetch'; | |
| import Qs from 'qs'; | |
| import Reindex from 'reindex-js'; | |
| import { omit } from 'lodash'; | |
| const API_URL = 'http://api.themoviedb.org/3'; | |
| const API_KEY = process.env.TMDB_API_KEY; | |
| const reindex = new Reindex(process.env.REINDEX_URL); | |
| reindex.setToken(process.env.REINDEX_TOKEN); | |
| async function get(path, params = {}) { | |
| const query = Qs.stringify({ | |
| api_key: API_KEY, | |
| ...params, | |
| }); | |
| const url = `${API_URL}/${path}?${query}`; | |
| // console.log('get', url); | |
| const response = await fetch(url, { | |
| headers: { Accept: 'application/json' }, | |
| }); | |
| if (!response.ok) { | |
| console.error( | |
| 'Request failed,', | |
| 'path:', path, | |
| 'status:', response.status, | |
| 'body:', await response.text() | |
| ); | |
| const error = new Error(`Request failed, ${path}`); | |
| error.name = response.status === 404 ? 'NotFound' : 'HTTPError'; | |
| throw error; | |
| } | |
| let data; | |
| try { | |
| data = response.json(); | |
| } catch (e) { | |
| console.error('Could not parse JSON from response', response.text()); | |
| throw e; | |
| } | |
| return data; | |
| } | |
| const DATE_KEYS = { | |
| air_date: null, | |
| birthday: null, | |
| deathday: null, | |
| first_air_date: null, | |
| last_air_date: null, | |
| release_date: null, | |
| }; | |
| async function run(mutation, inputType, input) { | |
| input = omit(input, 'id'); | |
| for (const key in input) { | |
| if (key in DATE_KEYS && !input[key]) { | |
| input[key] = null; | |
| } | |
| } | |
| const query = `mutation ($input: ${inputType}!) { | |
| ${mutation}(input: $input) { id } | |
| }`; | |
| const result = await reindex.query(query, { input }); | |
| if (result.errors) { | |
| console.error('ERROR:', result.errors.map((e) => e.message).join('\n')); | |
| return null; | |
| } | |
| return result.data[mutation].id; | |
| } | |
| function createMovie(input) { | |
| return run('createMovie', '_CreateMovieInput', input); | |
| } | |
| function createTvSeries(input) { | |
| return run('createTvSeries', '_CreateTvSeriesInput', omit(input, 'seasons')); | |
| } | |
| function createTvEpisode(input, seasonId) { | |
| input = { | |
| ...omit(input, 'crew', 'guest_stars'), | |
| season: seasonId, | |
| }; | |
| return run('createTvEpisode', '_CreateTvEpisodeInput', input); | |
| } | |
| function createTvSeason(input, tvSeriesId) { | |
| input = { | |
| ...omit(input, 'episodes'), | |
| series: tvSeriesId, | |
| }; | |
| return run('createTvSeason', '_CreateTvSeasonInput', input); | |
| } | |
| function createPerson(input) { | |
| return run('createPerson', '_CreatePersonInput', input); | |
| } | |
| const personsById = {}; | |
| async function getOrCreatePerson(personId) { | |
| if (!personsById[personId]) { | |
| let person; | |
| try { | |
| person = await get(`person/${personId}`); | |
| personsById[personId] = await createPerson(person); | |
| } catch (e) { | |
| // Some persons are not found. | |
| if (e.name !== 'NotFound') { | |
| throw e; | |
| } | |
| } | |
| } | |
| return personsById[personId]; | |
| } | |
| async function createCredits(credits, creditType, mediaType, mediaId) { | |
| for (const credit of credits) { | |
| const input = { | |
| ...omit(credit, 'cast_id', 'credit_id'), | |
| credit_type: creditType, | |
| media_type: mediaType, | |
| person: await getOrCreatePerson(credit.id), | |
| [mediaType === 'movie' ? 'movie' : 'series']: mediaId, | |
| }; | |
| await run('createCredit', '_CreateCreditInput', input); | |
| } | |
| } | |
| async function main() { | |
| try { | |
| for (const mediaType of ['movie', 'tv']) { | |
| for (let page = 1; page <= 5; page++) { | |
| const {results} = await get(`${mediaType}/popular`, { page }); | |
| for (const result of results) { | |
| const media = await get(`${mediaType}/${result.id}`); | |
| let id; | |
| if (mediaType === 'movie') { | |
| id = await createMovie(media); | |
| } else { | |
| id = await createTvSeries(media); | |
| } | |
| const credits = await get(`${mediaType}/${media.id}/credits`); | |
| console.log(media.name || media.title, id); | |
| await createCredits(credits.cast, 'cast', mediaType, id); | |
| await createCredits(credits.crew.slice(0, 3), 'crew', mediaType, id); | |
| if (media.seasons) { | |
| for (const {season_number} of media.seasons) { | |
| const season = await get( | |
| `${mediaType}/${media.id}/season/${season_number}`, | |
| ); | |
| const seasonId = await createTvSeason(season, id); | |
| for (const episode of season.episodes) { | |
| await createTvEpisode(episode, seasonId); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } catch (e) { | |
| console.error('FAIL', e.stack); | |
| throw e; | |
| } | |
| } | |
| main(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "name": "reindex-tmdb", | |
| "version": "0.1.0", | |
| "private": true, | |
| "description": "Import data from the Movie Database into Reindex", | |
| "scripts": { | |
| "import": "babel-node importTMDb.js", | |
| "lint": "eslint *.js", | |
| "test": "echo \"Error: no test specified\" && exit 1" | |
| }, | |
| "license": "MIT", | |
| "dependencies": { | |
| "babel": "^5.8.29", | |
| "lodash": "^3.10.1", | |
| "node-fetch": "^1.3.3", | |
| "qs": "^5.2.0", | |
| "reindex-js": "^0.3.0" | |
| }, | |
| "devDependencies": { | |
| "babel-eslint": "^4.1.3", | |
| "eslint": "^1.7.3", | |
| "eslint-config-airbnb": "^0.1.0" | |
| } | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| { | |
| "name": "TvSeason", | |
| "kind": "OBJECT", | |
| "interfaces": [ | |
| "Node" | |
| ], | |
| "fields": [ | |
| { | |
| "name": "air_date", | |
| "type": "DateTime" | |
| }, | |
| { | |
| "name": "episodes", | |
| "type": "Connection", | |
| "ofType": "TvEpisode", | |
| "reverseName": "season" | |
| }, | |
| { | |
| "name": "id", | |
| "type": "ID", | |
| "nonNull": true | |
| }, | |
| { | |
| "name": "name", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "overview", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "poster_path", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "season_number", | |
| "type": "Int" | |
| }, | |
| { | |
| "name": "series", | |
| "type": "TvSeries", | |
| "reverseName": "seasons" | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Person", | |
| "kind": "OBJECT", | |
| "interfaces": [ | |
| "Node" | |
| ], | |
| "fields": [ | |
| { | |
| "name": "adult", | |
| "type": "Boolean" | |
| }, | |
| { | |
| "name": "also_known_as", | |
| "type": "List", | |
| "ofType": "String" | |
| }, | |
| { | |
| "name": "biography", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "birthday", | |
| "type": "DateTime" | |
| }, | |
| { | |
| "name": "credits", | |
| "type": "Connection", | |
| "ofType": "Credit", | |
| "reverseName": "person" | |
| }, | |
| { | |
| "name": "deathday", | |
| "type": "DateTime" | |
| }, | |
| { | |
| "name": "homepage", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "id", | |
| "type": "ID", | |
| "nonNull": true | |
| }, | |
| { | |
| "name": "imdb_id", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "name", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "place_of_birth", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "popularity", | |
| "type": "Float" | |
| }, | |
| { | |
| "name": "profile_path", | |
| "type": "String" | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "TvEpisode", | |
| "kind": "OBJECT", | |
| "interfaces": [ | |
| "Node" | |
| ], | |
| "fields": [ | |
| { | |
| "name": "air_date", | |
| "type": "DateTime" | |
| }, | |
| { | |
| "name": "episode_number", | |
| "type": "Int" | |
| }, | |
| { | |
| "name": "id", | |
| "type": "ID", | |
| "nonNull": true | |
| }, | |
| { | |
| "name": "name", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "overview", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "production_code", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "season", | |
| "type": "TvSeason", | |
| "reverseName": "episodes" | |
| }, | |
| { | |
| "name": "season_number", | |
| "type": "Int" | |
| }, | |
| { | |
| "name": "still_path", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "vote_average", | |
| "type": "Float" | |
| }, | |
| { | |
| "name": "vote_count", | |
| "type": "Int" | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Country", | |
| "kind": "OBJECT", | |
| "interfaces": [], | |
| "fields": [ | |
| { | |
| "name": "iso_3166_1", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "name", | |
| "type": "String" | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Genre", | |
| "kind": "OBJECT", | |
| "interfaces": [], | |
| "fields": [ | |
| { | |
| "name": "id", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "name", | |
| "type": "String" | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Creator", | |
| "kind": "OBJECT", | |
| "interfaces": [], | |
| "fields": [ | |
| { | |
| "name": "id", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "name", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "profile_path", | |
| "type": "String" | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Movie", | |
| "kind": "OBJECT", | |
| "interfaces": [ | |
| "Node" | |
| ], | |
| "fields": [ | |
| { | |
| "name": "adult", | |
| "type": "Boolean" | |
| }, | |
| { | |
| "name": "backdrop_path", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "belongs_to_collection", | |
| "type": "Collection" | |
| }, | |
| { | |
| "name": "budget", | |
| "type": "Int" | |
| }, | |
| { | |
| "name": "credits", | |
| "type": "Connection", | |
| "ofType": "Credit", | |
| "reverseName": "movie" | |
| }, | |
| { | |
| "name": "genres", | |
| "type": "List", | |
| "ofType": "Genre" | |
| }, | |
| { | |
| "name": "homepage", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "id", | |
| "type": "ID", | |
| "nonNull": true | |
| }, | |
| { | |
| "name": "imdb_id", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "original_language", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "original_title", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "overview", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "popularity", | |
| "type": "Float" | |
| }, | |
| { | |
| "name": "poster_path", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "production_companies", | |
| "type": "List", | |
| "ofType": "Company" | |
| }, | |
| { | |
| "name": "production_countries", | |
| "type": "List", | |
| "ofType": "Country" | |
| }, | |
| { | |
| "name": "release_date", | |
| "type": "DateTime" | |
| }, | |
| { | |
| "name": "revenue", | |
| "type": "Int" | |
| }, | |
| { | |
| "name": "runtime", | |
| "type": "Int" | |
| }, | |
| { | |
| "name": "spoken_languages", | |
| "type": "List", | |
| "ofType": "Language" | |
| }, | |
| { | |
| "name": "status", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "tagline", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "title", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "video", | |
| "type": "Boolean" | |
| }, | |
| { | |
| "name": "vote_average", | |
| "type": "Float" | |
| }, | |
| { | |
| "name": "vote_count", | |
| "type": "Int" | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Language", | |
| "kind": "OBJECT", | |
| "interfaces": [], | |
| "fields": [ | |
| { | |
| "name": "iso_639_1", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "name", | |
| "type": "String" | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "User", | |
| "kind": "OBJECT", | |
| "interfaces": [ | |
| "Node" | |
| ], | |
| "fields": [ | |
| { | |
| "name": "id", | |
| "type": "ID", | |
| "nonNull": true | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "TvSeries", | |
| "kind": "OBJECT", | |
| "interfaces": [ | |
| "Node" | |
| ], | |
| "fields": [ | |
| { | |
| "name": "backdrop_path", | |
| "type": "String", | |
| "description": "File path for the backdrop image URL." | |
| }, | |
| { | |
| "name": "created_by", | |
| "type": "List", | |
| "description": "A list of creators of the TV series.", | |
| "ofType": "Creator" | |
| }, | |
| { | |
| "name": "credits", | |
| "type": "Connection", | |
| "description": "All the cast and crew credits for the TV series.", | |
| "ofType": "Credit", | |
| "reverseName": "series" | |
| }, | |
| { | |
| "name": "episode_run_time", | |
| "type": "List", | |
| "description": "A list of all episode run times (in minutes)", | |
| "ofType": "Int" | |
| }, | |
| { | |
| "name": "first_air_date", | |
| "type": "DateTime" | |
| }, | |
| { | |
| "name": "genres", | |
| "type": "List", | |
| "ofType": "Genre" | |
| }, | |
| { | |
| "name": "homepage", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "id", | |
| "type": "ID", | |
| "nonNull": true | |
| }, | |
| { | |
| "name": "in_production", | |
| "type": "Boolean" | |
| }, | |
| { | |
| "name": "languages", | |
| "type": "List", | |
| "ofType": "String" | |
| }, | |
| { | |
| "name": "last_air_date", | |
| "type": "DateTime" | |
| }, | |
| { | |
| "name": "name", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "networks", | |
| "type": "List", | |
| "ofType": "TvNetwork" | |
| }, | |
| { | |
| "name": "number_of_episodes", | |
| "type": "Int" | |
| }, | |
| { | |
| "name": "number_of_seasons", | |
| "type": "Int" | |
| }, | |
| { | |
| "name": "origin_country", | |
| "type": "List", | |
| "ofType": "String" | |
| }, | |
| { | |
| "name": "original_language", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "original_name", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "overview", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "popularity", | |
| "type": "Float", | |
| "description": "A number representing the relative popularity of the TV series." | |
| }, | |
| { | |
| "name": "poster_path", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "production_companies", | |
| "type": "List", | |
| "ofType": "Company" | |
| }, | |
| { | |
| "name": "seasons", | |
| "type": "Connection", | |
| "description": "A connection to all the seasons of this TV series.", | |
| "ofType": "TvSeason", | |
| "reverseName": "series" | |
| }, | |
| { | |
| "name": "status", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "type", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "vote_average", | |
| "type": "Float", | |
| "description": "Average rating of this TV series. (0-10)" | |
| }, | |
| { | |
| "name": "vote_count", | |
| "type": "Int" | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Collection", | |
| "kind": "OBJECT", | |
| "interfaces": [], | |
| "fields": [ | |
| { | |
| "name": "backdrop_path", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "id", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "name", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "poster_path", | |
| "type": "String" | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Credit", | |
| "kind": "OBJECT", | |
| "interfaces": [ | |
| "Node" | |
| ], | |
| "fields": [ | |
| { | |
| "name": "character", | |
| "type": "String", | |
| "description": "Name of a character. Included in cast credits." | |
| }, | |
| { | |
| "name": "credit_type", | |
| "type": "String", | |
| "description": "Credit type. Possible values are \"cast\" and \"crew\"." | |
| }, | |
| { | |
| "name": "department", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "id", | |
| "type": "ID", | |
| "nonNull": true | |
| }, | |
| { | |
| "name": "job", | |
| "type": "String", | |
| "description": "Job title. Included in crew credits." | |
| }, | |
| { | |
| "name": "media_type", | |
| "type": "String", | |
| "description": "Media type of this Credit. (`tv` or `movie`)." | |
| }, | |
| { | |
| "name": "movie", | |
| "type": "Movie", | |
| "description": "For movie credits, includes the movie of this Credit.", | |
| "reverseName": "credits" | |
| }, | |
| { | |
| "name": "name", | |
| "type": "String", | |
| "description": "Name of the person this Credit belongs to." | |
| }, | |
| { | |
| "name": "order", | |
| "type": "Int", | |
| "description": "A sequence number for default ordering of credits" | |
| }, | |
| { | |
| "name": "person", | |
| "type": "Person", | |
| "description": "The person this Credit belongs to.", | |
| "reverseName": "credits" | |
| }, | |
| { | |
| "name": "profile_path", | |
| "type": "String", | |
| "description": "The file path for the profile image URL." | |
| }, | |
| { | |
| "name": "series", | |
| "type": "TvSeries", | |
| "description": "For TV credits, includes the TV series of this Credit.", | |
| "reverseName": "credits" | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "TvNetwork", | |
| "kind": "OBJECT", | |
| "interfaces": [], | |
| "fields": [ | |
| { | |
| "name": "id", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "name", | |
| "type": "String" | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Company", | |
| "kind": "OBJECT", | |
| "interfaces": [], | |
| "fields": [ | |
| { | |
| "name": "id", | |
| "type": "String" | |
| }, | |
| { | |
| "name": "name", | |
| "type": "String" | |
| } | |
| ] | |
| } | |
| ] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment