Skip to content

Instantly share code, notes, and snippets.

@HoverBaum
Created March 26, 2018 08:37
Show Gist options
  • Save HoverBaum/0e197eb89ce1c82cef1f1f16d1afe579 to your computer and use it in GitHub Desktop.
Save HoverBaum/0e197eb89ce1c82cef1f1f16d1afe579 to your computer and use it in GitHub Desktop.
Code example for blogpost about migrating from wordpress to Contentful.
[{
link: 'link to wordpress iage.jpg',
description: 'describe the image',
title: 'and title it',
postId: 'because linking back is nice'
}, ...]
category: {
'en-US': {
sys: {
type: 'Link',
linkType: 'Entry',
id: categoryId
}
}
}
const createAndPublishAssets = () => {
space = await getTheRightSpace()
const createAndPublishSingleAsset = () => {
let asset
try {
asset = await createAssetInContentful()
}
try {
asset = await asset.publish()
}
createAndPublishSingleAsset( nextAsset )
}
createAndPublishSingleAsset( firstAsset )
}
const createAndPublishAssets = (assets, managementToken, spaceId, locale, simpleLog = console.log) => new Promise(async resolve => {
simpleLog('Creating Contentful client')
const client = contentful.createClient({
accessToken: managementToken,
logHandler: (level, data) => simpleLog(`${level} | ${data}`)
})
const iterableAssets = makeIterator(assets)
const space = await client.getSpace(spaceId)
const cmsAssets = []
const assetProcessingTimes = []
const inProcess = new Map()
let processedAssetsCounter = 0
const createAndPublishSingleAsset = async ({asset, done, index}) => {
if (done) {
// Check if there are still requests in process.
if (inProcess.size > 0) return false
return resolve(cmsAssets)
}
// Process the asset.
const start = Date.now()
const id = '' + start + Math.round(Math.random() * 100)
inProcess.set(id, true)
let cmsAsset
try {
cmsAsset = await space.createAsset({
fields: {
title: {
[locale]: asset.title
},
description: {
[locale]: asset.description
},
file: {
[locale]: {
contentType: 'image/jpep',
fileName: `${asset.title.toLowerCase().replace(/\s/g, '-')}.jpg`,
upload: encodeURI(asset.link)
}
}
}
})
} catch (e) {
simpleLog(`Asset "${asset.title}" failed to create, retrying...`)
createAndPublishSingleAsset({
asset,
done,
index
})
}
try {
const processedCMSAsset = await cmsAsset.processForAllLocales()
const publishedCMSAsset = await processedCMSAsset.publish()
// Save mapping information to WordPress
publishedCMSAsset.wpAsset = asset
cmsAssets.push(publishedCMSAsset)
assetProcessingTimes.push((Date.now() - start) / 1000)
inProcess.clear(id)
const eta = Math.floor(assetProcessingTimes.reduce((a, b) => a + b, 0)/assetProcessingTimes.length * (assets.length - index) / 60)
processedAssetsCounter += 1
simpleLog(`Processed asset ${processedAssetsCounter}/${assets.length} - eta: ${eta}m`)
createAndPublishSingleAsset(iterableAssets.next())
} catch (e) {
simpleLog(`Asset "${asset.title}" failed to process, retrying...`)
await cmsAsset.delete()
createAndPublishSingleAsset({
asset,
done,
index
})
}
}
// Run multiple processings in parallel while still keeping it better debuggable.
// Because Contentful has a rate limit (10/s) and we want nice console output.
// Manual experiments yielded three as a good border for no errors.
simpleLog('Starting to create assets')
createAndPublishSingleAsset(iterableAssets.next())
createAndPublishSingleAsset(iterableAssets.next())
createAndPublishSingleAsset(iterableAssets.next())
})
const createAndPublishCategories = async (categories, spaceId, managementToken, simpleLog = console.log) => {
const client = contentful.createClient({
accessToken: managementToken,
logHandler: (level, data) => simpleLog(`${level} | ${data}`)
})
const space = await client.getSpace(spaceId)
const createdCategories = await Promise.all(categories.map(category => new Promise(async resolve => {
let cmsCategory
try {
cmsCategory = await space.createEntry('blogCategory', {
fields: {
categoryName: {
'en-US': category.name
}
}
})
} catch(e) {
throw(Error(e))
}
try {
await cmsCategory.publish()
} catch(e) {
throw(Error(e))
}
// Save mapping information to contentful.
cmsCategory.wpCategory = category
resolve(cmsCategory)
})))
return createdCategories
}
const createBlogPosts = async (posts, assets, categories, managementToken, spaceId, simpleLog = console.log) => {
const client = contentful.createClient({
accessToken: managementToken,
logHandler: (level, data) => simpleLog(`${level} | ${data}`)
})
const space = await client.getSpace(spaceId)
const linkMap = new Map()
assets.forEach(asset => linkMap.set(asset.wpAsset.link, asset.fields.file['en-US'].url))
return Promise.all(posts.map(post => new Promise(async resolve => {
const cmsHeroImageAsset = assets.find(asset => asset.wpAsset.mediaNumber === post.featured_media)
const heroImageId = cmsHeroImageAsset.sys.id
const cmsCategory = categories.find(category => category.wpCategory.categoryNumber === post.category)
const categoryId = cmsCategory.sys.id
let cmsPost
try {
cmsPost = await space.createEntry('blogPost', {
fields: {
title: {
'en-US': post.title
},
body: {
'en-US': replaceWPWithContentfulLinks(post.body, linkMap)
},
slug: {
'en-US': post.slug
},
publishDate: {
'en-US': post.publishDate
},
heroImage: {
'en-US': {
sys: {
type: 'Link',
linkType: 'Asset',
id: heroImageId
}
}
},
category: {
'en-US': {
sys: {
type: 'Link',
linkType: 'Entry',
id: categoryId
}
}
}
}
})
} catch(e) {
throw(Error(e))
}
try {
await cmsPost.publish()
} catch(e) {
throw(Error(e))
}
resolve(cmsPost)
})))
}
const replaceWPWithContentfulLinks = (text, linkMap) => {
let replacedText = text
linkMap.forEach((newUrl, oldUrl) => {
replacedText = replacedText.replace(oldUrl, newUrl)
})
return replacedText
}
const generateAssetsList = (posts, baseUrl, simpleLog = console.log) => new Promise(async resolve =>{
const apiURL = `${baseUrl.replace(/\/$/, '')}/wp-json/wp/v2/media`
simpleLog('Reducing posts to asset numbers')
let infosFetched = 0
// First add the featured_media images and get ther URLs.
const featuredAssets = await Promise.all(posts.reduce((all, post) => {
if (!post.featured_media) return all
return all.concat([{
mediaNumber: post.featured_media,
postId: post.id
}])
}, [])
.map(async ({mediaNumber, postId}, i , array) => {
const featuredMedia = await getJSON(`${apiURL}/${mediaNumber}`)
infosFetched += 1
simpleLog(`Getting info about assets ${infosFetched}/${array.length}`)
return {
mediaNumber,
link: featuredMedia.guid.rendered,
title: featuredMedia.title.rendered || `asset${i}`,
description: featuredMedia.alt_text || '',
postId
}
// After all this we also add images from the body of posts.
}))
const assets = featuredAssets.concat(posts.reduce((all, post) => {
const images = post.bodyImages ? post.bodyImages : []
return all.concat(images)
}, []))
resolve(assets)
})
const generateAssetsList = (posts, baseUrl, simpleLog = console.log) => new Promise(async resolve => {
const apiURL = `${baseUrl.replace(/\/$/, '')}/wp-json/wp/v2/categories`
// First reduce posts to an array of category numbers.
simpleLog('Reducing posts to category numbers')
const categories = await Promise.all(posts.reduce((all, post) => {
if(!post.category) return all
if(all.indexOf(post.category) > -1) return all
return all.concat([post.category])
}, [])
.map(async categoryNumber => {
simpleLog(`Getting information about categories`)
const categoryData = await getJSON(`${apiURL}/${categoryNumber}`)
return {
categoryNumber,
name: categoryData.name,
slug: categoryData.slug,
description: categoryData.description
}
}))
resolve(categories)
})
const exportBlogposts = (apiUrl, log) => new Promise(resolve => {
const exportPageOfPosts = (apiUrl, page = 1, allPosts = []) => {
log(`Getting posts for page ${page}`)
const url = `${apiUrl}?page=${page}`
https.get(url, (res) => {
// When we get a 404 back we went one page over those with posts.
// So we are done now.
if(res.statusCode === 400) {
return resolve(allPosts)
}
let result = ''
res.on('data', (d) => {
result += d.toString()
})
res.on('end', async () => {
blogPosts = JSON.parse(result)
return exportPageOfPosts(apiUrl, page + 1, allPosts.concat(blogPosts))
})
}).on('error', (e) => {
throw(Error('Error while exporting blogposts', e))
})
}
exportPageOfPosts(apiUrl)
})
const transformPosts = posts => posts.map(post => {
delete post._links
delete post.guid
delete post.excerpt
delete post.author
delete post.comment_status
delete post.ping_status
delete post.template
delete post.format
delete post.meta
delete post.status
delete post.type
post.publishDate = post.date_gmt + '+00:00'
delete post.date_gmt
delete post.date
delete post.modified
delete post.modified_gmt
delete post.tags
delete post.sticky
post.body = `<div>${post.content.rendered}</div>`
delete post.content
post.title = post.title.rendered
post.slug = post.slug
post.category = post.categories[0]
delete post.categories
return extractBodyImages(post)
})
const extractBodyImages = post =>{
const regex = /<img.*?src="(.*?)"[\s\S]*?alt="(.*?)"/g
post.bodyImages = []
while (foundImage = regex.exec(post.body)) {
const alt = foundImage[2] ? foundImage[2].replace(/_/g, ' ') : ''
post.bodyImages.push({
link: foundImage[1],
description: alt,
title: alt,
postId: post.id
})
}
return post
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment