Last active
August 23, 2019 11:33
-
-
Save jtraulle/a8ed0244a2a77498f29ac32fd2bc0f65 to your computer and use it in GitHub Desktop.
Customized kunena3.rb Discourse import script for Dolibarr french forum migration
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # frozen_string_literal: true | |
| require "mysql2" | |
| require File.expand_path(File.dirname(__FILE__) + "/base.rb") | |
| class ImportScripts::Kunena < ImportScripts::Base | |
| DB_HOST ||= ENV['DB_HOST'] || "172.17.0.1" | |
| DB_NAME ||= ENV['DB_NAME'] || "kunena" | |
| DB_USER ||= ENV['DB_USER'] || "root" | |
| DB_PW ||= ENV['DB_PW'] || "root" | |
| KUNENA_PREFIX ||= ENV['KUNENA_PREFIX'] || "gvrsi_" | |
| IMAGE_PREFIX ||= ENV['IMAGE_PREFIX'] || "https://www.dolibarr.fr/media/kunena/attachments" | |
| PARENT_FIELD ||= ENV['PARENT_FIELD'] || "parent_id" | |
| DISCOURSE_BASEURL ||= ENV['DISCOURSE_BASEURL'] || "http://localhost" | |
| ATTACHMENTS_LOCAL_FOLDER ||= ENV['ATTACHMENTS_LOCAL_FOLDER'] || "/kunena_attachments" | |
| BATCH_SIZE ||= 1000 | |
| def initialize | |
| super | |
| @users = {} | |
| @client = Mysql2::Client.new( | |
| host: DB_HOST, | |
| username: DB_USER, | |
| password: DB_PW, | |
| database: DB_NAME | |
| ) | |
| end | |
| def execute | |
| import_users | |
| import_categories | |
| import_posts | |
| import_likes | |
| end | |
| def import_users | |
| puts "", "Importing users..." | |
| last_user_id = -1 | |
| unified_users_query = <<-SQL | |
| SELECT | |
| id, | |
| username, | |
| email, | |
| password, | |
| registerDate, | |
| signature, | |
| location, | |
| websiteurl, | |
| birthdate, | |
| avatar, | |
| moderator, | |
| 1 AS ordering | |
| FROM #{KUNENA_PREFIX}users | |
| INNER JOIN #{KUNENA_PREFIX}kunena_users ON #{KUNENA_PREFIX}kunena_users.userid = #{KUNENA_PREFIX}users.id | |
| UNION | |
| SELECT | |
| userid, | |
| name, | |
| SUBSTRING_INDEX(GROUP_CONCAT(email), ',',1) AS email, | |
| NULL AS password, | |
| FROM_UNIXTIME(MIN(time)) AS registerDate, | |
| NULL AS signature, | |
| NULL AS location, | |
| NULL AS website, | |
| NULL AS birthdate, | |
| NULL AS avatar, | |
| 0 AS moderator, | |
| 2 AS ordering | |
| FROM #{KUNENA_PREFIX}kunena_messages | |
| WHERE userid NOT IN(SELECT id FROM #{KUNENA_PREFIX}users) | |
| AND email IS NOT NULL AND email <> '' | |
| GROUP BY userid, name | |
| UNION | |
| SELECT | |
| userid, | |
| name, | |
| NULL AS email, | |
| NULL AS password, | |
| FROM_UNIXTIME(MIN(time)) AS registerDate, | |
| NULL AS signature, | |
| NULL AS location, | |
| NULL AS website, | |
| NULL AS birthdate, | |
| NULL AS avatar, | |
| 0 AS moderator, | |
| 3 AS ordering | |
| FROM #{KUNENA_PREFIX}kunena_messages | |
| WHERE userid NOT IN(SELECT id FROM #{KUNENA_PREFIX}users) | |
| AND (email IS NULL OR email = '') | |
| AND userid NOT IN( | |
| SELECT DISTINCT userid | |
| FROM #{KUNENA_PREFIX}kunena_messages | |
| WHERE userid NOT IN(SELECT id FROM #{KUNENA_PREFIX}users) | |
| AND email IS NOT NULL AND email <> '' | |
| GROUP BY userid | |
| ) | |
| GROUP BY userid, name | |
| ORDER BY ordering, id | |
| SQL | |
| # We populate this array on each user creation to be able to check if we have duplicates emails | |
| emails = [] | |
| total = @client.query(" | |
| SELECT COUNT(*) count | |
| FROM | |
| ( | |
| #{unified_users_query} | |
| ) count; | |
| ").first['count'] | |
| batches(BATCH_SIZE) do |offset| | |
| users = @client.query(" | |
| #{unified_users_query} | |
| LIMIT #{BATCH_SIZE} | |
| OFFSET #{offset}; | |
| ", cache_rows: false).to_a | |
| break if users.empty? | |
| last_user_id = users[-1]["id"] | |
| user_ids = users.map { |u| u["id"] } | |
| create_users(users, total: total, offset: offset) do |u| | |
| # If the current email has been already used for a previous user | |
| # we discard the email to be able to create the account and keep the previous username | |
| if u["email"].presence | |
| if emails.include?(u["email"]) | |
| u["email"] = fake_email | |
| else | |
| emails << u["email"] | |
| end | |
| end | |
| { | |
| id: u["id"], | |
| moderator: (u["moderator"].to_i == 1), | |
| created_at: u["registerDate"], | |
| email: u["email"].presence || fake_email, | |
| username: u["username"], | |
| website: u["websiteurl"], | |
| location: u["location"], | |
| date_of_birth: u["birthdate"], | |
| bio_raw: u["signature"], | |
| suspended_at: u["banned"].present? ? Time.zone.now : nil, | |
| suspended_till: u["banned"].present? ? 100.years.from_now : nil, | |
| post_create_action: proc do |user| | |
| if u["avatar"].present? | |
| UserAvatar.import_url_for_user("https://www.dolibarr.fr/media/kunena/avatars/" + u["avatar"], user) rescue nil | |
| end | |
| user.custom_fields['import_pass'] = u["password"] | |
| user.save | |
| end | |
| } | |
| end | |
| end | |
| end | |
| def import_categories | |
| puts '', "Importing categories..." | |
| categories = [ { | |
| "id": 1, | |
| "name": (+"Annonces & actualités"), | |
| "description": (+"Vous trouverez dans cette catégorie des annonces et actualités en lien avec le projet Dolibarr : nouvelles versions, évènements, mise à jour des outils communautaire du projet, etc."), | |
| "position": 1, | |
| "parent_category_id": 0 | |
| }, { | |
| "id": 2, | |
| "name": (+"Installer mon Dolibarr"), | |
| "description": (+"Obtenir de l’aide sur l’installation de Dolibarr quelque soit votre système (GNU/Linux, macOS, Windows, système NAS), votre méthode d’installation (depuis les sources, DoliWamp, DoliDeb, DoliRpm) et votre environnement (serveur local, mutualisé, dédié, cloud)."), | |
| "position": 2, | |
| "parent_category_id": 0 | |
| }, { | |
| "id": 3, | |
| "name": (+"Utiliser mon Dolibarr"), | |
| "description": (+"Obtenir de l’aide sur l’utilisation des modules fournis avec Dolibarr, suggérer des nouvelles fonctionnalités ou signaler des anomalies."), | |
| "position": 3, | |
| "parent_category_id": 0 | |
| }, { | |
| "id": 4, | |
| "name": (+"Modules GRC/GRF"), | |
| "description": (+"Tiers, Contrats/Abonnements, Interventions, Commandes, Expéditions, Tickets"), | |
| "position": 4, | |
| "parent_category_id": 3 | |
| }, { | |
| "id": 5, | |
| "name": (+"Modules GRH"), | |
| "description": (+"Utilisateurs, Adhérents, Congés, Notes de frais"), | |
| "position": 5, | |
| "parent_category_id": 3 | |
| }, { | |
| "id": 6, | |
| "name": (+"Modules PM"), | |
| "description": (+"Produits, Services"), | |
| "position": 6, | |
| "parent_category_id": 3 | |
| }, { | |
| "id": 7, | |
| "name": (+"Modules financiers"), | |
| "description": (+"Facturation, Banque/Caisse, Dons, Comptabilité, Comptabilité avancée"), | |
| "position": 7, | |
| "parent_category_id": 3 | |
| }, { | |
| "id": 8, | |
| "name": (+"Autres modules natifs"), | |
| "description": (+"Projets/Travail collaboratif, GED, Outils multi-modules, etc."), | |
| "position": 8, | |
| "parent_category_id": 3 | |
| }, { | |
| "id": 9, | |
| "name": (+"Modules externes du DoliStore"), | |
| "description": (+"Obtenir de l’aide sur l’utilisation des modules disponibles sur la platforme DoliStore."), | |
| "position": 9, | |
| "parent_category_id": 3 | |
| }, { | |
| "id": 10, | |
| "name": (+"Retours d'expérience"), | |
| "description": (+"Partager un retour d'expérience sur la mise en place de l'ERP/CRM Dolibarr au sein de votre entreprise."), | |
| "position": 10, | |
| "parent_category_id": 3 | |
| }, { | |
| "id": 11, | |
| "name": (+"Maintenir mon Dolibarr"), | |
| "description": (+"Obtenir de l’aide sur la mise à jour de Dolibarr vers une version plus récente, la migration d’une instance Dolibarr vers un nouveau serveur, les opérations de sauvegarde et de restauration (fichiers et base de données) et plus généralement, tout ce qui touche à la configuration et à la maintenance système de la machine hébergeant votre instance Dolibarr (cron, permissions, etc.)."), | |
| "position": 11, | |
| "parent_category_id": 0 | |
| }, { | |
| "id": 12, | |
| "name": (+"Développer pour Dolibarr"), | |
| "description": (+"Obtenir de l’aide concernant le développement de modules spécifiques ou thèmes pour Dolibarr mais également sur la façon d’utiliser les modules API REST et SOAP pour interconnecter Dolibarr avec une application externe."), | |
| "position": 12, | |
| "parent_category_id": 0 | |
| }, { | |
| "id": 13, | |
| "name": (+"Discuter entre Dolibarriens"), | |
| "description": (+"Cet espace de discussion vous permet d’échanger avec les autres utilisateurs de Dolibarr sur des sujets sans lien avec Dolibarr."), | |
| "position": 13, | |
| "parent_category_id": 0 | |
| } ] | |
| created = 0 | |
| total = categories.count | |
| categories.each do |c| | |
| h = { id: c[:id], name: c[:name], description: c[:description], position: c[:position].to_i } | |
| if c[:parent_category_id].to_i > 0 | |
| h[:parent_category_id] = category_id_from_imported_category_id(c[:parent_category_id]) | |
| end | |
| new_category = create_category(h, h[:id]) | |
| created_category(new_category) | |
| created += 1 | |
| print_status(created, total, get_start_time("categories")) | |
| end | |
| end | |
| def import_posts | |
| puts '', "Retreiving attachments info from kunena_attachments table..." | |
| attachments_records = @client.query(" | |
| SELECT id, mesid, folder, filename | |
| FROM #{KUNENA_PREFIX}kunena_attachments; | |
| ", cache_rows: false).to_a | |
| attachments = Hash.new | |
| attachments_records.each do |a| | |
| unless attachments[a['mesid']].is_a?(Array) | |
| attachments[a['mesid']] = [] | |
| end | |
| attachments[a['mesid']] << {"id" => a['id'], "folder" => a['folder'], "filename" => a['filename']} | |
| end | |
| puts '', "Importing topics and posts..." | |
| mapping_categories = { | |
| "14" => category_id_from_imported_category_id(1), | |
| "3" => category_id_from_imported_category_id(2), | |
| "8" => category_id_from_imported_category_id(11), | |
| "12" => category_id_from_imported_category_id(3), | |
| "5" => category_id_from_imported_category_id(3), | |
| "527" => category_id_from_imported_category_id(3), | |
| "529" => category_id_from_imported_category_id(3), | |
| "614" => category_id_from_imported_category_id(3), | |
| "15" => category_id_from_imported_category_id(10), | |
| "11" => category_id_from_imported_category_id(3), | |
| "514" => category_id_from_imported_category_id(3), | |
| "604" => 3, | |
| "531" => category_id_from_imported_category_id(9), | |
| "511" => category_id_from_imported_category_id(12), | |
| "510" => category_id_from_imported_category_id(4), | |
| "505" => category_id_from_imported_category_id(9), | |
| "507" => category_id_from_imported_category_id(8), | |
| "509" => category_id_from_imported_category_id(9), | |
| "526" => category_id_from_imported_category_id(9), | |
| "543" => category_id_from_imported_category_id(5), | |
| "545" => category_id_from_imported_category_id(4), | |
| "546" => category_id_from_imported_category_id(4), | |
| "547" => category_id_from_imported_category_id(7), | |
| "548" => category_id_from_imported_category_id(6), | |
| "549" => category_id_from_imported_category_id(8), | |
| "551" => category_id_from_imported_category_id(3), | |
| "553" => category_id_from_imported_category_id(8), | |
| "554" => category_id_from_imported_category_id(3), | |
| "556" => category_id_from_imported_category_id(11), | |
| "557" => category_id_from_imported_category_id(12), | |
| "558" => category_id_from_imported_category_id(8), | |
| "585" => category_id_from_imported_category_id(2), | |
| "586" => category_id_from_imported_category_id(2), | |
| "587" => category_id_from_imported_category_id(2), | |
| "607" => category_id_from_imported_category_id(2), | |
| "608" => category_id_from_imported_category_id(2), | |
| "537" => category_id_from_imported_category_id(3), | |
| "615" => category_id_from_imported_category_id(3), | |
| "616" => category_id_from_imported_category_id(3), | |
| "618" => category_id_from_imported_category_id(3), | |
| "515" => category_id_from_imported_category_id(3), | |
| "516" => category_id_from_imported_category_id(3), | |
| "524" => category_id_from_imported_category_id(3), | |
| "528" => category_id_from_imported_category_id(3), | |
| "522" => category_id_from_imported_category_id(3), | |
| "609" => category_id_from_imported_category_id(12), | |
| "610" => category_id_from_imported_category_id(12), | |
| "611" => category_id_from_imported_category_id(12), | |
| "532" => category_id_from_imported_category_id(9), | |
| "534" => category_id_from_imported_category_id(9), | |
| "536" => category_id_from_imported_category_id(8), | |
| "605" => category_id_from_imported_category_id(8), | |
| "606" => category_id_from_imported_category_id(8), | |
| "613" => category_id_from_imported_category_id(9), | |
| "544" => category_id_from_imported_category_id(5), | |
| "550" => category_id_from_imported_category_id(5), | |
| "579" => category_id_from_imported_category_id(5), | |
| "598" => category_id_from_imported_category_id(5), | |
| "580" => category_id_from_imported_category_id(4), | |
| "589" => category_id_from_imported_category_id(4), | |
| "592" => category_id_from_imported_category_id(4), | |
| "594" => category_id_from_imported_category_id(4), | |
| "595" => category_id_from_imported_category_id(4), | |
| "596" => category_id_from_imported_category_id(4), | |
| "597" => category_id_from_imported_category_id(4), | |
| "600" => category_id_from_imported_category_id(4), | |
| "581" => category_id_from_imported_category_id(4), | |
| "582" => category_id_from_imported_category_id(4), | |
| "572" => category_id_from_imported_category_id(7), | |
| "573" => category_id_from_imported_category_id(7), | |
| "574" => category_id_from_imported_category_id(7), | |
| "575" => category_id_from_imported_category_id(7), | |
| "576" => category_id_from_imported_category_id(7), | |
| "577" => category_id_from_imported_category_id(7), | |
| "588" => category_id_from_imported_category_id(7), | |
| "602" => category_id_from_imported_category_id(7), | |
| "568" => category_id_from_imported_category_id(6), | |
| "569" => category_id_from_imported_category_id(6), | |
| "570" => category_id_from_imported_category_id(6), | |
| "571" => category_id_from_imported_category_id(6), | |
| "593" => category_id_from_imported_category_id(6), | |
| "552" => category_id_from_imported_category_id(3), | |
| "591" => category_id_from_imported_category_id(3), | |
| "561" => category_id_from_imported_category_id(8), | |
| "567" => category_id_from_imported_category_id(3), | |
| "590" => category_id_from_imported_category_id(8), | |
| "603" => category_id_from_imported_category_id(8), | |
| "555" => category_id_from_imported_category_id(7), | |
| "559" => category_id_from_imported_category_id(11), | |
| "578" => category_id_from_imported_category_id(7), | |
| "584" => category_id_from_imported_category_id(8), | |
| "560" => category_id_from_imported_category_id(8), | |
| "601" => category_id_from_imported_category_id(8), | |
| } | |
| #SELECT id FROM #{KUNENA_PREFIX}kunena_topics | |
| #21428,28012,28637,40680,42506,56540,56838,59279,59433,61168,62661,62681,62730,62844,62846,64351,64371,64380,64475,64599,64629,64778 | |
| total_count = @client.query(" | |
| SELECT COUNT(*) count | |
| FROM #{KUNENA_PREFIX}kunena_messages m | |
| WHERE thread IN( | |
| SELECT id FROM #{KUNENA_PREFIX}kunena_topics | |
| ); | |
| ").first['count'] | |
| batches(BATCH_SIZE) do |offset| | |
| results = @client.query(" | |
| SELECT m.id id, | |
| m.thread thread, | |
| m.parent parent, | |
| m.catid catid, | |
| m.userid userid, | |
| s.icon_id = 8 AS solved, | |
| s.ordering AS pinned, | |
| s.locked, | |
| s.subject subject, | |
| m.time time, | |
| t.message message | |
| FROM #{KUNENA_PREFIX}kunena_messages m | |
| INNER JOIN #{KUNENA_PREFIX}kunena_messages_text t ON m.id = t.mesid | |
| INNER JOIN #{KUNENA_PREFIX}kunena_topics s ON m.thread = s.id | |
| WHERE thread IN( | |
| SELECT id FROM #{KUNENA_PREFIX}kunena_topics | |
| ) | |
| ORDER BY m.id | |
| LIMIT #{BATCH_SIZE} | |
| OFFSET #{offset}; | |
| ", cache_rows: false) | |
| break if results.size < 1 | |
| next if all_records_exist? :posts, results.map { |p| p['id'].to_i } | |
| create_posts(results, total: total_count, offset: offset) do |m| | |
| skip = false | |
| post = {} | |
| post[:id] = m['id'] | |
| post[:user_id] = user_id_from_imported_user_id(m['userid']) || -1 | |
| # Break line after [code] only for lines that starts by [code] and followed by anything before a line break | |
| m["message"] = m["message"].gsub(/(\[code\])(.+)[^\n]+/, "\\1\n\\2") | |
| # Before making any alteration to the post text, we need to remove [code][/code] | |
| # this is because we do not want to alter code blocks | |
| codeblocks = [] | |
| codeblocks_to_remove = m["message"].scan(/(\[code\].+?\[\/code\])/m) | |
| codeblock_number = 0 | |
| for codeblock in codeblocks_to_remove | |
| codeblock_value = codeblock[0] | |
| codeblocks << codeblock_value | |
| m["message"] = m["message"].gsub(codeblock_value, "[!--###kunena-codeblock-#{codeblock_number}###--]") | |
| codeblock_number += 1 | |
| end | |
| # Escape some commonly used chars to not be wrongly parsed as Markdown | |
| m["message"] = m["message"].gsub(/>/, "\\>") | |
| m["message"] = m["message"].gsub(/\* /, "\\\\* ") | |
| m["message"] = m["message"].gsub(/- /, "\\- ") | |
| m["message"] = m["message"].gsub(/\+ /, "\\\\+ ") | |
| # Convert some emojis shortcuts to preserve custom emojis previously used | |
| m["message"] = m["message"].gsub(/:\)/, ":happy:") | |
| m["message"] = m["message"].gsub(/:\(/, ":unhappy:") | |
| m["message"] = m["message"].gsub(/:P/, ":tongue:") | |
| m["message"] = m["message"].gsub(/:S/, ":confused:") | |
| m["message"] = m["message"].gsub(/B\)/, ":sunglasses:") | |
| # We keep a list of ids of attachments and pathsof legacy attachment to be able to add | |
| # non inserted attachment on the bottom of the post | |
| legacy_attachments_path = m["message"].scan(/\[(?:(?:img)|(?:file)).*?\](?:https?:\/\/)?www\.dolibarr\.(?:(?:fr)|(?:org))\/media\/kunena\/attachments(\/legacy\/(?:(?:images)|(?:files))\/.+?)\[\/(?:(?:img)|(?:file)).*?\]/).flatten | |
| attachments_ids = m["message"].scan(/\[attachment=([0-9]+)\].+?\[\/attachment\]/).flatten | |
| # Replace [attachment] BBCode tag by native Discourse attachment | |
| user_id = m['userid'] | |
| attachments_to_alter = m["message"].scan(/(\[attachment=[0-9]+\](.+?)\[\/attachment\])/) | |
| for attachment in attachments_to_alter | |
| original_attachment_to_replace = attachment[0] | |
| filename = attachment[1] | |
| path = "#{ATTACHMENTS_LOCAL_FOLDER}/#{user_id}/#{filename}" | |
| if File.exists?(path) | |
| upload = create_upload(post[:user_id], path, filename) | |
| if upload.present? && upload.persisted? | |
| upload_markdown = html_for_upload(upload, filename) | |
| m["message"] = m["message"].gsub(original_attachment_to_replace, "\n\n#{upload_markdown}") | |
| end | |
| else | |
| m["message"] = m["message"].gsub(original_attachment_to_replace, "> **La pièce jointe #{filename} est absente ou indisponible**") | |
| end | |
| end | |
| # Replace legacy attachment using [img] and [file] BBCode tags by native Discourse attachment | |
| attachments_to_alter = m["message"].scan(/(\[(?:(?:img)|(?:file)).*?\](?:https?:\/\/)?www\.dolibarr\.(?:(?:fr)|(?:org))\/media\/kunena\/attachments(\/legacy\/(?:(?:images)|(?:files))\/(.+?))\[\/(?:(?:img)|(?:file)).*?\])/) | |
| for attachment in attachments_to_alter | |
| original_attachment_to_replace = attachment[0] | |
| path = attachment[1] | |
| filename = attachment[2] | |
| path = "#{ATTACHMENTS_LOCAL_FOLDER}#{path}" | |
| if File.exists?(path) | |
| upload = create_upload(post[:user_id], path, filename) | |
| if upload.present? && upload.persisted? | |
| upload_markdown = html_for_upload(upload, filename) | |
| m["message"] = m["message"].gsub(original_attachment_to_replace, "\n\n#{upload_markdown}") | |
| end | |
| else | |
| m["message"] = m["message"].gsub(original_attachment_to_replace, "\n> **La pièce jointe #{filename} est absente ou indisponible**") | |
| end | |
| end | |
| # Print out unattached attachment at the bottom of the post | |
| if attachments.has_key? m["id"] | |
| attachments_to_add = "\n\n > Pièces jointes :\n" | |
| for attachment in attachments[m["id"]] | |
| filename = attachment["filename"] | |
| path = attachment["folder"].gsub("media/kunena/attachments", "") | |
| file_already_in_post_text = (attachments_ids.include?(attachment["id"].to_s) or legacy_attachments_path.include?("#{path}/#{filename}")) | |
| path = "#{ATTACHMENTS_LOCAL_FOLDER}#{path}/#{filename}" | |
| unless file_already_in_post_text | |
| if File.exists?(path) | |
| upload = create_upload(post[:user_id], path, filename) | |
| if upload.present? && upload.persisted? | |
| upload_markdown = attachment_html(upload, filename) | |
| attachments_to_add += "> * #{upload_markdown}\n" | |
| end | |
| end | |
| end | |
| end | |
| unless attachments_to_add == "\n\n > Pièces jointes :\n" | |
| m["message"] += attachments_to_add | |
| end | |
| end | |
| # Deal with quote block (add a line break after [quote] BBCode tag if there is none) | |
| m["message"] = m["message"].gsub(/\[quote([^\]]*)\]/, "\n\[quote\\1\]\n") | |
| m["message"] = m["message"].gsub(/(.*)\[\/quote\]/, "\\1\n\[/quote\]\n") | |
| # Replacing old posts reference in quote blocks by new ones | |
| posts_references_to_alter = m["message"].scan(/(\[quote="(.*)" post=(\d{1,6}))/) | |
| for result in posts_references_to_alter | |
| origin_pattern_to_replace = result[0] | |
| origin_post_user = result[1] | |
| origin_post_id = result[2] | |
| referenced_topic = topic_lookup_from_imported_post_id(origin_post_id) | |
| if referenced_topic | |
| topic_id = referenced_topic[:topic_id] | |
| post_number = referenced_topic[:post_number] | |
| m["message"] = m["message"].gsub(origin_pattern_to_replace, "\[quote=\"" + origin_post_user + ", post:" + post_number.to_s + ", topic:" + topic_id.to_s + "\"") | |
| end | |
| end | |
| # Removing remaining posts references in quote blocks for posts that does not exists anymore | |
| m["message"] = m["message"].gsub(/(\[quote[^ ]*) post=\d{1,6}(\])/, "\\1\\2") | |
| # Replacing old posts & topics forum URLs by new ones | |
| posts_url_to_alter = m["message"].scan(/((?:https?:\/\/)?www\.dolibarr\.fr\/forum\/[^\/]+\/(\d[^-]+)-[^# \n]+(?:#(\d+))?)/) | |
| for result in posts_url_to_alter | |
| origin_url = result[0] | |
| origin_topic_id = result[1] | |
| origin_post_id = result[2] | |
| referenced_topic = {} | |
| if origin_post_id.nil? | |
| origin_post_id = @client.query("SELECT id FROM #{KUNENA_PREFIX}kunena_messages WHERE thread = #{origin_topic_id} ORDER BY id LIMIT 1;").first['id'] rescue nil | |
| end | |
| unless origin_post_id.nil? | |
| referenced_topic = topic_lookup_from_imported_post_id(origin_post_id) | |
| if referenced_topic | |
| m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}" + referenced_topic[:url] ) | |
| else | |
| m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/#this-topic-no-longer-exists" ) | |
| end | |
| else | |
| m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/#this-topic-no-longer-exists" ) | |
| end | |
| end | |
| # Replacing old forum categories URL by new ones | |
| categories_url_to_alter = m["message"].scan(/((?:https?:\/\/)?www\.dolibarr\.fr\/forum\/([^\/ \n\]]+))[ \]]/) | |
| for result in categories_url_to_alter | |
| origin_url = result[0] | |
| origin_category_slug = result[1] | |
| origin_category_id = @client.query("SELECT id FROM #{KUNENA_PREFIX}kunena_categories WHERE alias = '#{origin_category_slug}' LIMIT 1;").first['id'] rescue nil | |
| unless origin_category_id.nil? | |
| category_id = mapping_categories[origin_category_id.to_s] | |
| category = Category.find_by(id: category_id) rescue nil | |
| unless category.nil? | |
| m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/c/" + category.slug ) | |
| else | |
| m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/#this-category-no-longuer-exists" ) | |
| end | |
| else | |
| m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/#this-category-no-longuer-exists" ) | |
| end | |
| end | |
| # Replacing old profile URL by new ones | |
| profile_url_to_alter = m["message"].scan(/((?:https?:\/\/)?www\.dolibarr\.fr\/forum\/profile\/userid-(\d+))/) | |
| for result in profile_url_to_alter | |
| origin_url = result[0] | |
| origin_user_id = result[1] | |
| user = find_user_by_import_id(origin_user_id) rescue nil | |
| unless user.nil? | |
| m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/u/" + user.username ) | |
| else | |
| m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/#this-user-no-longuer-exists" ) | |
| end | |
| end | |
| # Converting [table] definitions into Markdown syntax | |
| tables_to_alter = m["message"].scan(/(\[table\](.+?)\[\/table\])/m) | |
| for table in tables_to_alter | |
| original_table_to_replace = table[0] | |
| markdown_converted_table = "" | |
| lines = original_table_to_replace.scan(/\[tr\](.+?)\[\/tr\]/m) | |
| line_number = 1 | |
| for line in lines | |
| line_value = line[0] | |
| columns = line_value.scan(/\[td\](.+?)\[\/td\]/m) | |
| total_columns = columns.count | |
| column_number = 1 | |
| for column in columns | |
| column_value = column[0] | |
| if column_number == total_columns | |
| markdown_converted_table += "| " + column_value + " |\n" | |
| if line_number == 1 | |
| for i in 1..total_columns | |
| markdown_converted_table += "| - " | |
| end | |
| markdown_converted_table += "|\n" | |
| end | |
| else | |
| markdown_converted_table += "| " + column_value.gsub("|", "\|") + " " | |
| end | |
| column_number += 1 | |
| end | |
| line_number += 1 | |
| end | |
| m["message"] = m["message"].gsub(original_table_to_replace, markdown_converted_table ) | |
| end | |
| # Break line after [ul] only for lines that starts by [ul] and followed by [li] tag on the same line | |
| m["message"] = m["message"].gsub(/(^\[ul\])( *\[li\].+$)/m, "\\1\n\\2") | |
| # Replace [strike][/strike] BBCode tag by <s></s> HTML equivalent for markdown parser | |
| m["message"] = m["message"].gsub(/\[strike\]([^\[]*)\[\/strike\]/, "<s>\\1</s>") | |
| # Break line before/after ---------- (horizontal line) | |
| m["message"] = m["message"].gsub(/(-{5,200})[^ \S]/m, "\n\\1\n") | |
| # We restore the previously removed [code] blocks after all replacements has been made | |
| codeblocks_to_insert = m["message"].scan(/(\[!--###kunena-codeblock-(\d+)###--\])/m) | |
| for codeblock in codeblocks_to_insert | |
| codeblock_marker_to_replace = codeblock[0] | |
| codeblock_number_to_replace = codeblock[1] | |
| m["message"] = m["message"].gsub(codeblock_marker_to_replace, codeblocks[codeblock_number_to_replace.to_i]) | |
| codeblock_number += 1 | |
| end | |
| post[:raw] = m["message"] | |
| post[:created_at] = Time.zone.at(m['time']) | |
| if m['parent'] == 0 | |
| if mapping_categories.key?(m['catid'].to_s) | |
| post[:category] = mapping_categories[m['catid'].to_s] | |
| else | |
| puts "Catégorie " + m['catid'].to_s + " non trouvée dans le mapping." | |
| post[:category] = 1 | |
| end | |
| post[:title] = m['subject'] | |
| else | |
| parent = topic_lookup_from_imported_post_id(m['parent']) | |
| if parent | |
| post[:topic_id] = parent[:topic_id] | |
| post[:reply_to_post_number] = parent[:post_number] if parent[:post_number] > 1 | |
| else | |
| # If we got here, this is probably because a user used the Reply button at the bottom of a topic's post | |
| # and not the reply button of the topic itselft. In that case, we fetch the post id of the first post | |
| # of the topic and use it instead | |
| thread_id = m['thread'] | |
| first_topic_post = @client.query(" | |
| SELECT id, parent FROM #{KUNENA_PREFIX}kunena_messages | |
| WHERE thread = #{thread_id} | |
| ORDER BY id LIMIT 1; | |
| ") | |
| parent_post_id = first_topic_post.first['id'] | |
| parent = topic_lookup_from_imported_post_id(parent_post_id) | |
| if parent | |
| post[:topic_id] = parent[:topic_id] | |
| post[:reply_to_post_number] = parent[:post_number] if parent[:post_number] > 1 | |
| else | |
| # If the first post of the topic has been deleted, we use the first remaining available (default Kunena behaviour) | |
| if first_topic_post.first['parent'] != 0 | |
| if mapping_categories.key?(m['catid'].to_s) | |
| post[:category] = mapping_categories[m['catid'].to_s] | |
| else | |
| puts "SC Catégorie " + m['catid'].to_s + " non trouvée dans le mapping." | |
| post[:category] = 1 | |
| end | |
| post[:title] = m['subject'] | |
| else | |
| puts "Parent post #{parent_post_id} doesn't exist. Skipping #{m["id"]}: #{m["subject"][0..40]}" | |
| skip = true | |
| end | |
| end | |
| end | |
| end | |
| if m["pinned"].to_i == 1 and m['parent'].to_i == 0 | |
| post[:pinned_until] = DateTime.now.next_year(1000).to_time | |
| post[:pinned_at] = Time.zone.at(m['time']) | |
| end | |
| post[:post_create_action] = proc do |action_post| | |
| topic_id = action_post.topic.id | |
| Jobs.enqueue_at(topic.pinned_until, :unpin_topic, topic_id: topic_id) if action_post.topic.pinned_until | |
| title_contain_solved_pattern = m['subject'] =~ /(?:Resolu)|(?:Résolu)|(?:Solved)/i | |
| if (m['solved'].to_i == 1 or title_contain_solved_pattern) and m['parent'].to_i == 0 | |
| time = Time.zone.at(m['time']) | |
| DB.exec <<-SQL | |
| INSERT INTO topic_custom_fields (name, value, topic_id, created_at, updated_at) | |
| VALUES ('accepted_answer_post_id', -1, #{topic_id}, '#{time}', '#{time}') | |
| SQL | |
| end | |
| if m['locked'].to_i == 1 | |
| DB.exec <<-SQL | |
| UPDATE topics SET closed = true WHERE id = #{topic_id} | |
| SQL | |
| end | |
| end | |
| skip ? nil : post | |
| end | |
| end | |
| end | |
| def import_likes | |
| puts "", "Importing post likes..." | |
| total_count = @client.query("SELECT COUNT(*) count FROM #{KUNENA_PREFIX}kunena_thankyou;").first['count'] | |
| count = 0 | |
| batches(BATCH_SIZE) do |offset| | |
| likes = @client.query(" | |
| SELECT postid, userid | |
| FROM #{KUNENA_PREFIX}kunena_thankyou | |
| LIMIT #{BATCH_SIZE} | |
| OFFSET #{offset}; | |
| ", cache_rows: false).to_a | |
| break if likes.empty? | |
| likes.each do |result| | |
| print_status(count += 1, total_count, get_start_time("import_likes")) | |
| next unless user_id = user_id_from_imported_user_id(result["userid"]) | |
| next unless post_id = post_id_from_imported_post_id(result["postid"]) | |
| next unless user = User.find_by(id: user_id) | |
| next unless post = Post.find_by(id: post_id) | |
| PostActionCreator.like(user, post) rescue nil | |
| end | |
| end | |
| end | |
| end | |
| ImportScripts::Kunena.new.perform |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment