Sync Gmail to Outlook

.gitignore

Gemfile.lock
credentials.json
ews_config.yaml
token.yaml

README.md

This will sync all of your emails, except for Trash and Spam, from Gmail to Outlook. It will create Outlook "categories" based on the Gmail "labels" attached to each message.

Edit the ews_config.yaml before you begin

---
endpoint: <YOUR EWS ENDPOINT... something like 'https://outlook.office365.com/EWS/Exchange.asmx'>
user: <your outlook email>
pass: <your outlook password>
destination_folder: <Folder name you want to import messages into. It will be created if it doesn't exist>

Install the dependencies

bundle

Enable the Gmail API

You need to enable the Gmail API for your user here: https://developers.google.com/gmail/api/quickstart/ruby

Run the sync

ruby gmail_convert.rb

Gemfile

# frozen_string_literal: true

source "https://rubygems.org"

git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }

gem "google-apis-gmail_v1"
gem "pry"
gem "viewpoint", ">= 1.1.1"
gem "rubocop", "~> 0.89.1"

gmail_convert.rb

#!/usr/bin/env ruby

require "google/apis/gmail_v1"
require "googleauth"
require "googleauth/stores/file_token_store"
require "fileutils"

require 'viewpoint'
require 'base64'
require 'yaml'

module GmailConverter
  class GmailService
    OOB_URI = "urn:ietf:wg:oauth:2.0:oob".freeze
    APPLICATION_NAME = "Gmail Converter".freeze
    CREDENTIALS_PATH = "credentials.json".freeze
    # The file token.yaml stores the user's access and refresh tokens, and is
    # created automatically when the authorization flow completes for the first
    # time.
    TOKEN_PATH = "token.yaml".freeze
    SCOPE = Google::Apis::GmailV1::AUTH_GMAIL_READONLY

    attr_reader :user_id

    def initialize(user_id: 'me')
      @user_id = user_id
    end

    def label(label_id)
      labels[label_id]
    end

    def labels
      @labels ||= service.list_user_labels(user_id).labels.each_with_object({}) { |l, a| a[l.id] = l.name }
    end

    def service
      @service ||= Google::Apis::GmailV1::GmailService.new.tap do |service|
        service.client_options.application_name = APPLICATION_NAME
        service.authorization = authorize!
      end
    end

    def thread(thread_id)
      service.get_user_thread user_id, thread_id, format: :minimal
    end

    def message(message_id, &block)
      service.get_user_message user_id, message_id, format: :raw, &block
    end

    def messages(**args)
      service.list_user_messages user_id, include_spam_trash: false, **args
    end

    private

    ##
    # Ensure valid credentials, either by restoring from the saved credentials
    # files or intitiating an OAuth2 authorization. If authorization is required,
    # the user's default browser will be launched to approve the request.
    #
    # @return [Google::Auth::UserRefreshCredentials] OAuth2 credentials
    def authorize!
      client_id = Google::Auth::ClientId.from_file CREDENTIALS_PATH
      token_store = Google::Auth::Stores::FileTokenStore.new file: TOKEN_PATH
      authorizer = Google::Auth::UserAuthorizer.new client_id, SCOPE, token_store
      user_id = "default"
      credentials = authorizer.get_credentials user_id
      if credentials.nil?
        url = authorizer.get_authorization_url base_url: OOB_URI
        puts "Open the following URL in the browser and enter the " \
            "resulting code after authorization:\n" + url
        code = gets
        credentials = authorizer.get_and_store_credentials_from_code(
          user_id: user_id, code: code, base_url: OOB_URI
        )
      end
      credentials
    end
  end

  class EwsService
    PidTagMessageFlags = {
      mfRead:         0x00000001,
      mfUnsent:       0x00000008,
      mfResend:       0x00000080,
      mfUnmodified:   0x00000002,
      mfSubmitted:    0x00000004,
      mfHasAttach:    0x00000010,
      mfFromMe:       0x00000020,
      mfFAI:          0x00000040,
      mfNotifyRead:   0x00000100,
      mfNotifyUnread: 0x00000200,
      mfEverRead:     0x00000400,
      mfInternet:     0x00002000,
      mfUntrusted:    0x00008000
    }.freeze

    attr_reader :service

    def initialize(endpoint, user, pass)
      @service = Viewpoint::EWSClient.new endpoint, user, pass
    end

    def create_request(destination_folder_id)
      {
        message_disposition:  "SaveOnly",
        saved_item_folder_id: { id: destination_folder_id },
        items:                []
      }
    end

    def folder_for(name)
      service.get_folder_by_name(name) || service.make_folder(name)
    end
  end

  class Main
    EWS_CONFIG = "ews_config.yaml".freeze

    attr_reader :google, :ews, :errors

    def initialize(max_messages: nil, thread: nil, start_page: nil)
      @max_messages = max_messages
      @thread = thread
      @start_page = start_page
      @google = GmailService.new
      @ews_config = YAML.load(File.read(EWS_CONFIG))
      @ews = EwsService.new @ews_config['endpoint'], @ews_config['user'], @ews_config['pass']
      @errors = []
    end

    def run!
      if @thread
        migrate_thread!
      else
        migrate_messages!
      end
    end

    def migrate_thread!
      ews_request = ews.create_request(destination_folder.id)
      copy_messages google.thread(@thread).messages
    end

    def copy_messages(messages)
      ews_request = ews.create_request(destination_folder.id)

      messages.each do |message|
        google.message(message.id) do |res, err|
          if err
            errors << err
          else
            ews_request[:items] << format_message(res)
          end
        end
      end
      ews.service.ews.create_item ews_request
    end

    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
    def migrate_messages!
      count = 0
      retries = 0
      page_token = @start_page
      loop do
        max_results = @max_messages && (@max_messages - count) < 50 ? @max_messages - count : 50
        messages = google.messages(page_token: page_token, max_results: max_results)
        puts "Current page: #{page_token || 'first page'}, Next page: #{messages.next_page_token}"
        copy_messages(messages.messages)
        count += messages.messages.length
        page_token = messages.next_page_token
        puts "#{messages.messages.length} of #{count} total messages migrated."
        break if page_token.nil? || (@max_messages && count >= @max_messages)

        retries = 0
      rescue StandardError => e
        puts "Error#(#{e.message})"
        if retries < 5
          puts "retrying..."
          retries += 1
          sleep 10
          retry
        end
        raise
      end
    end
    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity

    private

    def destination_folder
      @destination_folder ||= ews.folder_for(@ews_config['destination_folder'])
    end

    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
    def format_message(raw_message)
      mime_content = raw_message.raw
      message_flags = EwsService::PidTagMessageFlags[:mfInternet]
      unless raw_message.label_ids.include?("UNREAD")
        message_flags |= EwsService::PidTagMessageFlags[:mfRead]
      end
      if raw_message.label_ids.include?("SENT") || raw_message.label_ids.include?("CHAT")
        message_flags |= EwsService::PidTagMessageFlags[:mfFromMe]
        lines = mime_content.lines
        new_line = "Received: by 127.0.0.1 with EWS id #{raw_message.id};\r\n\t#{Time.at((raw_message.internal_date / 1000).round).strftime('%a, %d %b %Y %H:%M:%S %z (%Z)')}\r\n"
        lines.insert(1, new_line)
        mime_content = lines.join("")
      end

      {
        message: {
          importance:          "Normal",
          mime_content:        Base64.encode64(mime_content),
          extended_properties: [
            {
              extended_field_uri: {
                property_tag:  "0x0E07",
                property_type: "Integer"
              },
              value:              message_flags
            },
            {
              extended_field_uri: {
                property_tag:  "0x0071",
                property_type: "Binary"
              },
              value:              raw_message.thread_id
            }
          ],
          categories:          raw_message.label_ids.map { |id| google.label(id) }
        }
      }
    end
    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
  end
end

#GmailConverter::Main.new(max_messages: 10).run!
GmailConverter::Main.new.run!