curlコマンドを使ったSlackチャンネルのエクスポート

0_README.md

curlコマンドを使ったSlackチャンネルのエクスポート

Slack apiで特定のチャンネルのエクスポートを行うスクリプト

実行準備

1. Slack api Tokenの取得

• 参考 https://tearoom6.hateblo.jp/entry/2020/05/25/091422

2. expoort_slack_channel.rbへのTokenの設定 L:9 TOKEN=''の部分を変更

実行例

$ ruby expoort_slack_channel.rb channel1 channel2

実行結果

slack_logs/が作成される

$ ls slack_logs/
channels.json  members.json  channel1/ channel2/
$ ls slack_logs/channel1
members.json members.csv messages.json messages.json

curlを使った経緯

はじめnet::httpやslack-apiを使ったが、初め数回はレスポンスが返ってきたが、しばらくしたらinvalid_authのエラーになった。 その場合でもcurlなら大丈夫だった。何故？

ファイルのダウンロード

仕様が変わったのか、url_private_downloadだとダウンロードページのhtmlとなる。 苦肉の策としてブラウザを叩いてダウンロードするようにした。 messagesの取得後にまとめてダウンロードする方が良いかも。

参考

• TOKEN取得
  • https://tearoom6.hateblo.jp/entry/2020/05/25/091422
• Slack API
  • https://api.slack.com/methods/users.list
  • https://api.slack.com/methods/conversations.list
  • https://api.slack.com/methods/conversations.history
  • https://api.slack.com/methods/conversations.replies

export_slack_channel.rb

require 'open3'
require 'pp'
require 'json'
require 'fileutils'
require 'oj'
require 'csv'
require 'cgi/util'

BASE = 'https://slack.com/api'
# see https://tearoom6.hateblo.jp/entry/2020/05/25/091422
TOKEN = 'xoxp-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
DELAY = 0.3

# curlでSlack APIを叩く
def slack_api_get(path, params = {})
  sleep DELAY
  auth = "Authorization: Bearer #{TOKEN}"
  cmd = %W[/usr/bin/curl -X POST -H "#{auth}" "#{BASE}#{path}"]
  params.each do |k,v|
    cmd << "-F"
    cmd << "#{k}=#{v}"
  end
  o = `#{cmd.join(" ")} 2>/dev/null`
  unless $?.success?
    puts o
    exit 1
  end
  res = JSON.parse(o)
  unless res['ok']
    pp res
    exit 1
  end
  next_cursor = res.dig('response_metadata', 'next_cursor')
  [res, next_cursor]
end

# 添付ファイルの取得
def slack_download(url, path)
  # url_private_downloadでは直接アクセスできず、ダウンロードページの表示となるため
  # 面倒だがブラウザでダウンロードする
  cmd = %W[/opt/google/chrome/chrome "#{url}"]
  `#{cmd.join(" ")} 2>/dev/null`
end

# ユーザ一覧取得
def users_list(params = {})
  members = []
  loop do
    res, cursor = slack_api_get('/users.list', params)
    members << res['members']
    if cursor and !cursor.empty?
      params[:cursor] = cursor
    else
      return members.flatten
    end
  end
end

# チャンネル一覧取得
def conversations_list(params = {})
  list = []
  loop do
    res, cursor = slack_api_get('/conversations.list', params)
    list << res['channels']
    if cursor and !cursor.empty?
      params[:cursor] = cursor
    else
      return list.flatten
    end
  end
end

# チャンネルメンバー一覧取得
def conversations_members(params = {})
  list = []
  loop do
    res, cursor = slack_api_get('/conversations.members', params)
    list << res['members']
    if cursor and !cursor.empty?
      params[:cursor] = cursor
    else
      return list.flatten
    end
  end
end

# チャンネルログ取得 
def conversations_history(channel, members, params = {})
  messages = []
  params.merge!(channel: channel)
  loop do
    res, cursor = slack_api_get('/conversations.history', params)
    print '.'
    res['messages'].each do |msg|
      messages << msg
      modify_message(msg, members)
      if msg['thread_ts'] && !msg['thread_ts'].empty?
        messages << conversations_replies(channel, members, msg['thread_ts'])
      end
    end
    if cursor and !cursor.empty?
      params[:cursor] = cursor
    else
      return messages.flatten.sort! do |a,b| a['ts'] <=> b['ts'] end
    end
  end
end

# メンバーIDを名前に変換、テキストのunescape
def modify_message(msg, members)
  msg['user'] = members[msg['user']]['name'] if members[msg['user']]
  msg['text'].gsub!(%r|<@(\w+)?>|){ "@#{members[$1]['name']}" }
end

# スレッドメッセージ取得
def conversations_replies(channel, members, thread_ts, params = {})
  messages = []
  params.merge!(channel: channel, ts: thread_ts)
  loop do
    res, cursor = slack_api_get('/conversations.replies', params)
    messages << res['messages']
    if cursor and !cursor.empty?
      params[:cursor] = cursor
    else
      return messages
    end
  end
end

# 添付ファイル取得
def download_files(messages, dir)
  messages.each do |msg|
    if msg['files']
      msg['files'].each do |f|
        _path = File.join(dir, f['name'])
        path = _path.dup
        # ファイル名重複の場合
        1.step do |i|
          if File.exist?(path)
            path = "#{_path}_#{i}"
          else
            break
          end
        end
        slack_download(f['url_private_download'], path)
      end
    end
  end
end

def export_csv(messages, path)
  CSV.open(path, 'w') do |csv|
    csv << ['timestamp','user','text','file1','file2','file3']
    messages.each do |msg|
      row = []
      row << time = Time.at(msg['ts'].to_i).strftime('%Y/%m/%d %H:%M:%S')
      row << msg['user']
      row << CGI.unescape_html(msg['text'])
      files = msg['files']
      files ||= []
      files[0] ||= {'name' => ""}
      files[1] ||= {'name' => ""}
      files[2] ||= {'name' => ""}
      row << files[0]['name']
      row << files[1]['name']
      row << files[2]['name']
      csv << row
    end
  end
end

# main

if ARGV.size < 1
  puts "$ export_slack_channel.rb <channel>"
  exit 1
end

OUT_DIR='slack_logs'
#FileUtils.rm_rf(OUT_DIR)
FileUtils.mkdir_p(OUT_DIR)

# 認証テスト
puts "---- auth.test"
pp slack_api_get('/auth.test')

# メンバー一覧取得
puts "---- members"
list = users_list
members = list.map {|member| [member['id'], member] }.to_h
File.open(File.join(OUT_DIR,'members.json'),"w") do |io|
  io.write Oj.dump(members)
end

# チャンネル一覧取得
puts "---- channels"
channels = conversations_list
File.open(File.join(OUT_DIR,'channels.json'),"w") do |io|
  io.write Oj.dump(channels)
end

ARGV.each do |channel|
  FileUtils.mkdir_p(File.join(OUT_DIR, channel))
  
  # 引数のチャンネルのidを取得
  puts "---- #{channel}"
  ch = channels.find do |ch|
    ch['name'] == channel
  end
  unless ch
    puts "channel #{ch} not exist"
    puts "channels:"
    puts channels.map{|c| c['name']}.join(' ')
    exit 1
  end
  ch_id = ch['id']
  
  # チャンネルメンバー
  list = conversations_members(channel: ch_id)
  ch_members = list.map do |id|
    members[id]
  end
  # json書き出し
  File.open(File.join(OUT_DIR, channel, "members.json"),"w") do |io|
    io.write Oj.dump(ch_members)
  end
  # CSV作成
  CSV.open(File.join(OUT_DIR, channel, "members.csv"), 'w') do |csv|
    csv << %w|id name display_name|
    ch_members.each do |m|
      profile = m['profile']
      csv << [m['id'], m['name'], profile['display_name']]
    end
  end
  
  # チャンネルログ取得
  messages = conversations_history(ch_id, members, oldest: 0)
  # json書き出し
  File.open(File.join(OUT_DIR, channel, "messages.json"),"w") do |io|
    io.write Oj.dump(messages)
  end
  # CSV作成
  CSV.open(File.join(OUT_DIR, channel, "messages.csv"), 'w') do |csv|
    csv << ['timestamp','user','text','file1','file2','file3']
    messages.each do |msg|
      row = []
      row << time = Time.at(msg['ts'].to_i).strftime('%Y/%m/%d %H:%M:%S')
      row << msg['user']
      row << CGI.unescape_html(msg['text'])
      files = msg['files']
      files ||= []
      files[0] ||= {'name' => ""}
      files[1] ||= {'name' => ""}
      files[2] ||= {'name' => ""}
      row << files[0]['name']
      row << files[1]['name']
      row << files[2]['name']
      csv << row
    end
  end
  
  # 添付ファイル取得
  #download_files(messages, FILES_DIR)

  puts
end

exit 0