Archive Migration Functions

archive-migration.sh

standard_list="url,index-pattern,action,query,alert,graph-workspace,tag,visualization,canvas-element,canvas-workpad,dashboard,search,lens,map,cases,uptime-dynamic-settings,osquery-saved-query,osquery-pack,infrastructure-ui-source,metrics-explorer-view,inventory-view,infrastructure-monitoring-log-view,apm-indices"

orig_archive="x-pack/test/functional/es_archives/logstash/example_pipelines"
new_archive="x-pack/test/functional/fixtures/kbn_archiver/spaces/disabled_features"
test_config="x-pack/test/functional/apps/logstash/config.ts"

_types() {
  local archiveJson=$1

  if [[ -f "$archiveJson" ]]; then
    local typesList=$(jq 'select(.value.source.type | length > 0) | .value.source.type' $archiveJson)
    echo "$typesList" | sed 's/^.//;s/.$//' | sort | uniq
    git clean -f "$archiveJson" >/dev/null
  fi
}

types() {
  local archive=${1:-$orig_archive}
  local archiveJson="${archive}/data.json"

  if [[ ! -f "$archiveJson" ]]; then
    local zip=$(find "$archive" -maxdepth 1 -type f -name '*.gz')
    gunzip --keep "$zip"
  fi

  _types $archiveJson
}

migrate() {
  set -x
  node scripts/es_archiver.js load "$orig_archive" --config "$test_config"
  node scripts/kbn_archiver.js save "$new_archive" --config "$test_config" --type "$standard_list"
  node scripts/es_archiver.js unload "$orig_archive"
  set +x
}

usages() {
  local archive=${1:-$orig_archive}

  grep -q "x-pack" <<<"$archive"
  exitCode=$?

  if [[ $exitCode = 0 ]]; then pushd x-pack; fi
  set -x
  find . -type f -print0 | xargs -0 grep -n $archive | cut -d ':' -f 1 | uniq
  set +x
  if [[ $exitCode = 0 ]]; then popd; fi
}

load_logstash() {
  set -x
  node scripts/es_archiver.js load x-pack/test/functional/es_archives/logstash_functional --config $test_config
  set +x
}

load_es() {
  set -x
  node scripts/es_archiver.js load $orig_archive --config $test_config
  set +x
}

unload_es() {
  set -x
  node scripts/es_archiver.js unload $orig_archive
  set +x
}

save_es_archive_without_kibana_objects() {
  # pass the index name on the cli
  set -x
  node scripts/es_archiver.js save $orig_archive --config $test_config $index_name
  set +x
}

save_kbn() {
  set -x
  node scripts/kbn_archiver.js --config $test_config save $new_archive --type $standard_list
  set +x
  #  node scripts/kbn_archiver.js --config x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts save x-pack/test/functional/fixtures/kbn_archiver/saved_objects/default_space --type search,index-pattern,visualization,dashboard,lens,map,graph-workspace,query,tag,url,canvas-workpad
}

load_kbn() {
  set -x
  node scripts/kbn_archiver.js --config $test_config load $new_archive
  set +x
}

load_created_kbn_archive() {
  set -x
  node scripts/kbn_archiver.js --config $test_config load $created_archive
  set +x
}

unload_kbn() {
  set -x
  node scripts/kbn_archiver.js --config $test_config unload $new_archive
  set +x
}

ping_server() {
  set -x
  server_addr="http://elastic:changeme@localhost:9220"
  curl -k "${server_addr}/_cat/health?v=true"
  set +x
}

server() {
  local config=${1:-$test_config}

  set -x
  node scripts/functional_tests_server.js --config $config
  set +x
}

allowed_types='{
 "types": [
   {
     "name": "config",
     "namespaceType": "single",
     "hidden": false,
     "displayName": "config"
   },
   {
     "name": "url",
     "namespaceType": "single",
     "hidden": false,
     "displayName": "url"
   },
   {
     "name": "index-pattern",
     "namespaceType": "multiple",
     "hidden": false,
     "displayName": "data view"
   },
   {
     "name": "action",
     "namespaceType": "multiple-isolated",
     "hidden": true,
     "displayName": "connector"
   },
   {
     "name": "query",
     "namespaceType": "multiple-isolated",
     "hidden": false,
     "displayName": "query"
   },
   {
     "name": "alert",
     "namespaceType": "multiple-isolated",
     "hidden": true,
     "displayName": "rule"
   },
   {
     "name": "graph-workspace",
     "namespaceType": "multiple-isolated",
     "hidden": false,
     "displayName": "graph-workspace"
   },
   {
     "name": "tag",
     "namespaceType": "multiple-isolated",
     "hidden": false,
     "displayName": "tag"
   },
   {
     "name": "visualization",
     "namespaceType": "multiple-isolated",
     "hidden": false,
     "displayName": "visualization"
   },
   {
     "name": "canvas-element",
     "namespaceType": "multiple-isolated",
     "hidden": false,
     "displayName": "canvas-element"
   },
   {
     "name": "canvas-workpad",
     "namespaceType": "multiple-isolated",
     "hidden": false,
     "displayName": "canvas-workpad"
   },
   {
     "name": "dashboard",
     "namespaceType": "multiple-isolated",
     "hidden": false,
     "displayName": "dashboard"
   },
   {
     "name": "search",
     "namespaceType": "multiple-isolated",
     "hidden": false,
     "displayName": "search"
   },
   {
     "name": "lens",
     "namespaceType": "multiple-isolated",
     "hidden": false,
     "displayName": "lens"
   },
   {
     "name": "map",
     "namespaceType": "multiple-isolated",
     "hidden": false,
     "displayName": "map"
   },
   {
     "name": "cases",
     "namespaceType": "multiple-isolated",
     "hidden": true,
     "displayName": "cases"
   },
   {
     "name": "uptime-dynamic-settings",
     "namespaceType": "single",
     "hidden": false,
     "displayName": "uptime-dynamic-settings"
   },
   {
     "name": "osquery-saved-query",
     "namespaceType": "multiple-isolated",
     "hidden": false,
     "displayName": "osquery-saved-query"
   },
   {
     "name": "osquery-pack",
     "namespaceType": "multiple-isolated",
     "hidden": false,
     "displayName": "osquery-pack"
   },
   {
     "name": "infrastructure-ui-source",
     "namespaceType": "single",
     "hidden": false,
     "displayName": "infrastructure-ui-source"
   },
   {
     "name": "metrics-explorer-view",
     "namespaceType": "single",
     "hidden": false,
     "displayName": "metrics-explorer-view"
   },
   {
     "name": "inventory-view",
     "namespaceType": "single",
     "hidden": false,
     "displayName": "inventory-view"
   },
   {
     "name": "infrastructure-monitoring-log-view",
     "namespaceType": "multiple-isolated",
     "hidden": false,
     "displayName": "log view"
   },
   {
     "name": "apm-indices",
     "namespaceType": "single",
     "hidden": false,
     "displayName": "apm-indices"
   },
   {
     "name": "csp_rule",
     "namespaceType": "agnostic",
     "hidden": false,
     "displayName": "csp_rule"
   },
   {
     "name": "csp-rule-template",
     "namespaceType": "agnostic",
     "hidden": false,
     "displayName": "csp-rule-template"
   }
 ]
}'
_allowed_types() {
  echo "$allowed_types" | jq '.types | .[].name' | sort
}

These functions are the most used:

• server
• types
• migrate
• usages

Note from Lee: If it has other than kibana data within, we need to recreate the original es archive w/o the kibana data...essentially overwrite it.
We could use some automation around this.