Skip to content

Instantly share code, notes, and snippets.

@giabao
Last active October 2, 2024 21:41
Show Gist options
  • Save giabao/f4c3de705f1d7f2c1fd0cde02e7b841d to your computer and use it in GitHub Desktop.
Save giabao/f4c3de705f1d7f2c1fd0cde02e7b841d to your computer and use it in GitHub Desktop.
bitwarden deduplicate import
  1. (optional) Import Data from chrome, firefox, keepass,..
  2. Export Vault to bitwarden.json file
  3. Get Ammonite
sudo sh -c '(echo "#!/usr/bin/env sh" && curl -L https://github.com/com-lihaoyi/Ammonite/releases/download/2.4.0/2.13-2.4.0) > /usr/local/bin/amm && chmod +x /usr/local/bin/amm' && amm
  1. Download the dedup.scala file below then run it in ammonite
amm dedup.scala bitwarden.json

This will create file bitwarden_out.json

  1. Purge Vault
  2. Re-Import Data from bitwarden_out.json
  3. Done
import upickle._
@main def dedup(filename: String): Unit = {
val path = os.pwd / filename
val json = ujson.read(os.read(path))
val items = json("items").arr
/* debug only
val itemsWithMatch = items.filter(
_("login").obj.get("uris") match {
case None => false
case Some(uris) => uris.arr.exists(! _("match").isNull)
}
)
println(itemsWithMatch.length)
val itemsWithOrganizationId = items.filter(! _("organizationId").isNull)
println(itemsWithOrganizationId.length)
*/
// only interested in those fields
def interested(d: ujson.Value) = {
val l = d("login").obj
( d("organizationId"),
d("notes"),
l("username"),
l("password"),
l("totp"),
for {
uris <- l.get("uris").toList
uri <- uris.arr
} yield (uri("match"), uri("uri"))
)
}
val out = items.distinctBy(interested)
json("items") = out
val outName = path.baseName + "_out." + path.ext
os.write.over(os.pwd / outName, ujson.write(json))
println(s"Removed ${items.length - out.length} from ${items.length}, remains ${out.length} entries.")
}
@Motzumoto
Copy link

Motzumoto commented Sep 15, 2022

lain@Lain:/mnt/c/Users/Lain/Desktop$ amm dedup.scala bitwarden.json
java.util.NoSuchElementException: key not found: login
  scala.collection.MapOps.default(Map.scala:274)
  scala.collection.MapOps.default$(Map.scala:273)
  scala.collection.AbstractMap.default(Map.scala:405)
  scala.collection.MapOps.apply(Map.scala:176)
  scala.collection.MapOps.apply$(Map.scala:175)
  scala.collection.AbstractMap.apply(Map.scala:405)
  ujson.Value$Selector$StringSelector.apply(Value.scala:141)
  ujson.Value.apply(Value.scala:101)
  ujson.Value.apply$(Value.scala:101)
  ujson.Obj.apply(Value.scala:234)
  ammonite.$file.dedup$.interested$1(dedup.scala:22)
  ammonite.$file.dedup$.$anonfun$dedup$3(dedup.scala:35)
  scala.collection.StrictOptimizedSeqOps.distinctBy(StrictOptimizedSeqOps.scala:30)
  scala.collection.StrictOptimizedSeqOps.distinctBy$(StrictOptimizedSeqOps.scala:24)
  scala.collection.mutable.ArrayBuffer.distinctBy(ArrayBuffer.scala:43)
  ammonite.$file.dedup$.dedup(dedup.scala:35)
  ammonite.$file.dedup$$routes$.$anonfun$apply$1(dedup.scala:49)
  ammonite.$file.dedup$$routes$.$anonfun$apply$1$adapted(dedup.scala:49)
  mainargs.Invoker$.$anonfun$invoke0$3(Invoker.scala:59)
  mainargs.Result.flatMap(Result.scala:13)
  mainargs.Result.flatMap$(Result.scala:12)
  mainargs.Result$Success.flatMap(Result.scala:23)
  mainargs.Invoker$.invoke0(Invoker.scala:58)
  mainargs.Invoker$.invoke(Invoker.scala:70)
  mainargs.Invoker$.$anonfun$runMains$1(Invoker.scala:87)
  mainargs.Result.flatMap(Result.scala:13)
  mainargs.Result.flatMap$(Result.scala:12)
  mainargs.Result$Success.flatMap(Result.scala:23)
  mainargs.Invoker$.groupArgs$1(Invoker.scala:87)
  mainargs.Invoker$.runMains(Invoker.scala:91)
  ammonite.main.Scripts$.$anonfun$runScript$5(Scripts.scala:110)
  ammonite.util.Util$.withContextClassloader(Util.scala:24)
  ammonite.main.Scripts$.$anonfun$runScript$4(Scripts.scala:86)
  ammonite.util.Res$Success.flatMap(Res.scala:62)
  ammonite.main.Scripts$.$anonfun$runScript$2(Scripts.scala:53)
  ammonite.util.Res$Success.flatMap(Res.scala:62)
  ammonite.main.Scripts$.$anonfun$runScript$1(Scripts.scala:32)
  ammonite.util.Res$Success.flatMap(Res.scala:62)
  ammonite.main.Scripts$.runScript(Scripts.scala:28)
  ammonite.Main.runScript(Main.scala:252)
  ammonite.MainRunner.$anonfun$runScript$1(MainRunner.scala:67)
  ammonite.MainRunner.watchLoop(MainRunner.scala:53)
  ammonite.MainRunner.runScript(MainRunner.scala:67)
  ammonite.AmmoniteMain$.main0(AmmoniteMain.scala:112)
  ammonite.AmmoniteMain$.main(AmmoniteMain.scala:38)
  ammonite.AmmoniteMain.main(AmmoniteMain.scala)

@aquada
Copy link

aquada commented Jul 25, 2023

I get the same results

@rolzing
Copy link

rolzing commented Aug 10, 2023

Work for me.
just download from here
Get Ammonite

@c2b2pss
Copy link

c2b2pss commented Oct 2, 2024

$ amm dedup.scala bitwarden.json
java.util.NoSuchElementException: key not found: login
  scala.collection.MapOps.default(Map.scala:274)
  scala.collection.MapOps.default$(Map.scala:273)
  scala.collection.AbstractMap.default(Map.scala:405)
  scala.collection.MapOps.apply(Map.scala:176)
  scala.collection.MapOps.apply$(Map.scala:175)
  scala.collection.AbstractMap.apply(Map.scala:405)
  ujson.Value$Selector$StringSelector.apply(Value.scala:141)
  ujson.Value.apply(Value.scala:101)
  ujson.Value.apply$(Value.scala:101)
  ujson.Obj.apply(Value.scala:234)
  ammonite.$file.dedup$.interested$1(dedup.scala:22)
  ammonite.$file.dedup$.$anonfun$dedup$3(dedup.scala:35)
  scala.collection.StrictOptimizedSeqOps.distinctBy(StrictOptimizedSeqOps.scala:30)
  scala.collection.StrictOptimizedSeqOps.distinctBy$(StrictOptimizedSeqOps.scala:24)
  scala.collection.mutable.ArrayBuffer.distinctBy(ArrayBuffer.scala:43)
  ammonite.$file.dedup$.dedup(dedup.scala:35)
  ammonite.$file.dedup$$routes$.$anonfun$apply$1(dedup.scala:49)
  ammonite.$file.dedup$$routes$.$anonfun$apply$1$adapted(dedup.scala:49)
  mainargs.Invoker$.$anonfun$invoke0$3(Invoker.scala:59)
  mainargs.Result.flatMap(Result.scala:13)
  mainargs.Result.flatMap$(Result.scala:12)
  mainargs.Result$Success.flatMap(Result.scala:23)
  mainargs.Invoker$.invoke0(Invoker.scala:58)
  mainargs.Invoker$.invoke(Invoker.scala:70)
  mainargs.Invoker$.$anonfun$runMains$1(Invoker.scala:87)
  mainargs.Result.flatMap(Result.scala:13)
  mainargs.Result.flatMap$(Result.scala:12)
  mainargs.Result$Success.flatMap(Result.scala:23)
  mainargs.Invoker$.groupArgs$1(Invoker.scala:87)
  mainargs.Invoker$.runMains(Invoker.scala:91)
  ammonite.main.Scripts$.$anonfun$runScript$5(Scripts.scala:110)
  ammonite.util.Util$.withContextClassloader(Util.scala:24)
  ammonite.main.Scripts$.$anonfun$runScript$4(Scripts.scala:86)
  ammonite.util.Res$Success.flatMap(Res.scala:62)
  ammonite.main.Scripts$.$anonfun$runScript$2(Scripts.scala:53)
  ammonite.util.Res$Success.flatMap(Res.scala:62)
  ammonite.main.Scripts$.$anonfun$runScript$1(Scripts.scala:32)
  ammonite.util.Res$Success.flatMap(Res.scala:62)
  ammonite.main.Scripts$.runScript(Scripts.scala:28)
  ammonite.Main.runScript(Main.scala:252)
  ammonite.MainRunner.$anonfun$runScript$1(MainRunner.scala:67)
  ammonite.MainRunner.watchLoop(MainRunner.scala:53)
  ammonite.MainRunner.runScript(MainRunner.scala:67)
  ammonite.AmmoniteMain$.main0(AmmoniteMain.scala:112)
  ammonite.AmmoniteMain$.main(AmmoniteMain.scala:38)
  ammonite.AmmoniteMain.main(AmmoniteMain.scala)

Same error. Key 'login' not found. Using ammonite latest (2024)
Using the version in above original instructions also gives same error.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment