people_json_filter_bad_data.rb

json_filter.rb

require 'json'
require 'pp'

file_path = "./current_dossis.json"
dossis = JSON.parse(File.read(file_path))

named_dossis = dossis.reject { |d| d.fetch("name") == nil }
named_dossis = named_dossis.reject { |d| d.fetch("name") == "" }
named_dossis = named_dossis.reject { |d| d.fetch("name").downcase == "nil" }
named_dossis = named_dossis.reject { |d| d.fetch("name").to_s.include? "http" }

i = 0
results = []

while named_dossis.size > 574
  p named_dossis.size
  dossi = named_dossis[i]  
  common_dossis = named_dossis.select do |d|
    if d && dossi
      d.fetch("name") == dossi.fetch("name")
    end
  end
  
  if common_dossis.count > 1
    dossi.each do |key, value|
      next if key == "picture" || key == "comments"
      dossi[key] = common_dossis.map { |cd| cd[key] }.uniq.join("\n")
    end
  end
  
  results << dossi
  
  named_dossis.delete_if do |d|
    if d && dossi
      d.fetch("name") == dossi.fetch("name")
    end
  end
  
  i += 1
end

p results.size

File.open("output.json","w") do |f|
  f.write(results.to_json)
end