lib/pupa/processor.rb
def deduplicate(objects)
      losers_to_winners = build_losers_to_winners_map(objects)

      # Remove all losers.
      losers_to_winners.each_key do |key|
        objects.delete(key)
      end

      # Swap the IDs of losers for the IDs of winners.
      objects.each do |id,object|
        object.foreign_keys.each do |property|
          value = object[property]
          if value && losers_to_winners.key?(value)
            object[property] = losers_to_winners[value]
          end
        end
      end

      objects
    end
def deduplicate(objects)
      losers_to_winners = build_losers_to_winners_map(objects)

      # Remove all losers.
      losers_to_winners.each_key do |key|
        objects.delete(key)
      end

      # Swap the IDs of losers for the IDs of winners.
      objects.each do |id,object|
        object.foreign_keys.each do |property|
          value = object[property]
          if value && losers_to_winners.key?(value)
            object[property] = losers_to_winners[value]
          end
        end
      end

      objects
    end
def deduplicate ( objects ) losers_to_winners = build_losers_to_winners_map ( objects ) # Remove all losers. losers_to_winners . each_key do | key | objects . delete ( key ) end # Swap the IDs of losers for the IDs of winners. objects . each do | id , object | object . foreign_keys . each do | property | value = object [ property ] if value && losers_to_winners . key? ( value ) object [ property ] = losers_to_winners [ value ] end end end objects end
Removes all duplicate objects and re-assigns any foreign keys.

 @param [Hash] objects a hash of scraped objects keyed by ID
 @return [Hash] the objects without duplicates
Removes all duplicate objects and re - assigns any foreign keys .