lib/pupa/processor.rb
def import
      @report[:import] = {}

      objects = deduplicate(load_scraped_objects)

      object_id_to_database_id = {}

      if use_dependency_graph?(objects)
        dependency_graph = build_dependency_graph(objects)

        # Replace object IDs with database IDs in foreign keys and save objects.
        dependency_graph.tsort.each do |id|
          object = objects[id]
          resolve_foreign_keys(object, object_id_to_database_id)
          # The dependency graph strategy only works if there are no foreign objects.

          database_id = import_object(object)
          object_id_to_database_id[id] = database_id
          object_id_to_database_id[database_id] = database_id
        end
      else
        size = objects.size

        # Should be O(n²). If there are foreign objects, we do not know all the
        # edges in the graph, and therefore cannot build a dependency graph or
        # derive any evaluation order.
        #
        # An exception is raised if a foreign object matches multiple documents
        # in the database. However, if a matching object is not yet saved, this
        # exception may not be raised.
        loop do
          progress_made = false

          objects.delete_if do |id,object|
            begin
              resolve_foreign_keys(object, object_id_to_database_id)
              resolve_foreign_objects(object, object_id_to_database_id)
              progress_made = true

              database_id = import_object(object)
              object_id_to_database_id[id] = database_id
              object_id_to_database_id[database_id] = database_id
            rescue Pupa::Errors::MissingDatabaseIdError
              false
            end
          end

          break if objects.empty? || !progress_made
        end

        unless objects.empty?
          raise Errors::UnprocessableEntity, "couldn't resolve #{objects.size}/#{size} objects:\n  #{objects.values.map{|object| JSON.dump(object.foreign_properties)}.join("\n  ")}"
        end
      end

      # Ensure that fingerprints uniquely identified objects.
      counts = {}
      object_id_to_database_id.each do |object_id,database_id|
        unless object_id == database_id
          (counts[database_id] ||= []) << object_id
        end
      end
      duplicates = counts.select do |_,object_ids|
        object_ids.size > 1
      end
      unless duplicates.empty?
        raise Errors::DuplicateDocumentError, "multiple objects written to same document:\n" + duplicates.map{|database_id,object_ids| "  #{database_id} <- #{object_ids.join(' ')}"}.join("\n")
      end
    end
def import
      @report[:import] = {}

      objects = deduplicate(load_scraped_objects)

      object_id_to_database_id = {}

      if use_dependency_graph?(objects)
        dependency_graph = build_dependency_graph(objects)

        # Replace object IDs with database IDs in foreign keys and save objects.
        dependency_graph.tsort.each do |id|
          object = objects[id]
          resolve_foreign_keys(object, object_id_to_database_id)
          # The dependency graph strategy only works if there are no foreign objects.

          database_id = import_object(object)
          object_id_to_database_id[id] = database_id
          object_id_to_database_id[database_id] = database_id
        end
      else
        size = objects.size

        # Should be O(n²). If there are foreign objects, we do not know all the
        # edges in the graph, and therefore cannot build a dependency graph or
        # derive any evaluation order.
        #
        # An exception is raised if a foreign object matches multiple documents
        # in the database. However, if a matching object is not yet saved, this
        # exception may not be raised.
        loop do
          progress_made = false

          objects.delete_if do |id,object|
            begin
              resolve_foreign_keys(object, object_id_to_database_id)
              resolve_foreign_objects(object, object_id_to_database_id)
              progress_made = true

              database_id = import_object(object)
              object_id_to_database_id[id] = database_id
              object_id_to_database_id[database_id] = database_id
            rescue Pupa::Errors::MissingDatabaseIdError
              false
            end
          end

          break if objects.empty? || !progress_made
        end

        unless objects.empty?
          raise Errors::UnprocessableEntity, "couldn't resolve #{objects.size}/#{size} objects:\n  #{objects.values.map{|object| JSON.dump(object.foreign_properties)}.join("\n  ")}"
        end
      end

      # Ensure that fingerprints uniquely identified objects.
      counts = {}
      object_id_to_database_id.each do |object_id,database_id|
        unless object_id == database_id
          (counts[database_id] ||= []) << object_id
        end
      end
      duplicates = counts.select do |_,object_ids|
        object_ids.size > 1
      end
      unless duplicates.empty?
        raise Errors::DuplicateDocumentError, "multiple objects written to same document:\n" + duplicates.map{|database_id,object_ids| "  #{database_id} <- #{object_ids.join(' ')}"}.join("\n")
      end
    end
def import @report [ :import ] = { } objects = deduplicate ( load_scraped_objects ) object_id_to_database_id = { } if use_dependency_graph? ( objects ) dependency_graph = build_dependency_graph ( objects ) # Replace object IDs with database IDs in foreign keys and save objects. dependency_graph . tsort . each do | id | object = objects [ id ] resolve_foreign_keys ( object , object_id_to_database_id ) # The dependency graph strategy only works if there are no foreign objects. database_id = import_object ( object ) object_id_to_database_id [ id ] = database_id object_id_to_database_id [ database_id ] = database_id end else size = objects . size # Should be O(n²). If there are foreign objects, we do not know all the # edges in the graph, and therefore cannot build a dependency graph or # derive any evaluation order. # # An exception is raised if a foreign object matches multiple documents # in the database. However, if a matching object is not yet saved, this # exception may not be raised. loop do progress_made = false objects . delete_if do | id , object | begin resolve_foreign_keys ( object , object_id_to_database_id ) resolve_foreign_objects ( object , object_id_to_database_id ) progress_made = true database_id = import_object ( object ) object_id_to_database_id [ id ] = database_id object_id_to_database_id [ database_id ] = database_id rescue Pupa :: Errors :: MissingDatabaseIdError false end end break if objects . empty? || ! progress_made end unless objects . empty? raise Errors :: UnprocessableEntity , "couldn't resolve #{objects.size}/#{size} objects:\n  #{objects.values.map{|object| JSON.dump(object.foreign_properties)}.join("\n  ")}" end end # Ensure that fingerprints uniquely identified objects. counts = { } object_id_to_database_id . each do | object_id , database_id | unless object_id == database_id ( counts [ database_id ] ||= [ ] ) << object_id end end duplicates = counts . select do | _ , object_ids | object_ids . size > 1 end unless duplicates . empty? raise Errors :: DuplicateDocumentError , "multiple objects written to same document:\n" + duplicates . map { | database_id , object_ids | "  #{database_id} <- #{object_ids.join(' ')}" } . join ( "\n" ) end end
Saves scraped objects to a database.

 @raises [TSort::Cyclic] if the dependency graph is cyclic
 @raises [Pupa::Errors::UnprocessableEntity] if an object's foreign keys or
   foreign objects cannot be resolved
 @raises [Pupa::Errors::DuplicateDocumentError] if duplicate objects were
   inadvertently saved to the database
Saves scraped objects to a database .