lib/pupa/runner.rb
def run(args, overrides = {})
      rest = opts.parse!(args)

      @options = OpenStruct.new(options.to_h.merge(overrides))

      if options.actions.empty?
        options.actions = %w(scrape import)
      end
      if options.tasks.empty?
        options.tasks = @processor_class.tasks
      end

      processor = @processor_class.new(options.output_dir,
        pipelined: options.pipelined,
        cache_dir: options.cache_dir,
        expires_in: options.expires_in,
        value_max_bytes: options.value_max_bytes,
        memcached_username: options.memcached_username,
        memcached_password: options.memcached_password,
        database_url: options.database_url,
        validate: options.validate,
        level: options.level,
        faraday_options: options.faraday_options,
        options: Hash[*rest])

      options.actions.each do |action|
        unless action == 'scrape' || processor.respond_to?(action)
          abort %(`#{action}` is not a #{opts.program_name} action. See `#{opts.program_name} --help` for a list of available actions.)
        end
      end

      if %w(DEBUG INFO).include?(options.level)
        puts "processor: #{@processor_class}"
        puts "actions: #{options.actions.join(', ')}"
        puts "tasks: #{options.tasks.join(', ')}"
      end

      if options.level == 'DEBUG'
        %w(output_dir pipelined cache_dir expires_in value_max_bytes memcached_username memcached_password database_url validate level).each do |option|
          puts "#{option}: #{options[option]}"
        end
        unless rest.empty?
          puts "options: #{rest.join(' ')}"
        end
      end

      exit if options.dry_run

      report = {
        plan: {
          processor: @processor_class,
          options: Marshal.load(Marshal.dump(options)).to_h,
          arguments: rest,
        },
        start: Time.now.utc,
      }

      if options.actions.delete('scrape')
        processor.store.clear
        report[:scrape] = {}
        options.tasks.each do |task_name|
          report[:scrape][task_name] = processor.dump_scraped_objects(task_name)
        end
      end

      options.actions.each do |action|
        processor.send(action)
        if processor.report.key?(action.to_sym)
          report.update(action.to_sym => processor.report[action.to_sym])
        end
      end

      if %w(DEBUG INFO).include?(options.level)
        report[:end] = Time.now.utc
        report[:time] = report[:end] - report[:start]
        puts JSON.dump(report)
      end
    end
def run(args, overrides = {})
      rest = opts.parse!(args)

      @options = OpenStruct.new(options.to_h.merge(overrides))

      if options.actions.empty?
        options.actions = %w(scrape import)
      end
      if options.tasks.empty?
        options.tasks = @processor_class.tasks
      end

      processor = @processor_class.new(options.output_dir,
        pipelined: options.pipelined,
        cache_dir: options.cache_dir,
        expires_in: options.expires_in,
        value_max_bytes: options.value_max_bytes,
        memcached_username: options.memcached_username,
        memcached_password: options.memcached_password,
        database_url: options.database_url,
        validate: options.validate,
        level: options.level,
        faraday_options: options.faraday_options,
        options: Hash[*rest])

      options.actions.each do |action|
        unless action == 'scrape' || processor.respond_to?(action)
          abort %(`#{action}` is not a #{opts.program_name} action. See `#{opts.program_name} --help` for a list of available actions.)
        end
      end

      if %w(DEBUG INFO).include?(options.level)
        puts "processor: #{@processor_class}"
        puts "actions: #{options.actions.join(', ')}"
        puts "tasks: #{options.tasks.join(', ')}"
      end

      if options.level == 'DEBUG'
        %w(output_dir pipelined cache_dir expires_in value_max_bytes memcached_username memcached_password database_url validate level).each do |option|
          puts "#{option}: #{options[option]}"
        end
        unless rest.empty?
          puts "options: #{rest.join(' ')}"
        end
      end

      exit if options.dry_run

      report = {
        plan: {
          processor: @processor_class,
          options: Marshal.load(Marshal.dump(options)).to_h,
          arguments: rest,
        },
        start: Time.now.utc,
      }

      if options.actions.delete('scrape')
        processor.store.clear
        report[:scrape] = {}
        options.tasks.each do |task_name|
          report[:scrape][task_name] = processor.dump_scraped_objects(task_name)
        end
      end

      options.actions.each do |action|
        processor.send(action)
        if processor.report.key?(action.to_sym)
          report.update(action.to_sym => processor.report[action.to_sym])
        end
      end

      if %w(DEBUG INFO).include?(options.level)
        report[:end] = Time.now.utc
        report[:time] = report[:end] - report[:start]
        puts JSON.dump(report)
      end
    end
def run ( args , overrides = { } ) rest = opts . parse! ( args ) @options = OpenStruct . new ( options . to_h . merge ( overrides ) ) if options . actions . empty? options . actions = %w( scrape import ) end if options . tasks . empty? options . tasks = @processor_class . tasks end processor = @processor_class . new ( options . output_dir , pipelined : options . pipelined , cache_dir : options . cache_dir , expires_in : options . expires_in , value_max_bytes : options . value_max_bytes , memcached_username : options . memcached_username , memcached_password : options . memcached_password , database_url : options . database_url , validate : options . validate , level : options . level , faraday_options : options . faraday_options , options : Hash [ rest ] ) options . actions . each do | action | unless action == 'scrape' || processor . respond_to? ( action ) abort %(`#{action}` is not a #{opts.program_name} action. See `#{opts.program_name} --help` for a list of available actions.) end end if %w( DEBUG INFO ) . include? ( options . level ) puts "processor: #{@processor_class}" puts "actions: #{options.actions.join(', ')}" puts "tasks: #{options.tasks.join(', ')}" end if options . level == 'DEBUG' %w( output_dir pipelined cache_dir expires_in value_max_bytes memcached_username memcached_password database_url validate level ) . each do | option | puts "#{option}: #{options[option]}" end unless rest . empty? puts "options: #{rest.join(' ')}" end end exit if options . dry_run report = { plan : { processor : @processor_class , options : Marshal . load ( Marshal . dump ( options ) ) . to_h , arguments : rest , } , start : Time . now . utc , } if options . actions . delete ( 'scrape' ) processor . store . clear report [ :scrape ] = { } options . tasks . each do | task_name | report [ :scrape ] [ task_name ] = processor . dump_scraped_objects ( task_name ) end end options . actions . each do | action | processor . send ( action ) if processor . report . key? ( action . to_sym ) report . update ( action . to_sym => processor . report [ action . to_sym ] ) end end if %w( DEBUG INFO ) . include? ( options . level ) report [ :end ] = Time . now . utc report [ :time ] = report [ :end ] - report [ :start ] puts JSON . dump ( report ) end end
Runs the action.

 @example Run from a command-line script

     runner.run(ARGV)

 @example Override the command-line options

     runner.run(ARGV, expires_in: 3600) # 1 hour

 @param [Array] args command-line arguments
 @param [Hash] overrides any overridden options
Runs the action .