src/main/java/com/conveyal/gtfs/loader/Feed.java
public ValidationResult validate () {
        long validationStartTime = System.currentTimeMillis();
        // Create an empty validation result that will have its fields populated by certain validators.
        ValidationResult validationResult = new ValidationResult();
        // Error tables should already be present from the initial load.
        // Reconnect to the existing error tables.
        SQLErrorStorage errorStorage = null;
        try {
            errorStorage = new SQLErrorStorage(dataSource.getConnection(), tablePrefix, false);
        } catch (SQLException | InvalidNamespaceException ex) {
            throw new StorageException(ex);
        }
        int errorCountBeforeValidation = errorStorage.getErrorCount();

        List<FeedValidator> feedValidators = Arrays.asList(
                new MisplacedStopValidator(this, errorStorage, validationResult),
                new DuplicateStopsValidator(this, errorStorage),
                new TimeZoneValidator(this, errorStorage),
                new NewTripTimesValidator(this, errorStorage),
                new NamesValidator(this, errorStorage));

        for (FeedValidator feedValidator : feedValidators) {
            String validatorName = feedValidator.getClass().getSimpleName();
            try {
                LOG.info("Running {}.", validatorName);
                int errorCountBefore = errorStorage.getErrorCount();
                // todo why not just pass the feed and errorstorage in here?
                feedValidator.validate();
                LOG.info("{} found {} errors.", validatorName, errorStorage.getErrorCount() - errorCountBefore);
            } catch (Exception e) {
                // store an error if the validator fails
                // FIXME: should the exception be stored?
                String badValue = String.join(":", validatorName, e.toString());
                errorStorage.storeError(NewGTFSError.forFeed(VALIDATOR_FAILED, badValue));
                LOG.error("{} failed.", validatorName);
                LOG.error(e.toString());
                e.printStackTrace();
            }
        }
        // Signal to all validators that validation is complete and allow them to report on results / status.
        for (FeedValidator feedValidator : feedValidators) {
            try {
                feedValidator.complete(validationResult);
            } catch (Exception e) {
                String badValue = String.join(":", feedValidator.getClass().getSimpleName(), e.toString());
                errorStorage.storeError(NewGTFSError.forFeed(VALIDATOR_FAILED, badValue));
                LOG.error("Validator failed completion stage.", e);
            }
        }
        // Total validation errors accounts for errors found during both loading and validation. Otherwise, this value
        // may be confusing if it reads zero but there were a number of data type or referential integrity errors found
        // during feed loading stage.
        int totalValidationErrors = errorStorage.getErrorCount();
        LOG.info("Errors found during load stage: {}", errorCountBeforeValidation);
        LOG.info("Errors found by validators: {}", totalValidationErrors - errorCountBeforeValidation);
        errorStorage.commitAndClose();
        long validationEndTime = System.currentTimeMillis();
        long totalValidationTime = validationEndTime - validationStartTime;
        LOG.info("{} validators completed in {} milliseconds.", feedValidators.size(), totalValidationTime);

        // update validation result fields
        validationResult.errorCount = totalValidationErrors;
        validationResult.validationTime = totalValidationTime;

        // FIXME: Validation result date and int[] fields need to be set somewhere.
        return validationResult;
    }
public ValidationResult validate () {
        long validationStartTime = System.currentTimeMillis();
        // Create an empty validation result that will have its fields populated by certain validators.
        ValidationResult validationResult = new ValidationResult();
        // Error tables should already be present from the initial load.
        // Reconnect to the existing error tables.
        SQLErrorStorage errorStorage = null;
        try {
            errorStorage = new SQLErrorStorage(dataSource.getConnection(), tablePrefix, false);
        } catch (SQLException | InvalidNamespaceException ex) {
            throw new StorageException(ex);
        }
        int errorCountBeforeValidation = errorStorage.getErrorCount();

        List<FeedValidator> feedValidators = Arrays.asList(
                new MisplacedStopValidator(this, errorStorage, validationResult),
                new DuplicateStopsValidator(this, errorStorage),
                new TimeZoneValidator(this, errorStorage),
                new NewTripTimesValidator(this, errorStorage),
                new NamesValidator(this, errorStorage));

        for (FeedValidator feedValidator : feedValidators) {
            String validatorName = feedValidator.getClass().getSimpleName();
            try {
                LOG.info("Running {}.", validatorName);
                int errorCountBefore = errorStorage.getErrorCount();
                // todo why not just pass the feed and errorstorage in here?
                feedValidator.validate();
                LOG.info("{} found {} errors.", validatorName, errorStorage.getErrorCount() - errorCountBefore);
            } catch (Exception e) {
                // store an error if the validator fails
                // FIXME: should the exception be stored?
                String badValue = String.join(":", validatorName, e.toString());
                errorStorage.storeError(NewGTFSError.forFeed(VALIDATOR_FAILED, badValue));
                LOG.error("{} failed.", validatorName);
                LOG.error(e.toString());
                e.printStackTrace();
            }
        }
        // Signal to all validators that validation is complete and allow them to report on results / status.
        for (FeedValidator feedValidator : feedValidators) {
            try {
                feedValidator.complete(validationResult);
            } catch (Exception e) {
                String badValue = String.join(":", feedValidator.getClass().getSimpleName(), e.toString());
                errorStorage.storeError(NewGTFSError.forFeed(VALIDATOR_FAILED, badValue));
                LOG.error("Validator failed completion stage.", e);
            }
        }
        // Total validation errors accounts for errors found during both loading and validation. Otherwise, this value
        // may be confusing if it reads zero but there were a number of data type or referential integrity errors found
        // during feed loading stage.
        int totalValidationErrors = errorStorage.getErrorCount();
        LOG.info("Errors found during load stage: {}", errorCountBeforeValidation);
        LOG.info("Errors found by validators: {}", totalValidationErrors - errorCountBeforeValidation);
        errorStorage.commitAndClose();
        long validationEndTime = System.currentTimeMillis();
        long totalValidationTime = validationEndTime - validationStartTime;
        LOG.info("{} validators completed in {} milliseconds.", feedValidators.size(), totalValidationTime);

        // update validation result fields
        validationResult.errorCount = totalValidationErrors;
        validationResult.validationTime = totalValidationTime;

        // FIXME: Validation result date and int[] fields need to be set somewhere.
        return validationResult;
    }
public ValidationResult validate ( ) { long validationStartTime = System . currentTimeMillis ( ) ; // Create an empty validation result that will have its fields populated by certain validators. ValidationResult validationResult = new ValidationResult ( ) ; // Error tables should already be present from the initial load. // Reconnect to the existing error tables. SQLErrorStorage errorStorage = null ; try { errorStorage = new SQLErrorStorage ( dataSource . getConnection ( ) , tablePrefix , false ) ; } catch ( SQLException | InvalidNamespaceException ex ) { throw new StorageException ( ex ) ; } int errorCountBeforeValidation = errorStorage . getErrorCount ( ) ; List < FeedValidator > feedValidators = Arrays . asList ( new MisplacedStopValidator ( this , errorStorage , validationResult ) , new DuplicateStopsValidator ( this , errorStorage ) , new TimeZoneValidator ( this , errorStorage ) , new NewTripTimesValidator ( this , errorStorage ) , new NamesValidator ( this , errorStorage ) ) ; for ( FeedValidator feedValidator : feedValidators ) { String validatorName = feedValidator . getClass ( ) . getSimpleName ( ) ; try { LOG . info ( "Running {}." , validatorName ) ; int errorCountBefore = errorStorage . getErrorCount ( ) ; // todo why not just pass the feed and errorstorage in here? feedValidator . validate ( ) ; LOG . info ( "{} found {} errors." , validatorName , errorStorage . getErrorCount ( ) - errorCountBefore ) ; } catch ( Exception e ) { // store an error if the validator fails // FIXME: should the exception be stored? String badValue = String . join ( ":" , validatorName , e . toString ( ) ) ; errorStorage . storeError ( NewGTFSError . forFeed ( VALIDATOR_FAILED , badValue ) ) ; LOG . error ( "{} failed." , validatorName ) ; LOG . error ( e . toString ( ) ) ; e . printStackTrace ( ) ; } } // Signal to all validators that validation is complete and allow them to report on results / status. for ( FeedValidator feedValidator : feedValidators ) { try { feedValidator . complete ( validationResult ) ; } catch ( Exception e ) { String badValue = String . join ( ":" , feedValidator . getClass ( ) . getSimpleName ( ) , e . toString ( ) ) ; errorStorage . storeError ( NewGTFSError . forFeed ( VALIDATOR_FAILED , badValue ) ) ; LOG . error ( "Validator failed completion stage." , e ) ; } } // Total validation errors accounts for errors found during both loading and validation. Otherwise, this value // may be confusing if it reads zero but there were a number of data type or referential integrity errors found // during feed loading stage. int totalValidationErrors = errorStorage . getErrorCount ( ) ; LOG . info ( "Errors found during load stage: {}" , errorCountBeforeValidation ) ; LOG . info ( "Errors found by validators: {}" , totalValidationErrors - errorCountBeforeValidation ) ; errorStorage . commitAndClose ( ) ; long validationEndTime = System . currentTimeMillis ( ) ; long totalValidationTime = validationEndTime - validationStartTime ; LOG . info ( "{} validators completed in {} milliseconds." , feedValidators . size ( ) , totalValidationTime ) ; // update validation result fields validationResult . errorCount = totalValidationErrors ; validationResult . validationTime = totalValidationTime ; // FIXME: Validation result date and int[] fields need to be set somewhere. return validationResult ; }
TODO check whether validation has already occurred, overwrite results.
TODO allow validation within feed loading process, so the same connection can be used, and we're certain loaded data is 100% visible.
That would also avoid having to reconnect the error storage to the DB.
TODO check whether validation has already occurred overwrite results . TODO allow validation within feed loading process so the same connection can be used and we re certain loaded data is 100% visible . That would also avoid having to reconnect the error storage to the DB .