src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java
public FeedLoadResult loadTables () {

        // This result object will be returned to the caller to summarize the feed and report any critical errors.
        FeedLoadResult result = new FeedLoadResult();

        try {
            // Begin tracking time. FIXME: should this follow the connect/register and begin with the table loads?
            long startTime = System.currentTimeMillis();
            // We get a single connection object and share it across several different methods.
            // This ensures that actions taken in one method are visible to all subsequent SQL statements.
            // If we create a schema or table on one connection, then access it in a separate connection, we have no
            // guarantee that it exists when the accessing statement is executed.
            connection = dataSource.getConnection();
            File gtfsFile = new File(gtfsFilePath);
            this.zip = new ZipFile(gtfsFilePath);
            // Generate a unique prefix that will identify this feed.
            // Prefixes ("schema" names) based on feed_id and feed_version get very messy, so we use random unique IDs.
            // We don't want to use an auto-increment numeric primary key because these need to be alphabetical.
            // Although ID collisions are theoretically possible, they are improbable in the extreme because our IDs
            // are long enough to have as much entropy as a UUID. So we don't really need to check for uniqueness and
            // retry in a loop.
            // TODO handle the case where we don't want any prefix.
            this.tablePrefix = randomIdString();
            result.filename = gtfsFilePath;
            result.uniqueIdentifier = tablePrefix;
            
            // The order of the following four lines should not be changed because the schema needs to be in place
            // before the error storage can be constructed, which in turn needs to exist in case any errors are
            // encountered during the loading process.
            {
                createSchema(connection, tablePrefix);
                //the SQLErrorStorage constructor expects the tablePrefix to contain the dot separator.
                this.errorStorage = new SQLErrorStorage(connection, tablePrefix + ".", true);
                //registerFeed accesses this.tablePrefix which shouldn't contain the dot separator.
                registerFeed(gtfsFile);
                // Include the dot separator in the table prefix from this point onwards.
                // This allows everything to work even when there's no prefix.
                this.tablePrefix += ".";
            }
            // Load each table in turn, saving some summary information about what happened during each table load
            result.agency = load(Table.AGENCY);
            result.calendar = load(Table.CALENDAR);
            result.calendarDates = load(Table.CALENDAR_DATES);
            result.routes = load(Table.ROUTES);
            result.fareAttributes = load(Table.FARE_ATTRIBUTES);
            result.fareRules = load(Table.FARE_RULES);
            result.feedInfo = load(Table.FEED_INFO);
            result.shapes = load(Table.SHAPES);
            result.stops = load(Table.STOPS);
            result.transfers = load(Table.TRANSFERS);
            result.trips = load(Table.TRIPS); // refs routes
            result.frequencies = load(Table.FREQUENCIES); // refs trips
            result.stopTimes = load(Table.STOP_TIMES);
            result.errorCount = errorStorage.getErrorCount();
            // This will commit and close the single connection that has been shared between all preceding load steps.
            errorStorage.commitAndClose();
            zip.close();
            result.completionTime = System.currentTimeMillis();
            result.loadTimeMillis = result.completionTime - startTime;
            LOG.info("Loading tables took {} sec", result.loadTimeMillis / 1000);
        } catch (Exception ex) {
            // TODO catch exceptions separately while loading each table so load can continue, store in TableLoadResult
            LOG.error("Exception while loading GTFS file: {}", ex.toString());
            ex.printStackTrace();
            result.fatalException = ex.toString();
        }
        return result;
    }
public FeedLoadResult loadTables () {

        // This result object will be returned to the caller to summarize the feed and report any critical errors.
        FeedLoadResult result = new FeedLoadResult();

        try {
            // Begin tracking time. FIXME: should this follow the connect/register and begin with the table loads?
            long startTime = System.currentTimeMillis();
            // We get a single connection object and share it across several different methods.
            // This ensures that actions taken in one method are visible to all subsequent SQL statements.
            // If we create a schema or table on one connection, then access it in a separate connection, we have no
            // guarantee that it exists when the accessing statement is executed.
            connection = dataSource.getConnection();
            File gtfsFile = new File(gtfsFilePath);
            this.zip = new ZipFile(gtfsFilePath);
            // Generate a unique prefix that will identify this feed.
            // Prefixes ("schema" names) based on feed_id and feed_version get very messy, so we use random unique IDs.
            // We don't want to use an auto-increment numeric primary key because these need to be alphabetical.
            // Although ID collisions are theoretically possible, they are improbable in the extreme because our IDs
            // are long enough to have as much entropy as a UUID. So we don't really need to check for uniqueness and
            // retry in a loop.
            // TODO handle the case where we don't want any prefix.
            this.tablePrefix = randomIdString();
            result.filename = gtfsFilePath;
            result.uniqueIdentifier = tablePrefix;
            
            // The order of the following four lines should not be changed because the schema needs to be in place
            // before the error storage can be constructed, which in turn needs to exist in case any errors are
            // encountered during the loading process.
            {
                createSchema(connection, tablePrefix);
                //the SQLErrorStorage constructor expects the tablePrefix to contain the dot separator.
                this.errorStorage = new SQLErrorStorage(connection, tablePrefix + ".", true);
                //registerFeed accesses this.tablePrefix which shouldn't contain the dot separator.
                registerFeed(gtfsFile);
                // Include the dot separator in the table prefix from this point onwards.
                // This allows everything to work even when there's no prefix.
                this.tablePrefix += ".";
            }
            // Load each table in turn, saving some summary information about what happened during each table load
            result.agency = load(Table.AGENCY);
            result.calendar = load(Table.CALENDAR);
            result.calendarDates = load(Table.CALENDAR_DATES);
            result.routes = load(Table.ROUTES);
            result.fareAttributes = load(Table.FARE_ATTRIBUTES);
            result.fareRules = load(Table.FARE_RULES);
            result.feedInfo = load(Table.FEED_INFO);
            result.shapes = load(Table.SHAPES);
            result.stops = load(Table.STOPS);
            result.transfers = load(Table.TRANSFERS);
            result.trips = load(Table.TRIPS); // refs routes
            result.frequencies = load(Table.FREQUENCIES); // refs trips
            result.stopTimes = load(Table.STOP_TIMES);
            result.errorCount = errorStorage.getErrorCount();
            // This will commit and close the single connection that has been shared between all preceding load steps.
            errorStorage.commitAndClose();
            zip.close();
            result.completionTime = System.currentTimeMillis();
            result.loadTimeMillis = result.completionTime - startTime;
            LOG.info("Loading tables took {} sec", result.loadTimeMillis / 1000);
        } catch (Exception ex) {
            // TODO catch exceptions separately while loading each table so load can continue, store in TableLoadResult
            LOG.error("Exception while loading GTFS file: {}", ex.toString());
            ex.printStackTrace();
            result.fatalException = ex.toString();
        }
        return result;
    }
public FeedLoadResult loadTables ( ) { // This result object will be returned to the caller to summarize the feed and report any critical errors. FeedLoadResult result = new FeedLoadResult ( ) ; try { // Begin tracking time. FIXME: should this follow the connect/register and begin with the table loads? long startTime = System . currentTimeMillis ( ) ; // We get a single connection object and share it across several different methods. // This ensures that actions taken in one method are visible to all subsequent SQL statements. // If we create a schema or table on one connection, then access it in a separate connection, we have no // guarantee that it exists when the accessing statement is executed. connection = dataSource . getConnection ( ) ; File gtfsFile = new File ( gtfsFilePath ) ; this . zip = new ZipFile ( gtfsFilePath ) ; // Generate a unique prefix that will identify this feed. // Prefixes ("schema" names) based on feed_id and feed_version get very messy, so we use random unique IDs. // We don't want to use an auto-increment numeric primary key because these need to be alphabetical. // Although ID collisions are theoretically possible, they are improbable in the extreme because our IDs // are long enough to have as much entropy as a UUID. So we don't really need to check for uniqueness and // retry in a loop. // TODO handle the case where we don't want any prefix. this . tablePrefix = randomIdString ( ) ; result . filename = gtfsFilePath ; result . uniqueIdentifier = tablePrefix ; // The order of the following four lines should not be changed because the schema needs to be in place // before the error storage can be constructed, which in turn needs to exist in case any errors are // encountered during the loading process. { createSchema ( connection , tablePrefix ) ; //the SQLErrorStorage constructor expects the tablePrefix to contain the dot separator. this . errorStorage = new SQLErrorStorage ( connection , tablePrefix + "." , true ) ; //registerFeed accesses this.tablePrefix which shouldn't contain the dot separator. registerFeed ( gtfsFile ) ; // Include the dot separator in the table prefix from this point onwards. // This allows everything to work even when there's no prefix. this . tablePrefix += "." ; } // Load each table in turn, saving some summary information about what happened during each table load result . agency = load ( Table . AGENCY ) ; result . calendar = load ( Table . CALENDAR ) ; result . calendarDates = load ( Table . CALENDAR_DATES ) ; result . routes = load ( Table . ROUTES ) ; result . fareAttributes = load ( Table . FARE_ATTRIBUTES ) ; result . fareRules = load ( Table . FARE_RULES ) ; result . feedInfo = load ( Table . FEED_INFO ) ; result . shapes = load ( Table . SHAPES ) ; result . stops = load ( Table . STOPS ) ; result . transfers = load ( Table . TRANSFERS ) ; result . trips = load ( Table . TRIPS ) ; // refs routes result . frequencies = load ( Table . FREQUENCIES ) ; // refs trips result . stopTimes = load ( Table . STOP_TIMES ) ; result . errorCount = errorStorage . getErrorCount ( ) ; // This will commit and close the single connection that has been shared between all preceding load steps. errorStorage . commitAndClose ( ) ; zip . close ( ) ; result . completionTime = System . currentTimeMillis ( ) ; result . loadTimeMillis = result . completionTime - startTime ; LOG . info ( "Loading tables took {} sec" , result . loadTimeMillis / 1000 ) ; } catch ( Exception ex ) { // TODO catch exceptions separately while loading each table so load can continue, store in TableLoadResult LOG . error ( "Exception while loading GTFS file: {}" , ex . toString ( ) ) ; ex . printStackTrace ( ) ; result . fatalException = ex . toString ( ) ; } return result ; }
There appears to be no advantage to loading tables in parallel, as the whole loading process is I/O bound.
There appears to be no advantage to loading tables in parallel as the whole loading process is I / O bound .