src/main/java/com/conveyal/gtfs/GTFSFeed.java
public void loadFromFile(ZipFile zip, String fid) throws Exception {
        if (this.loaded) throw new UnsupportedOperationException("Attempt to load GTFS into existing database");

        // NB we don't have a single CRC for the file, so we combine all the CRCs of the component files. NB we are not
        // simply summing the CRCs because CRCs are (I assume) uniformly randomly distributed throughout the width of a
        // long, so summing them is a convolution which moves towards a Gaussian with mean 0 (i.e. more concentrated
        // probability in the center), degrading the quality of the hash. Instead we XOR. Assuming each bit is independent,
        // this will yield a nice uniformly distributed result, because when combining two bits there is an equal
        // probability of any input, which means an equal probability of any output. At least I think that's all correct.
        // Repeated XOR is not commutative but zip.stream returns files in the order they are in the central directory
        // of the zip file, so that's not a problem.
        checksum = zip.stream().mapToLong(ZipEntry::getCrc).reduce((l1, l2) -> l1 ^ l2).getAsLong();

        db.getAtomicLong("checksum").set(checksum);

        new FeedInfo.Loader(this).loadTable(zip);
        // maybe we should just point to the feed object itself instead of its ID, and null out its stoptimes map after loading
        if (fid != null) {
            feedId = fid;
            LOG.info("Feed ID is undefined, pester maintainers to include a feed ID. Using file name {}.", feedId); // TODO log an error, ideally feeds should include a feedID
        }
        else if (feedId == null || feedId.isEmpty()) {
            feedId = new File(zip.getName()).getName().replaceAll("\\.zip$", "");
            LOG.info("Feed ID is undefined, pester maintainers to include a feed ID. Using file name {}.", feedId); // TODO log an error, ideally feeds should include a feedID
        }
        else {
            LOG.info("Feed ID is '{}'.", feedId);
        }

        db.getAtomicString("feed_id").set(feedId);

        new Agency.Loader(this).loadTable(zip);

        // calendars and calendar dates are joined into services. This means a lot of manipulating service objects as
        // they are loaded; since mapdb keys/values are immutable, load them in memory then copy them to MapDB once
        // we're done loading them
        Map<String, Service> serviceTable = new HashMap<>();
        new Calendar.Loader(this, serviceTable).loadTable(zip);
        new CalendarDate.Loader(this, serviceTable).loadTable(zip);
        this.services.putAll(serviceTable);
        serviceTable = null; // free memory

        // Same deal
        Map<String, Fare> fares = new HashMap<>();
        new FareAttribute.Loader(this, fares).loadTable(zip);
        new FareRule.Loader(this, fares).loadTable(zip);
        this.fares.putAll(fares);
        fares = null; // free memory

        new Route.Loader(this).loadTable(zip);
        new ShapePoint.Loader(this).loadTable(zip);
        new Stop.Loader(this).loadTable(zip);
        new Transfer.Loader(this).loadTable(zip);
        new Trip.Loader(this).loadTable(zip);
        new Frequency.Loader(this).loadTable(zip);
        new StopTime.Loader(this).loadTable(zip); // comment out this line for quick testing using NL feed
        LOG.info("{} errors", errors.size());
        for (GTFSError error : errors) {
            LOG.info("{}", error);
        }
        LOG.info("Building stop to stop times index");
        Bind.histogram(stop_times, stopCountByStopTime, (key, stopTime) -> stopTime.stop_id);
        Bind.secondaryKeys(stop_times, stopStopTimeSet, (key, stopTime) -> new String[] {stopTime.stop_id});
        LOG.info("Building trips per service index");
        Bind.secondaryKeys(trips, tripsPerService, (key, trip) -> new String[] {trip.service_id});
        LOG.info("Building services per date index");
        Bind.secondaryKeys(services, servicesPerDate, (key, service) -> {

            LocalDate startDate = service.calendar != null
                    ? service.calendar.start_date
                    : service.calendar_dates.keySet().stream().sorted().findFirst().get();
            LocalDate endDate = service.calendar != null
                    ? service.calendar.end_date
                    : service.calendar_dates.keySet().stream().sorted().reduce((first, second) -> second).get();
            // end date for Period.between is not inclusive
            int daysOfService = (int) ChronoUnit.DAYS.between(startDate, endDate.plus(1, ChronoUnit.DAYS));
            return IntStream.range(0, daysOfService)
                    .mapToObj(offset -> startDate.plusDays(offset))
                    .filter(service::activeOn)
                    .map(date -> date.format(dateFormatter))
                    .toArray(size -> new String[size]);
        });

        loaded = true;
    }
public void loadFromFile(ZipFile zip, String fid) throws Exception {
        if (this.loaded) throw new UnsupportedOperationException("Attempt to load GTFS into existing database");

        // NB we don't have a single CRC for the file, so we combine all the CRCs of the component files. NB we are not
        // simply summing the CRCs because CRCs are (I assume) uniformly randomly distributed throughout the width of a
        // long, so summing them is a convolution which moves towards a Gaussian with mean 0 (i.e. more concentrated
        // probability in the center), degrading the quality of the hash. Instead we XOR. Assuming each bit is independent,
        // this will yield a nice uniformly distributed result, because when combining two bits there is an equal
        // probability of any input, which means an equal probability of any output. At least I think that's all correct.
        // Repeated XOR is not commutative but zip.stream returns files in the order they are in the central directory
        // of the zip file, so that's not a problem.
        checksum = zip.stream().mapToLong(ZipEntry::getCrc).reduce((l1, l2) -> l1 ^ l2).getAsLong();

        db.getAtomicLong("checksum").set(checksum);

        new FeedInfo.Loader(this).loadTable(zip);
        // maybe we should just point to the feed object itself instead of its ID, and null out its stoptimes map after loading
        if (fid != null) {
            feedId = fid;
            LOG.info("Feed ID is undefined, pester maintainers to include a feed ID. Using file name {}.", feedId); // TODO log an error, ideally feeds should include a feedID
        }
        else if (feedId == null || feedId.isEmpty()) {
            feedId = new File(zip.getName()).getName().replaceAll("\\.zip$", "");
            LOG.info("Feed ID is undefined, pester maintainers to include a feed ID. Using file name {}.", feedId); // TODO log an error, ideally feeds should include a feedID
        }
        else {
            LOG.info("Feed ID is '{}'.", feedId);
        }

        db.getAtomicString("feed_id").set(feedId);

        new Agency.Loader(this).loadTable(zip);

        // calendars and calendar dates are joined into services. This means a lot of manipulating service objects as
        // they are loaded; since mapdb keys/values are immutable, load them in memory then copy them to MapDB once
        // we're done loading them
        Map<String, Service> serviceTable = new HashMap<>();
        new Calendar.Loader(this, serviceTable).loadTable(zip);
        new CalendarDate.Loader(this, serviceTable).loadTable(zip);
        this.services.putAll(serviceTable);
        serviceTable = null; // free memory

        // Same deal
        Map<String, Fare> fares = new HashMap<>();
        new FareAttribute.Loader(this, fares).loadTable(zip);
        new FareRule.Loader(this, fares).loadTable(zip);
        this.fares.putAll(fares);
        fares = null; // free memory

        new Route.Loader(this).loadTable(zip);
        new ShapePoint.Loader(this).loadTable(zip);
        new Stop.Loader(this).loadTable(zip);
        new Transfer.Loader(this).loadTable(zip);
        new Trip.Loader(this).loadTable(zip);
        new Frequency.Loader(this).loadTable(zip);
        new StopTime.Loader(this).loadTable(zip); // comment out this line for quick testing using NL feed
        LOG.info("{} errors", errors.size());
        for (GTFSError error : errors) {
            LOG.info("{}", error);
        }
        LOG.info("Building stop to stop times index");
        Bind.histogram(stop_times, stopCountByStopTime, (key, stopTime) -> stopTime.stop_id);
        Bind.secondaryKeys(stop_times, stopStopTimeSet, (key, stopTime) -> new String[] {stopTime.stop_id});
        LOG.info("Building trips per service index");
        Bind.secondaryKeys(trips, tripsPerService, (key, trip) -> new String[] {trip.service_id});
        LOG.info("Building services per date index");
        Bind.secondaryKeys(services, servicesPerDate, (key, service) -> {

            LocalDate startDate = service.calendar != null
                    ? service.calendar.start_date
                    : service.calendar_dates.keySet().stream().sorted().findFirst().get();
            LocalDate endDate = service.calendar != null
                    ? service.calendar.end_date
                    : service.calendar_dates.keySet().stream().sorted().reduce((first, second) -> second).get();
            // end date for Period.between is not inclusive
            int daysOfService = (int) ChronoUnit.DAYS.between(startDate, endDate.plus(1, ChronoUnit.DAYS));
            return IntStream.range(0, daysOfService)
                    .mapToObj(offset -> startDate.plusDays(offset))
                    .filter(service::activeOn)
                    .map(date -> date.format(dateFormatter))
                    .toArray(size -> new String[size]);
        });

        loaded = true;
    }
public void loadFromFile ( ZipFile zip , String fid ) throws Exception { if ( this . loaded ) throw new UnsupportedOperationException ( "Attempt to load GTFS into existing database" ) ; // NB we don't have a single CRC for the file, so we combine all the CRCs of the component files. NB we are not // simply summing the CRCs because CRCs are (I assume) uniformly randomly distributed throughout the width of a // long, so summing them is a convolution which moves towards a Gaussian with mean 0 (i.e. more concentrated // probability in the center), degrading the quality of the hash. Instead we XOR. Assuming each bit is independent, // this will yield a nice uniformly distributed result, because when combining two bits there is an equal // probability of any input, which means an equal probability of any output. At least I think that's all correct. // Repeated XOR is not commutative but zip.stream returns files in the order they are in the central directory // of the zip file, so that's not a problem. checksum = zip . stream ( ) . mapToLong ( ZipEntry :: getCrc ) . reduce ( ( l1 , l2 ) -> l1 ^ l2 ) . getAsLong ( ) ; db . getAtomicLong ( "checksum" ) . set ( checksum ) ; new FeedInfo . Loader ( this ) . loadTable ( zip ) ; // maybe we should just point to the feed object itself instead of its ID, and null out its stoptimes map after loading if ( fid != null ) { feedId = fid ; LOG . info ( "Feed ID is undefined, pester maintainers to include a feed ID. Using file name {}." , feedId ) ; // TODO log an error, ideally feeds should include a feedID } else if ( feedId == null || feedId . isEmpty ( ) ) { feedId = new File ( zip . getName ( ) ) . getName ( ) . replaceAll ( "\\.zip$" , "" ) ; LOG . info ( "Feed ID is undefined, pester maintainers to include a feed ID. Using file name {}." , feedId ) ; // TODO log an error, ideally feeds should include a feedID } else { LOG . info ( "Feed ID is '{}'." , feedId ) ; } db . getAtomicString ( "feed_id" ) . set ( feedId ) ; new Agency . Loader ( this ) . loadTable ( zip ) ; // calendars and calendar dates are joined into services. This means a lot of manipulating service objects as // they are loaded; since mapdb keys/values are immutable, load them in memory then copy them to MapDB once // we're done loading them Map < String , Service > serviceTable = new HashMap <> ( ) ; new Calendar . Loader ( this , serviceTable ) . loadTable ( zip ) ; new CalendarDate . Loader ( this , serviceTable ) . loadTable ( zip ) ; this . services . putAll ( serviceTable ) ; serviceTable = null ; // free memory // Same deal Map < String , Fare > fares = new HashMap <> ( ) ; new FareAttribute . Loader ( this , fares ) . loadTable ( zip ) ; new FareRule . Loader ( this , fares ) . loadTable ( zip ) ; this . fares . putAll ( fares ) ; fares = null ; // free memory new Route . Loader ( this ) . loadTable ( zip ) ; new ShapePoint . Loader ( this ) . loadTable ( zip ) ; new Stop . Loader ( this ) . loadTable ( zip ) ; new Transfer . Loader ( this ) . loadTable ( zip ) ; new Trip . Loader ( this ) . loadTable ( zip ) ; new Frequency . Loader ( this ) . loadTable ( zip ) ; new StopTime . Loader ( this ) . loadTable ( zip ) ; // comment out this line for quick testing using NL feed LOG . info ( "{} errors" , errors . size ( ) ) ; for ( GTFSError error : errors ) { LOG . info ( "{}" , error ) ; } LOG . info ( "Building stop to stop times index" ) ; Bind . histogram ( stop_times , stopCountByStopTime , ( key , stopTime ) -> stopTime . stop_id ) ; Bind . secondaryKeys ( stop_times , stopStopTimeSet , ( key , stopTime ) -> new String [ ] { stopTime . stop_id } ) ; LOG . info ( "Building trips per service index" ) ; Bind . secondaryKeys ( trips , tripsPerService , ( key , trip ) -> new String [ ] { trip . service_id } ) ; LOG . info ( "Building services per date index" ) ; Bind . secondaryKeys ( services , servicesPerDate , ( key , service ) -> { LocalDate startDate = service . calendar != null ? service . calendar . start_date : service . calendar_dates . keySet ( ) . stream ( ) . sorted ( ) . findFirst ( ) . get ( ) ; LocalDate endDate = service . calendar != null ? service . calendar . end_date : service . calendar_dates . keySet ( ) . stream ( ) . sorted ( ) . reduce ( ( first , second ) -> second ) . get ( ) ; // end date for Period.between is not inclusive int daysOfService = ( int ) ChronoUnit . DAYS . between ( startDate , endDate . plus ( 1 , ChronoUnit . DAYS ) ) ; return IntStream . range ( 0 , daysOfService ) . mapToObj ( offset -> startDate . plusDays ( offset ) ) . filter ( service :: activeOn ) . map ( date -> date . format ( dateFormatter ) ) . toArray ( size -> new String [ size ] ) ; } ) ; loaded = true ; }
The order in which we load the tables is important for two reasons.
1. We must load feed_info first so we know the feed ID before loading any other entities. This could be relaxed
by having entities point to the feed object rather than its ID String.
2. Referenced entities must be loaded before any entities that reference them. This is because we check
referential integrity while the files are being loaded. This is done on the fly during loading because it allows
us to associate a line number with errors in objects that don't have any other clear identifier.

Interestingly, all references are resolvable when tables are loaded in alphabetical order.
The order in which we load the tables is important for two reasons . 1 . We must load feed_info first so we know the feed ID before loading any other entities . This could be relaxed by having entities point to the feed object rather than its ID String . 2 . Referenced entities must be loaded before any entities that reference them . This is because we check referential integrity while the files are being loaded . This is done on the fly during loading because it allows us to associate a line number with errors in objects that don t have any other clear identifier .