diff --git a/src/main/java/com/conveyal/gtfs/GTFS.java b/src/main/java/com/conveyal/gtfs/GTFS.java index 52bc37f5a..32d1147c3 100644 --- a/src/main/java/com/conveyal/gtfs/GTFS.java +++ b/src/main/java/com/conveyal/gtfs/GTFS.java @@ -5,6 +5,7 @@ import com.conveyal.gtfs.loader.JdbcGtfsExporter; import com.conveyal.gtfs.loader.JdbcGtfsLoader; import com.conveyal.gtfs.loader.JdbcGtfsSnapshotter; +import com.conveyal.gtfs.loader.SnapshotResult; import com.conveyal.gtfs.util.InvalidNamespaceException; import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.databind.ObjectMapper; @@ -83,9 +84,9 @@ public static FeedLoadResult load (String filePath, DataSource dataSource) { * @param dataSource JDBC connection to existing database * @return FIXME should this be a separate SnapshotResult object? */ - public static FeedLoadResult makeSnapshot (String feedId, DataSource dataSource) { + public static SnapshotResult makeSnapshot (String feedId, DataSource dataSource) { JdbcGtfsSnapshotter snapshotter = new JdbcGtfsSnapshotter(feedId, dataSource); - FeedLoadResult result = snapshotter.copyTables(); + SnapshotResult result = snapshotter.copyTables(); return result; } diff --git a/src/main/java/com/conveyal/gtfs/error/SQLErrorStorage.java b/src/main/java/com/conveyal/gtfs/error/SQLErrorStorage.java index eea55f3a2..99b8a0df1 100644 --- a/src/main/java/com/conveyal/gtfs/error/SQLErrorStorage.java +++ b/src/main/java/com/conveyal/gtfs/error/SQLErrorStorage.java @@ -131,6 +131,7 @@ private void commit() { */ public void commitAndClose() { try { + LOG.info("Committing errors and closing SQL connection."); this.commit(); // Close the connection permanently (should be called only after errorStorage instance no longer needed). connection.close(); diff --git a/src/main/java/com/conveyal/gtfs/loader/Feed.java b/src/main/java/com/conveyal/gtfs/loader/Feed.java index c56fb099b..bb896196f 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Feed.java +++ b/src/main/java/com/conveyal/gtfs/loader/Feed.java @@ -81,9 +81,7 @@ public ValidationResult validate () { SQLErrorStorage errorStorage = null; try { errorStorage = new SQLErrorStorage(dataSource.getConnection(), tablePrefix, false); - } catch (SQLException ex) { - throw new StorageException(ex); - } catch (InvalidNamespaceException ex) { + } catch (SQLException | InvalidNamespaceException ex) { throw new StorageException(ex); } int errorCountBeforeValidation = errorStorage.getErrorCount(); diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java index ca05d7a74..9708c0f50 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java @@ -309,7 +309,7 @@ public FeedLoadResult exportTables() { // TableLoadResult. LOG.error("Exception while creating snapshot: {}", ex.toString()); ex.printStackTrace(); - result.fatalException = ex.getMessage(); + result.fatalException = ex.toString(); } return result; } @@ -354,7 +354,7 @@ private TableLoadResult export (Table table, Connection connection) { } catch (SQLException e) { LOG.error("failed to generate select statement for existing fields"); TableLoadResult tableLoadResult = new TableLoadResult(); - tableLoadResult.fatalException = e.getMessage(); + tableLoadResult.fatalException = e.toString(); e.printStackTrace(); return tableLoadResult; } @@ -402,7 +402,7 @@ private TableLoadResult export (Table table, String filterSql) { } catch (SQLException ex) { ex.printStackTrace(); } - tableLoadResult.fatalException = e.getMessage(); + tableLoadResult.fatalException = e.toString(); LOG.error("Exception while exporting tables", e); } return tableLoadResult; diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java index 92e83d17b..0a536b5ce 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java @@ -169,7 +169,7 @@ public FeedLoadResult loadTables () { // TODO catch exceptions separately while loading each table so load can continue, store in TableLoadResult LOG.error("Exception while loading GTFS file: {}", ex.toString()); ex.printStackTrace(); - result.fatalException = ex.getMessage(); + result.fatalException = ex.toString(); } return result; } @@ -251,7 +251,7 @@ private void registerFeed (File gtfsFile) { connection.commit(); LOG.info("Created new feed namespace: {}", insertStatement); } catch (Exception ex) { - LOG.error("Exception while registering new feed namespace in feeds table: {}", ex.getMessage()); + LOG.error("Exception while registering new feed namespace in feeds table", ex); DbUtils.closeQuietly(connection); } } diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java index 96f946044..415df8aa5 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java @@ -56,9 +56,9 @@ public JdbcGtfsSnapshotter(String feedId, DataSource dataSource) { /** * Copy primary entity tables as well as Pattern and PatternStops tables. */ - public FeedLoadResult copyTables() { + public SnapshotResult copyTables() { // This result object will be returned to the caller to summarize the feed and report any critical errors. - FeedLoadResult result = new FeedLoadResult(); + SnapshotResult result = new SnapshotResult(); try { long startTime = System.currentTimeMillis(); @@ -89,7 +89,7 @@ public FeedLoadResult copyTables() { copy(Table.PATTERNS, true); copy(Table.PATTERN_STOP, true); // see method comments fo why different logic is needed for this table - createScheduleExceptionsTable(); + result.scheduleExceptions = createScheduleExceptionsTable(); result.shapes = copy(Table.SHAPES, true); result.stops = copy(Table.STOPS, true); // TODO: Should we defer index creation on stop times? @@ -106,7 +106,7 @@ public FeedLoadResult copyTables() { // TableLoadResult. LOG.error("Exception while creating snapshot: {}", ex.toString()); ex.printStackTrace(); - result.fatalException = ex.getMessage(); + result.fatalException = ex.toString(); } return result; } @@ -145,7 +145,7 @@ private TableLoadResult copy (Table table, boolean createIndexes) { connection.commit(); LOG.info("Done."); } catch (Exception ex) { - tableLoadResult.fatalException = ex.getMessage(); + tableLoadResult.fatalException = ex.toString(); LOG.error("Error: ", ex); try { connection.rollback(); @@ -217,6 +217,11 @@ private TableLoadResult createScheduleExceptionsTable() { Multimap removedServiceForDate = HashMultimap.create(); Multimap addedServiceForDate = HashMultimap.create(); for (CalendarDate calendarDate : calendarDates) { + // Skip any null dates + if (calendarDate.date == null) { + LOG.warn("Encountered calendar date record with null value for date field. Skipping."); + continue; + } String date = calendarDate.date.format(DateTimeFormatter.BASIC_ISO_DATE); if (calendarDate.exception_type == 1) { addedServiceForDate.put(date, calendarDate.service_id); @@ -293,8 +298,8 @@ private TableLoadResult createScheduleExceptionsTable() { } connection.commit(); - } catch (SQLException e) { - tableLoadResult.fatalException = e.getMessage(); + } catch (Exception e) { + tableLoadResult.fatalException = e.toString(); LOG.error("Error creating schedule Exceptions: ", e); e.printStackTrace(); try { @@ -427,7 +432,7 @@ private void registerSnapshot () { connection.commit(); LOG.info("Created new snapshot namespace: {}", insertStatement); } catch (Exception ex) { - LOG.error("Exception while registering snapshot namespace in feeds table: {}", ex.getMessage()); + LOG.error("Exception while registering snapshot namespace in feeds table", ex); DbUtils.closeQuietly(connection); } } diff --git a/src/main/java/com/conveyal/gtfs/loader/SnapshotResult.java b/src/main/java/com/conveyal/gtfs/loader/SnapshotResult.java new file mode 100644 index 000000000..9c596e345 --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/loader/SnapshotResult.java @@ -0,0 +1,11 @@ +package com.conveyal.gtfs.loader; + +/** + * This contains the result of a feed snapshot operation. It is nearly identical to {@link FeedLoadResult} except that + * it has some additional tables that only exist for snapshots/editor feeds. + */ +public class SnapshotResult extends FeedLoadResult { + private static final long serialVersionUID = 1L; + + public TableLoadResult scheduleExceptions; +} diff --git a/src/main/java/com/conveyal/gtfs/validator/ServiceValidator.java b/src/main/java/com/conveyal/gtfs/validator/ServiceValidator.java index 7097da1c1..3b1639c26 100644 --- a/src/main/java/com/conveyal/gtfs/validator/ServiceValidator.java +++ b/src/main/java/com/conveyal/gtfs/validator/ServiceValidator.java @@ -117,8 +117,7 @@ public void complete(ValidationResult validationResult) { } } } catch (Exception ex) { - LOG.error(ex.getMessage()); - ex.printStackTrace(); + LOG.error("Error validating service entries (merging calendars and calendar_dates)", ex); // Continue on to next calendar entry. } } @@ -181,6 +180,11 @@ select durations.service_id, duration_seconds, days_active from ( LocalDate firstDate = LocalDate.MAX; LocalDate lastDate = LocalDate.MIN; for (LocalDate date : dateInfoForDate.keySet()) { + // If the date is invalid, skip. + if (date == null) { + LOG.error("Encountered null date. Did something go wrong with computeIfAbsent?"); + continue; + } if (date.isBefore(firstDate)) firstDate = date; if (date.isAfter(lastDate)) lastDate = date; } @@ -191,7 +195,8 @@ select durations.service_id, duration_seconds, days_active from ( validationResult.firstCalendarDate = firstDate; validationResult.lastCalendarDate = lastDate; // Is this any different? firstDate.until(lastDate, ChronoUnit.DAYS); - int nDays = (int) ChronoUnit.DAYS.between(firstDate, lastDate) + 1; + // If no days were found in the dateInfoForDate, nDays is a very large negative number, so we default to 0. + int nDays = Math.max(0, (int) ChronoUnit.DAYS.between(firstDate, lastDate) + 1); validationResult.dailyBusSeconds = new int[nDays]; validationResult.dailyTramSeconds = new int[nDays]; validationResult.dailyMetroSeconds = new int[nDays]; diff --git a/src/test/java/com/conveyal/gtfs/GTFSTest.java b/src/test/java/com/conveyal/gtfs/GTFSTest.java index 9a5acca3a..3e2c8fe69 100644 --- a/src/test/java/com/conveyal/gtfs/GTFSTest.java +++ b/src/test/java/com/conveyal/gtfs/GTFSTest.java @@ -1,9 +1,16 @@ package com.conveyal.gtfs; +import com.conveyal.gtfs.error.NewGTFSErrorType; import com.conveyal.gtfs.loader.FeedLoadResult; +import com.conveyal.gtfs.loader.SnapshotResult; +import com.conveyal.gtfs.storage.ExpectedFieldType; +import com.conveyal.gtfs.storage.PersistenceExpectation; +import com.conveyal.gtfs.storage.RecordExpectation; import com.conveyal.gtfs.validator.ValidationResult; import com.csvreader.CsvReader; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; import com.google.common.io.Files; import org.apache.commons.io.FileUtils; import org.apache.commons.io.input.BOMInputStream; @@ -22,9 +29,9 @@ import java.io.PrintStream; import java.nio.charset.Charset; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Collection; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -98,6 +105,26 @@ public void canLoadAndExportSimpleAgency() { ); } + /** + * Tests that a GTFS feed with bad date values in calendars.txt and calendar_dates.txt can pass the integration test. + */ + @Test + public void canLoadFeedWithBadDates () { + PersistenceExpectation[] expectations = PersistenceExpectation.list( + new PersistenceExpectation( + "calendar", + new RecordExpectation[]{ + new RecordExpectation("start_date", null) + } + ) + ); + assertThat( + "Integration test passes", + runIntegrationTestOnFolder("fake-agency-bad-calendar-date", nullValue(), expectations), + equalTo(true) + ); + } + /** * Tests whether or not "fake-agency" GTFS can be placed in a zipped subdirectory and loaded/exported successfully. */ @@ -119,11 +146,7 @@ public void canLoadAndExportSimpleAgencyInSubDirectory() { } // TODO Add error expectations argument that expects NewGTFSErrorType.TABLE_IN_SUBDIRECTORY error type. assertThat( - runIntegrationTestOnZipFile( - zipFileName, - nullValue(), - fakeAgencyPersistenceExpectations - ), + runIntegrationTestOnZipFile(zipFileName, nullValue(), fakeAgencyPersistenceExpectations), equalTo(true) ); } @@ -220,7 +243,7 @@ private boolean runIntegrationTestOnFolder( } /** - * A helper method that will run GTFS.main with a certain zip file. + * A helper method that will run GTFS#main with a certain zip file. * This tests whether a GTFS zip file can be loaded without any errors. * * After the GTFS is loaded, this will also initiate an export of a GTFS from the database and check @@ -251,7 +274,7 @@ private boolean runIntegrationTestOnZipFile( assertThat(validationResult.fatalException, is(fatalExceptionExpectation)); namespace = loadResult.uniqueIdentifier; - assertThatImportedGtfsMeetsExpectations(dbConnectionUrl, namespace, persistenceExpectations); + assertThatImportedGtfsMeetsExpectations(dataSource.getConnection(), namespace, persistenceExpectations); } catch (SQLException e) { TestUtils.dropDB(newDBName); e.printStackTrace(); @@ -261,9 +284,9 @@ private boolean runIntegrationTestOnZipFile( throw e; } - // Verify that exporting the feed (in non-editor mode) completes and data is outputed properly + // Verify that exporting the feed (in non-editor mode) completes and data is outputted properly try { - LOG.info("export GTFS from created namespase"); + LOG.info("export GTFS from created namespace"); File tempFile = exportGtfs(namespace, dataSource, false); assertThatExportedGtfsMeetsExpectations(tempFile, persistenceExpectations, false); } catch (IOException e) { @@ -278,9 +301,10 @@ private boolean runIntegrationTestOnZipFile( // Verify that making a snapshot from an existing feed database, then exporting that snapshot to a GTFS zip file // works as expected try { - LOG.info("copy GTFS from created namespase"); - FeedLoadResult copyResult = GTFS.makeSnapshot(namespace, dataSource); - LOG.info("export GTFS from copied namespase"); + LOG.info("copy GTFS from created namespace"); + SnapshotResult copyResult = GTFS.makeSnapshot(namespace, dataSource); + assertThatSnapshotIsErrorFree(copyResult); + LOG.info("export GTFS from copied namespace"); File tempFile = exportGtfs(copyResult.uniqueIdentifier, dataSource, true); assertThatExportedGtfsMeetsExpectations(tempFile, persistenceExpectations, true); } catch (IOException e) { @@ -293,6 +317,60 @@ private boolean runIntegrationTestOnZipFile( return true; } + private void assertThatLoadIsErrorFree(FeedLoadResult loadResult) { + assertThat(loadResult.fatalException, is(nullValue())); + assertThat(loadResult.agency.fatalException, is(nullValue())); + assertThat(loadResult.calendar.fatalException, is(nullValue())); + assertThat(loadResult.calendarDates.fatalException, is(nullValue())); + assertThat(loadResult.fareAttributes.fatalException, is(nullValue())); + assertThat(loadResult.fareRules.fatalException, is(nullValue())); + assertThat(loadResult.feedInfo.fatalException, is(nullValue())); + assertThat(loadResult.frequencies.fatalException, is(nullValue())); + assertThat(loadResult.routes.fatalException, is(nullValue())); + assertThat(loadResult.shapes.fatalException, is(nullValue())); + assertThat(loadResult.stops.fatalException, is(nullValue())); + assertThat(loadResult.stopTimes.fatalException, is(nullValue())); + assertThat(loadResult.transfers.fatalException, is(nullValue())); + assertThat(loadResult.trips.fatalException, is(nullValue())); + } + + private void assertThatSnapshotIsErrorFree(SnapshotResult snapshotResult) { + assertThatLoadIsErrorFree(snapshotResult); + assertThat(snapshotResult.scheduleExceptions.fatalException, is(nullValue())); + } + + private String zipFolderAndLoadGTFS(String folderName, DataSource dataSource, PersistenceExpectation[] persistenceExpectations) { + // zip up test folder into temp zip file + String zipFileName; + try { + zipFileName = TestUtils.zipFolderFiles(folderName, true); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + String namespace; + // Verify that loading the feed completes and data is stored properly + try { + // load and validate feed + LOG.info("load and validate feed"); + FeedLoadResult loadResult = GTFS.load(zipFileName, dataSource); + ValidationResult validationResult = GTFS.validate(loadResult.uniqueIdentifier, dataSource); + DataSource ds = GTFS.createDataSource( + dataSource.getConnection().getMetaData().getURL(), + null, + null + ); + assertThatLoadIsErrorFree(loadResult); + assertThat(validationResult.fatalException, is(nullValue())); + namespace = loadResult.uniqueIdentifier; + assertThatImportedGtfsMeetsExpectations(ds.getConnection(), namespace, persistenceExpectations); + } catch (SQLException | AssertionError e) { + e.printStackTrace(); + return null; + } + return namespace; + } + /** * Helper function to export a GTFS from the database to a temporary zip file. */ @@ -302,18 +380,34 @@ private File exportGtfs(String namespace, DataSource dataSource, boolean fromEdi return tempFile; } + private class ValuePair { + private final Object expected; + private final Object found; + private ValuePair (Object expected, Object found) { + this.expected = expected; + this.found = found; + } + } + /** * Run through the list of persistence expectations to make sure that the feed was imported properly into the * database. */ private void assertThatImportedGtfsMeetsExpectations( - String dbConnectionUrl, + Connection connection, String namespace, PersistenceExpectation[] persistenceExpectations ) throws SQLException { + // Store field mismatches here (to provide assertion statements with more details). + Multimap fieldsWithMismatches = ArrayListMultimap.create(); + // Check that no validators failed during validation. + assertThat( + "One or more validators failed during GTFS import.", + countValidationErrorsOfType(connection, namespace, NewGTFSErrorType.VALIDATOR_FAILED), + equalTo(0) + ); // run through testing expectations - LOG.info("testing expecations of record storage in the database"); - Connection conn = DriverManager.getConnection(dbConnectionUrl); + LOG.info("testing expectations of record storage in the database"); for (PersistenceExpectation persistenceExpectation : persistenceExpectations) { // select all entries from a table String sql = String.format( @@ -322,7 +416,7 @@ private void assertThatImportedGtfsMeetsExpectations( persistenceExpectation.tableName ); LOG.info(sql); - ResultSet rs = conn.prepareStatement(sql).executeQuery(); + ResultSet rs = connection.prepareStatement(sql).executeQuery(); boolean foundRecord = false; int numRecordsSearched = 0; while (rs.next()) { @@ -332,32 +426,48 @@ private void assertThatImportedGtfsMeetsExpectations( for (RecordExpectation recordExpectation: persistenceExpectation.recordExpectations) { switch (recordExpectation.expectedFieldType) { case DOUBLE: + double doubleVal = rs.getDouble(recordExpectation.fieldName); LOG.info(String.format( "%s: %f", recordExpectation.fieldName, - rs.getDouble(recordExpectation.fieldName) + doubleVal )); - if (rs.getDouble(recordExpectation.fieldName) != recordExpectation.doubleExpectation) { + if (doubleVal != recordExpectation.doubleExpectation) { allFieldsMatch = false; } break; case INT: + int intVal = rs.getInt(recordExpectation.fieldName); LOG.info(String.format( "%s: %d", recordExpectation.fieldName, - rs.getInt(recordExpectation.fieldName) + intVal )); - if (rs.getInt(recordExpectation.fieldName) != recordExpectation.intExpectation) { + if (intVal != recordExpectation.intExpectation) { + fieldsWithMismatches.put( + recordExpectation.fieldName, + new ValuePair(recordExpectation.stringExpectation, intVal) + ); allFieldsMatch = false; } break; case STRING: + String strVal = rs.getString(recordExpectation.fieldName); LOG.info(String.format( "%s: %s", recordExpectation.fieldName, - rs.getString(recordExpectation.fieldName) + strVal )); - if (!rs.getString(recordExpectation.fieldName).equals(recordExpectation.stringExpectation)) { + if (strVal == null && recordExpectation.stringExpectation == null) { + break; + } else if ( + (strVal == null && recordExpectation.stringExpectation != null) || + !strVal.equals(recordExpectation.stringExpectation) + ) { + fieldsWithMismatches.put( + recordExpectation.fieldName, + new ValuePair(recordExpectation.stringExpectation, strVal) + ); allFieldsMatch = false; } break; @@ -374,10 +484,28 @@ private void assertThatImportedGtfsMeetsExpectations( break; } } - assertThatPersistenceExpectationRecordWasFound(numRecordsSearched, foundRecord); + assertThatPersistenceExpectationRecordWasFound(numRecordsSearched, foundRecord, fieldsWithMismatches); } } + private static int countValidationErrorsOfType( + Connection connection, + String namespace, + NewGTFSErrorType errorType + ) throws SQLException { + String errorCheckSql = String.format( + "select * from %s.errors where error_type = '%s'", + namespace, + errorType); + LOG.info(errorCheckSql); + ResultSet errorResults = connection.prepareStatement(errorCheckSql).executeQuery(); + int errorCount = 0; + while (errorResults.next()) { + errorCount++; + } + return errorCount; + } + /** * Helper to assert that the GTFS that was exported to a zip file matches all data expectations defined in the * persistence expectations. @@ -387,7 +515,7 @@ private void assertThatExportedGtfsMeetsExpectations( PersistenceExpectation[] persistenceExpectations, boolean fromEditor ) throws IOException { - LOG.info("testing expecations of csv outputs in an exported gtfs"); + LOG.info("testing expectations of csv outputs in an exported gtfs"); ZipFile gtfsZipfile = new ZipFile(tempFile.getAbsolutePath()); @@ -432,7 +560,12 @@ private void assertThatExportedGtfsMeetsExpectations( val, expectation )); - if (!val.equals(expectation)) { + if (val.isEmpty() && expectation == null) { + // First check that the csv value is an empty string and that the expectation is null. Null + // exported from the database to a csv should round trip into an empty string, so this meets the + // expectation. + break; + } else if (!val.equals(expectation)) { // sometimes there are slight differences in decimal precision in various fields // check if the decimal delta is acceptable if (equalsWithNumericDelta(val, recordExpectation)) continue; @@ -446,7 +579,7 @@ private void assertThatExportedGtfsMeetsExpectations( foundRecord = true; } } - assertThatPersistenceExpectationRecordWasFound(numRecordsSearched, foundRecord); + assertThatPersistenceExpectationRecordWasFound(numRecordsSearched, foundRecord, null); } } @@ -469,17 +602,34 @@ private boolean equalsWithNumericDelta(String val, RecordExpectation recordExpec /** * Helper method to make sure a persistence expectation was actually found after searching through records */ - private void assertThatPersistenceExpectationRecordWasFound(int numRecordsSearched, boolean foundRecord) { + private void assertThatPersistenceExpectationRecordWasFound( + int numRecordsSearched, + boolean foundRecord, + Multimap mismatches + ) { assertThat( "No records found in the ResultSet/CSV file", numRecordsSearched, ComparatorMatcherBuilder.usingNaturalOrdering().greaterThan(0) ); - assertThat( - "The record as defined in the PersistenceExpectation was not found.", - foundRecord, - equalTo(true) - ); + if (mismatches != null) { + for (String field : mismatches.keySet()) { + Collection valuePairs = mismatches.get(field); + for (ValuePair valuePair : valuePairs) { + assertThat( + String.format("The value expected for %s was not found", field), + valuePair.expected, + equalTo(valuePair.found) + ); + } + } + } else { + assertThat( + "The record as defined in the PersistenceExpectation was not found.", + foundRecord, + equalTo(true) + ); + } } /** @@ -617,102 +767,4 @@ private void assertThatPersistenceExpectationRecordWasFound(int numRecordsSearch } ) }; - - /** - * A helper class to verify that data got stored in a particular table. - */ - private class PersistenceExpectation { - public String tableName; - // each persistence expectation has an array of record expectations which all must reference a single row - // if looking for multiple records in the same table, - // create numerous PersistenceExpectations with the same tableName - public RecordExpectation[] recordExpectations; - - - public PersistenceExpectation(String tableName, RecordExpectation[] recordExpectations) { - this.tableName = tableName; - this.recordExpectations = recordExpectations; - } - } - - private enum ExpectedFieldType { - INT, - DOUBLE, STRING - } - - /** - * A helper class to verify that data got stored in a particular record. - */ - private class RecordExpectation { - public double acceptedDelta; - public double doubleExpectation; - public String editorExpectation; - public ExpectedFieldType expectedFieldType; - public String fieldName; - public int intExpectation; - public String stringExpectation; - public boolean stringExpectationInCSV = false; - public boolean editorStringExpectation = false; - - public RecordExpectation(String fieldName, int intExpectation) { - this.fieldName = fieldName; - this.expectedFieldType = ExpectedFieldType.INT; - this.intExpectation = intExpectation; - } - - /** - * This extra constructor is a bit hacky in that it is only used for certain records that have - * an int type when stored in the database, but a string type when exported to GTFS - */ - public RecordExpectation(String fieldName, int intExpectation, String stringExpectation) { - this.fieldName = fieldName; - this.expectedFieldType = ExpectedFieldType.INT; - this.intExpectation = intExpectation; - this.stringExpectation = stringExpectation; - this.stringExpectationInCSV = true; - } - - /** - * This extra constructor is a also hacky in that it is only used for records that have - * an int type when stored in the database, and different values in the CSV export depending on - * whether or not it is a snapshot from the editor. Currently this only applies to stop_times.stop_sequence - */ - public RecordExpectation(String fieldName, int intExpectation, String stringExpectation, String editorExpectation) { - this.fieldName = fieldName; - this.expectedFieldType = ExpectedFieldType.INT; - this.intExpectation = intExpectation; - this.stringExpectation = stringExpectation; - this.stringExpectationInCSV = true; - this.editorStringExpectation = true; - this.editorExpectation = editorExpectation; - } - - public RecordExpectation(String fieldName, String stringExpectation) { - this.fieldName = fieldName; - this.expectedFieldType = ExpectedFieldType.STRING; - this.stringExpectation = stringExpectation; - } - - public RecordExpectation(String fieldName, double doubleExpectation, double acceptedDelta) { - this.fieldName = fieldName; - this.expectedFieldType = ExpectedFieldType.DOUBLE; - this.doubleExpectation = doubleExpectation; - this.acceptedDelta = acceptedDelta; - } - - public String getStringifiedExpectation(boolean fromEditor) { - if (fromEditor && editorStringExpectation) return editorExpectation; - if (stringExpectationInCSV) return stringExpectation; - switch (expectedFieldType) { - case DOUBLE: - return String.valueOf(doubleExpectation); - case INT: - return String.valueOf(intExpectation); - case STRING: - return stringExpectation; - default: - return null; - } - } - } } diff --git a/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java b/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java index 44a418b4a..c7fab1fa7 100644 --- a/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java +++ b/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java @@ -229,14 +229,16 @@ public void canCreateUpdateAndDeleteFares() throws IOException, SQLException, In )); } - private void assertThatSqlQueryYieldsZeroRows(String sql) throws SQLException { + void assertThatSqlQueryYieldsZeroRows(String sql) throws SQLException { assertThatSqlQueryYieldsRowCount(sql, 0); } private void assertThatSqlQueryYieldsRowCount(String sql, int expectedRowCount) throws SQLException { LOG.info(sql); - ResultSet resultSet = testDataSource.getConnection().prepareStatement(sql).executeQuery(); - assertThat(resultSet.getFetchSize(), equalTo(expectedRowCount)); + int recordCount = 0; + ResultSet rs = testDataSource.getConnection().prepareStatement(sql).executeQuery(); + while (rs.next()) recordCount++; + assertThat("Records matching query should equal expected count.", recordCount, equalTo(expectedRowCount)); } @Test diff --git a/src/test/java/com/conveyal/gtfs/storage/ExpectedFieldType.java b/src/test/java/com/conveyal/gtfs/storage/ExpectedFieldType.java new file mode 100644 index 000000000..1eb604f04 --- /dev/null +++ b/src/test/java/com/conveyal/gtfs/storage/ExpectedFieldType.java @@ -0,0 +1,6 @@ +package com.conveyal.gtfs.storage; + +public enum ExpectedFieldType { + INT, + DOUBLE, STRING +} diff --git a/src/test/java/com/conveyal/gtfs/storage/PersistenceExpectation.java b/src/test/java/com/conveyal/gtfs/storage/PersistenceExpectation.java new file mode 100644 index 000000000..1c1c1be7d --- /dev/null +++ b/src/test/java/com/conveyal/gtfs/storage/PersistenceExpectation.java @@ -0,0 +1,26 @@ +package com.conveyal.gtfs.storage; + +import com.conveyal.gtfs.GTFSTest; + +/** + * A helper class to verify that data got stored in a particular table. + */ +public class PersistenceExpectation { + public String tableName; + /** + * Each persistence expectation has an array of record expectations which all must reference a single row. + * If looking for multiple records in the same table, create numerous PersistenceExpectations with the same + * tableName. + */ + public RecordExpectation[] recordExpectations; + + + public PersistenceExpectation(String tableName, RecordExpectation[] recordExpectations) { + this.tableName = tableName; + this.recordExpectations = recordExpectations; + } + + public static PersistenceExpectation[] list (PersistenceExpectation... expectations) { + return expectations; + } +} diff --git a/src/test/java/com/conveyal/gtfs/storage/RecordExpectation.java b/src/test/java/com/conveyal/gtfs/storage/RecordExpectation.java new file mode 100644 index 000000000..b3394f27f --- /dev/null +++ b/src/test/java/com/conveyal/gtfs/storage/RecordExpectation.java @@ -0,0 +1,81 @@ +package com.conveyal.gtfs.storage; + +/** + * A helper class to verify that data got stored in a particular record. + */ +public class RecordExpectation { + public double acceptedDelta; + public double doubleExpectation; + public String editorExpectation; + public ExpectedFieldType expectedFieldType; + public String fieldName; + public int intExpectation; + public String stringExpectation; + public boolean stringExpectationInCSV = false; + public boolean editorStringExpectation = false; + + public RecordExpectation(String fieldName, int intExpectation) { + this.fieldName = fieldName; + this.expectedFieldType = ExpectedFieldType.INT; + this.intExpectation = intExpectation; + } + + public static RecordExpectation[] list(RecordExpectation... expectations) { + return expectations; + } + + /** + * This extra constructor is a bit hacky in that it is only used for certain records that have + * an int type when stored in the database, but a string type when exported to GTFS + */ + public RecordExpectation(String fieldName, int intExpectation, String stringExpectation) { + this.fieldName = fieldName; + this.expectedFieldType = ExpectedFieldType.INT; + this.intExpectation = intExpectation; + this.stringExpectation = stringExpectation; + this.stringExpectationInCSV = true; + } + + /** + * This extra constructor is a also hacky in that it is only used for records that have + * an int type when stored in the database, and different values in the CSV export depending on + * whether or not it is a snapshot from the editor. Currently this only applies to stop_times.stop_sequence + */ + public RecordExpectation(String fieldName, int intExpectation, String stringExpectation, String editorExpectation) { + this.fieldName = fieldName; + this.expectedFieldType = ExpectedFieldType.INT; + this.intExpectation = intExpectation; + this.stringExpectation = stringExpectation; + this.stringExpectationInCSV = true; + this.editorStringExpectation = true; + this.editorExpectation = editorExpectation; + } + + public RecordExpectation(String fieldName, String stringExpectation) { + this.fieldName = fieldName; + this.expectedFieldType = ExpectedFieldType.STRING; + this.stringExpectation = stringExpectation; + } + + public RecordExpectation(String fieldName, double doubleExpectation, double acceptedDelta) { + this.fieldName = fieldName; + this.expectedFieldType = ExpectedFieldType.DOUBLE; + this.doubleExpectation = doubleExpectation; + this.acceptedDelta = acceptedDelta; + } + + public String getStringifiedExpectation(boolean fromEditor) { + if (fromEditor && editorStringExpectation) return editorExpectation; + if (stringExpectationInCSV) return stringExpectation; + switch (expectedFieldType) { + case DOUBLE: + return String.valueOf(doubleExpectation); + case INT: + return String.valueOf(intExpectation); + case STRING: + return stringExpectation; + default: + return null; + } + } +} diff --git a/src/test/resources/fake-agency-bad-calendar-date/agency.txt b/src/test/resources/fake-agency-bad-calendar-date/agency.txt new file mode 100755 index 000000000..a916ce91b --- /dev/null +++ b/src/test/resources/fake-agency-bad-calendar-date/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url +1,Fake Transit,,,,,America/Los_Angeles,, diff --git a/src/test/resources/fake-agency-bad-calendar-date/calendar.txt b/src/test/resources/fake-agency-bad-calendar-date/calendar.txt new file mode 100755 index 000000000..03b2867f7 --- /dev/null +++ b/src/test/resources/fake-agency-bad-calendar-date/calendar.txt @@ -0,0 +1,2 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +04100312-8fe1-46a5-a9f2-556f39478f57,1,1,1,1,1,1,1,bad_date!,20170917 diff --git a/src/test/resources/fake-agency-bad-calendar-date/calendar_dates.txt b/src/test/resources/fake-agency-bad-calendar-date/calendar_dates.txt new file mode 100755 index 000000000..546ad12b5 --- /dev/null +++ b/src/test/resources/fake-agency-bad-calendar-date/calendar_dates.txt @@ -0,0 +1,3 @@ +service_id,date,exception_type +04100312-8fe1-46a5-a9f2-556f39478f57,bad_date,2 +04100312-8fe1-46a5-a9f2-556f39478f57,bad_date2,1 \ No newline at end of file diff --git a/src/test/resources/fake-agency-bad-calendar-date/fare_attributes.txt b/src/test/resources/fake-agency-bad-calendar-date/fare_attributes.txt new file mode 100755 index 000000000..3173d016d --- /dev/null +++ b/src/test/resources/fake-agency-bad-calendar-date/fare_attributes.txt @@ -0,0 +1,2 @@ +fare_id,price,currency_type,payment_method,transfers,transfer_duration +route_based_fare,1.23,USD,0,0,0 \ No newline at end of file diff --git a/src/test/resources/fake-agency-bad-calendar-date/fare_rules.txt b/src/test/resources/fake-agency-bad-calendar-date/fare_rules.txt new file mode 100755 index 000000000..05d2aadf8 --- /dev/null +++ b/src/test/resources/fake-agency-bad-calendar-date/fare_rules.txt @@ -0,0 +1,2 @@ +fare_id,route_id,origin_id,destination_id,contains_id +route_based_fare,1,,, \ No newline at end of file diff --git a/src/test/resources/fake-agency-bad-calendar-date/feed_info.txt b/src/test/resources/fake-agency-bad-calendar-date/feed_info.txt new file mode 100644 index 000000000..b617faf49 --- /dev/null +++ b/src/test/resources/fake-agency-bad-calendar-date/feed_info.txt @@ -0,0 +1,2 @@ +feed_publisher_name,feed_publisher_url,feed_lang,feed_version +Conveyal,http://www.conveyal.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/fake-agency-bad-calendar-date/frequencies.txt b/src/test/resources/fake-agency-bad-calendar-date/frequencies.txt new file mode 100755 index 000000000..9baceff3a --- /dev/null +++ b/src/test/resources/fake-agency-bad-calendar-date/frequencies.txt @@ -0,0 +1,2 @@ +trip_id,start_time,end_time,headway_secs,exact_times +frequency-trip,08:00:00,09:00:00,1800,0 \ No newline at end of file diff --git a/src/test/resources/fake-agency-bad-calendar-date/routes.txt b/src/test/resources/fake-agency-bad-calendar-date/routes.txt new file mode 100755 index 000000000..35ea7aa67 --- /dev/null +++ b/src/test/resources/fake-agency-bad-calendar-date/routes.txt @@ -0,0 +1,2 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +1,1,1,Route 1,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/fake-agency-bad-calendar-date/shapes.txt b/src/test/resources/fake-agency-bad-calendar-date/shapes.txt new file mode 100755 index 000000000..3f2e3fd13 --- /dev/null +++ b/src/test/resources/fake-agency-bad-calendar-date/shapes.txt @@ -0,0 +1,8 @@ +shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence,shape_dist_traveled +5820f377-f947-4728-ac29-ac0102cbc34e,37.0612132,-122.0074332,1,0.0000000 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0611720,-122.0075000,2,7.4997067 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0613590,-122.0076830,3,33.8739075 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0608780,-122.0082780,4,109.0402932 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0603590,-122.0088280,5,184.6078298 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0597610,-122.0093540,6,265.8053023 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0590660,-122.0099190,7,357.8617018 diff --git a/src/test/resources/fake-agency-bad-calendar-date/stop_times.txt b/src/test/resources/fake-agency-bad-calendar-date/stop_times.txt new file mode 100755 index 000000000..59a2623b9 --- /dev/null +++ b/src/test/resources/fake-agency-bad-calendar-date/stop_times.txt @@ -0,0 +1,7 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +2,09:00:00,09:00:00,4u6g,1,,0,0,0.0000000, +2,09:01:00,09:01:00,johv,2,,0,0,341.4491961, +frequency-trip,08:00:00,08:00:00,4u6g,1,,0,0,0.0000000, +frequency-trip,08:01:00,08:01:00,johv,2,,0,0,341.4491961, diff --git a/src/test/resources/fake-agency-bad-calendar-date/stops.txt b/src/test/resources/fake-agency-bad-calendar-date/stops.txt new file mode 100755 index 000000000..8e71f7b2e --- /dev/null +++ b/src/test/resources/fake-agency-bad-calendar-date/stops.txt @@ -0,0 +1,3 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +4u6g,,Butler Ln,,37.0612132,-122.0074332,,,0,,, +johv,,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, diff --git a/src/test/resources/fake-agency-bad-calendar-date/transfers.txt b/src/test/resources/fake-agency-bad-calendar-date/transfers.txt new file mode 100755 index 000000000..357103c47 --- /dev/null +++ b/src/test/resources/fake-agency-bad-calendar-date/transfers.txt @@ -0,0 +1 @@ +from_stop_id,to_stop_id,transfer_type,min_transfer_time diff --git a/src/test/resources/fake-agency-bad-calendar-date/trips.txt b/src/test/resources/fake-agency-bad-calendar-date/trips.txt new file mode 100755 index 000000000..f0ecc68ae --- /dev/null +++ b/src/test/resources/fake-agency-bad-calendar-date/trips.txt @@ -0,0 +1,4 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +1,a30277f8-e50a-4a85-9141-b1e0da9d429d,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,04100312-8fe1-46a5-a9f2-556f39478f57 +1,2,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,a +1,frequency-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,04100312-8fe1-46a5-a9f2-556f39478f57 \ No newline at end of file