catima importer: renumber IDs to merge properly

app/src/main/java/protect/card_locker/DBHelper.java

@@ -364,10 +364,10 @@ private static void updateFTS(final SQLiteDatabase db, final int id, final Strin
  whereAttrs(LoyaltyCardDbFTS.ID), withArgs(id));
  }
 
- public static long getMaxLoyaltyCardId(final SQLiteDatabase database) {
+ public static int getMaxLoyaltyCardId(final SQLiteDatabase database) {
  Cursor data = database.rawQuery("SELECT IFNULL(MAX(" + LoyaltyCardDbIds.ID + "), 0) FROM " + LoyaltyCardDbIds.TABLE, null, null);
  data.moveToFirst();
- long maxId = data.getLong(0);
+ int maxId = data.getInt(0);
  data.close();
  return maxId;
  }

app/src/main/java/protect/card_locker/Utils.java

@@ -58,6 +58,8 @@
 import java.util.GregorianCalendar;
 import java.util.Locale;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import protect.card_locker.preferences.Settings;
 
@@ -380,6 +382,23 @@ static public String getCardImageFileName(int loyaltyCardId, ImageLocationType t
  return cardImageFileNameBuilder.toString();
  }
 
+ static public String getRenamedCardImageFileName(final String fileName, final long idOffset) {
+ Pattern pattern = Pattern.compile("^(card_)(\\d+)(_(?:front|back|icon)\\.png)$");
+ Matcher matcher = pattern.matcher(fileName);
+ if (matcher.matches()) {
+ StringBuilder cardImageFileNameBuilder = new StringBuilder();
+ cardImageFileNameBuilder.append(matcher.group(1));
+ try {
+ cardImageFileNameBuilder.append(Integer.parseInt(matcher.group(2)) + idOffset);
+ } catch (NumberFormatException _e) {
+ return null;
+ }
+ cardImageFileNameBuilder.append(matcher.group(3));
+ return cardImageFileNameBuilder.toString();
+ }
+ return null;
+ }
+
  static public void saveCardImage(Context context, Bitmap bitmap, String fileName) throws FileNotFoundException {
  if (bitmap == null) {
  context.deleteFile(fileName);

app/src/main/java/protect/card_locker/importexport/CatimaImporter.java

@@ -40,7 +40,7 @@
  * A header is expected for the each table showing the names of the columns.
  */
 public class CatimaImporter implements Importer {
- public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set<String> newImageFiles, long maxLoyaltyCardId) throws IOException, FormatException, InterruptedException {
+ public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set<String> newImageFiles, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException {
  InputStream bufferedInputStream = new BufferedInputStream(input);
  bufferedInputStream.mark(100);
 
@@ -55,10 +55,14 @@ public void importData(Context context, SQLiteDatabase database, InputStream inp
 
  String fileName = Uri.parse(localFileHeader.getFileName()).getLastPathSegment();
  if (fileName.equals("catima.csv")) {
- importCSV(context, database, zipInputStream);
+ importCSV(context, database, zipInputStream, maxLoyaltyCardId);
  } else if (fileName.endsWith(".png")) {
- Utils.saveCardImage(context, ZipUtils.readImage(zipInputStream), fileName);
- newImageFiles.add(fileName);
+ String newFileName = Utils.getRenamedCardImageFileName(fileName, maxLoyaltyCardId);
+ if (newFileName == null) {
+ throw new FormatException("Unexpected PNG file in import: " + fileName);
+ }
+ Utils.saveCardImage(context, ZipUtils.readImage(zipInputStream), newFileName);
+ newImageFiles.add(newFileName);
  } else {
  throw new FormatException("Unexpected file in import: " + fileName);
  }
@@ -67,34 +71,34 @@ public void importData(Context context, SQLiteDatabase database, InputStream inp
  if (!isZipFile) {
  // This is not a zip file, try importing as bare CSV
  bufferedInputStream.reset();
- importCSV(context, database, bufferedInputStream);
+ importCSV(context, database, bufferedInputStream, maxLoyaltyCardId);
  }
 
  input.close();
  }
 
- public void importCSV(Context context, SQLiteDatabase database, InputStream input) throws IOException, FormatException, InterruptedException {
+ public void importCSV(Context context, SQLiteDatabase database, InputStream input, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException {
  BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8));
 
  int version = parseVersion(bufferedReader);
  switch (version) {
  case 1:
- parseV1(database, bufferedReader);
+ parseV1(database, bufferedReader, maxLoyaltyCardId);
  break;
  case 2:
- parseV2(context, database, bufferedReader);
+ parseV2(context, database, bufferedReader, maxLoyaltyCardId);
  break;
  default:
  throw new FormatException(String.format("No code to parse version %s", version));
  }
  }
 
- public void parseV1(SQLiteDatabase database, BufferedReader input) throws IOException, FormatException, InterruptedException {
+ public void parseV1(SQLiteDatabase database, BufferedReader input, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException {
  final CSVParser parser = new CSVParser(input, CSVFormat.RFC4180.builder().setHeader().build());
 
  try {
  for (CSVRecord record : parser) {
- importLoyaltyCard(database, record);
+ importLoyaltyCard(database, record, maxLoyaltyCardId);
 
  if (Thread.currentThread().isInterrupted()) {
  throw new InterruptedException();
@@ -107,7 +111,7 @@ public void parseV1(SQLiteDatabase database, BufferedReader input) throws IOExce
  }
  }
 
- public void parseV2(Context context, SQLiteDatabase database, BufferedReader input) throws IOException, FormatException, InterruptedException {
+ public void parseV2(Context context, SQLiteDatabase database, BufferedReader input, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException {
  int part = 0;
  StringBuilder stringPart = new StringBuilder();
 
@@ -133,15 +137,15 @@ public void parseV2(Context context, SQLiteDatabase database, BufferedReader inp
  break;
  case 2:
  try {
- parseV2Cards(context, database, stringPart.toString());
+ parseV2Cards(context, database, stringPart.toString(), maxLoyaltyCardId);
  sectionParsed = true;
  } catch (FormatException e) {
  // We may have a multiline field, try again
  }
  break;
  case 3:
  try {
- parseV2CardGroups(database, stringPart.toString());
+ parseV2CardGroups(database, stringPart.toString(), maxLoyaltyCardId);
  sectionParsed = true;
  } catch (FormatException e) {
  // We may have a multiline field, try again
@@ -195,7 +199,7 @@ public void parseV2Groups(SQLiteDatabase database, String data) throws IOExcepti
  }
  }
 
- public void parseV2Cards(Context context, SQLiteDatabase database, String data) throws IOException, FormatException, InterruptedException {
+ public void parseV2Cards(Context context, SQLiteDatabase database, String data, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException {
  // Parse cards
  final CSVParser cardParser = new CSVParser(new StringReader(data), CSVFormat.RFC4180.builder().setHeader().build());
 
@@ -216,11 +220,11 @@ public void parseV2Cards(Context context, SQLiteDatabase database, String data)
  }
 
  for (CSVRecord record : records) {
- importLoyaltyCard(database, record);
+ importLoyaltyCard(database, record, maxLoyaltyCardId);
  }
  }
 
- public void parseV2CardGroups(SQLiteDatabase database, String data) throws IOException, FormatException, InterruptedException {
+ public void parseV2CardGroups(SQLiteDatabase database, String data, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException {
  // Parse card group mappings
  final CSVParser cardGroupParser = new CSVParser(new StringReader(data), CSVFormat.RFC4180.builder().setHeader().build());
 
@@ -241,7 +245,7 @@ public void parseV2CardGroups(SQLiteDatabase database, String data) throws IOExc
  }
 
  for (CSVRecord record : records) {
- importCardGroupMapping(database, record);
+ importCardGroupMapping(database, record, maxLoyaltyCardId);
  }
  }
 
@@ -278,9 +282,12 @@ private int parseVersion(BufferedReader reader) throws IOException {
  * Import a single loyalty card into the database using the given
  * session.
  */
- private void importLoyaltyCard(SQLiteDatabase database, CSVRecord record)
+ private void importLoyaltyCard(SQLiteDatabase database, CSVRecord record, int maxLoyaltyCardId)
  throws FormatException {
  int id = CSVHelpers.extractInt(DBHelper.LoyaltyCardDbIds.ID, record);
+ if (id < 1) {
+ throw new FormatException("ID must be >= 1");
+ }
 
  String store = CSVHelpers.extractString(DBHelper.LoyaltyCardDbIds.STORE, record, "");
  if (store.isEmpty()) {
@@ -376,7 +383,8 @@ private void importLoyaltyCard(SQLiteDatabase database, CSVRecord record)
  // We catch this exception so we can still import old backups
  }
 
- DBHelper.insertLoyaltyCard(database, id, store, note, validFrom, expiry, balance, balanceType, cardId, barcodeId, barcodeType, headerColor, starStatus, lastUsed, archiveStatus);
+ int newId = id + maxLoyaltyCardId;
+ DBHelper.insertLoyaltyCard(database, newId, store, note, validFrom, expiry, balance, balanceType, cardId, barcodeId, barcodeType, headerColor, starStatus, lastUsed, archiveStatus);
  }
 
  /**
@@ -397,16 +405,20 @@ private void importGroup(SQLiteDatabase database, CSVRecord record) throws Forma
  * Import a single card to group mapping into the database using the given
  * session.
  */
- private void importCardGroupMapping(SQLiteDatabase database, CSVRecord record) throws FormatException {
+ private void importCardGroupMapping(SQLiteDatabase database, CSVRecord record, int maxLoyaltyCardId) throws FormatException {
  int cardId = CSVHelpers.extractInt(DBHelper.LoyaltyCardDbIdsGroups.cardID, record);
  String groupId = CSVHelpers.extractString(DBHelper.LoyaltyCardDbIdsGroups.groupID, record, null);
 
+ if (cardId < 1) {
+ throw new FormatException("Card ID must be >= 1");
+ }
  if (groupId == null) {
  throw new FormatException("Group has no ID: " + record);
  }
 
- List<Group> cardGroups = DBHelper.getLoyaltyCardGroups(database, cardId);
+ int newCardId = cardId + maxLoyaltyCardId;
+ List<Group> cardGroups = DBHelper.getLoyaltyCardGroups(database, newCardId);
  cardGroups.add(DBHelper.getGroup(database, groupId));
- DBHelper.setLoyaltyCardGroups(database, cardId, cardGroups);
+ DBHelper.setLoyaltyCardGroups(database, newCardId, cardGroups);
  }
 }

app/src/main/java/protect/card_locker/importexport/FidmeImporter.java

@@ -32,7 +32,7 @@
  * A header is expected for the each table showing the names of the columns.
  */
 public class FidmeImporter implements Importer {
- public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set<String> newImageFiles, long maxLoyaltyCardId) throws IOException, FormatException, JSONException, ParseException {
+ public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set<String> newImageFiles, int maxLoyaltyCardId) throws IOException, FormatException, JSONException, ParseException {
  // We actually retrieve a .zip file
  ZipInputStream zipInputStream = new ZipInputStream(input, password);
 

app/src/main/java/protect/card_locker/importexport/Importer.java

@@ -24,5 +24,5 @@ public interface Importer {
  * @throws IOException
  * @throws FormatException
  */
- void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set<String> newImageFiles, long maxLoyaltyCardId) throws IOException, FormatException, InterruptedException, JSONException, ParseException;
+ void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set<String> newImageFiles, int maxLoyaltyCardId) throws IOException, FormatException, InterruptedException, JSONException, ParseException;
 }

app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java

@@ -49,7 +49,7 @@ public static ImportExportResult importData(Context context, SQLiteDatabase data
  if (importer != null) {
  database.beginTransaction();
  try {
- long maxLoyaltyCardId = DBHelper.getMaxLoyaltyCardId(database);
+ int maxLoyaltyCardId = DBHelper.getMaxLoyaltyCardId(database);
  Log.d(TAG, "Current max loyalty card id: " + maxLoyaltyCardId);
  importer.importData(context, database, input, password, newImageFiles, maxLoyaltyCardId);
  database.setTransactionSuccessful();
@@ -75,6 +75,13 @@ public static ImportExportResult importData(Context context, SQLiteDatabase data
  Log.e(TAG, error);
  }
 
+ // FIXME
+ if (format == DataFormat.Catima) {
+ for (String fileName : newImageFiles) {
+ context.deleteFile(fileName);
+ }
+ }
+
  return new ImportExportResult(ImportExportResultType.GenericFailure, error);
  }
 }

app/src/main/java/protect/card_locker/importexport/StocardImporter.java

@@ -43,7 +43,7 @@
 public class StocardImporter implements Importer {
  private static final String TAG = "Catima";
 
- public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set<String> newImageFiles, long maxLoyaltyCardId) throws IOException, FormatException, JSONException, ParseException {
+ public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set<String> newImageFiles, int maxLoyaltyCardId) throws IOException, FormatException, JSONException, ParseException {
  HashMap<String, HashMap<String, Object>> loyaltyCardHashMap = new HashMap<>();
  HashMap<String, HashMap<String, Object>> providers = new HashMap<>();
 

app/src/main/java/protect/card_locker/importexport/VoucherVaultImporter.java

@@ -37,7 +37,7 @@
  * A header is expected for the each table showing the names of the columns.
  */
 public class VoucherVaultImporter implements Importer {
- public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set<String> newImageFiles, long maxLoyaltyCardId) throws IOException, FormatException, JSONException, ParseException {
+ public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password, Set<String> newImageFiles, int maxLoyaltyCardId) throws IOException, FormatException, JSONException, ParseException {
  BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8));
 
  StringBuilder sb = new StringBuilder();

app/src/test/java/protect/card_locker/ImportExportTest.java

@@ -201,7 +201,12 @@ private void addGroups(int groupsToAdd) {
  * where the smallest card's index is 1
  */
  private void checkLoyaltyCards() {
+ checkLoyaltyCards(false);
+ }
+
+ private void checkLoyaltyCards(boolean dups) {
  Cursor cursor = DBHelper.getLoyaltyCardCursor(mDatabase);
+ boolean first = true;
  int index = 1;
 
  while (cursor.moveToNext()) {
@@ -222,7 +227,16 @@ private void checkLoyaltyCards() {
  assertEquals(Integer.valueOf(index), card.headerColor);
  assertEquals(0, card.starStatus);
 
- index++;
+ if (dups) {
+ if (first) {
+ first = false;
+ } else {
+ first = true;
+ index++;
+ }
+ } else {
+ index++;
+ }
  }
  cursor.close();
  }
@@ -500,9 +514,9 @@ public void importExistingCardsNotReplace() throws IOException {
  result = MultiFormatImporter.importData(activity.getApplicationContext(), mDatabase, inData, DataFormat.Catima, null);
  assertEquals(ImportExportResultType.Success, result.resultType());
 
- assertEquals(NUM_CARDS, DBHelper.getLoyaltyCardCount(mDatabase));
+ assertEquals(NUM_CARDS * 2, DBHelper.getLoyaltyCardCount(mDatabase));
 
- checkLoyaltyCards();
+ checkLoyaltyCards(true);
 
  // Clear the database for the next format under test
  TestHelpers.getEmptyDb(activity);
@@ -764,7 +778,7 @@ public void importWithInvalidStarFieldV1() {
  // Import the CSV data
  result = MultiFormatImporter.importData(activity.getApplicationContext(), mDatabase, inputStream, DataFormat.Catima, null);
  assertEquals(ImportExportResultType.Success, result.resultType());
- assertEquals(1, DBHelper.getLoyaltyCardCount(mDatabase));
+ assertEquals(2, DBHelper.getLoyaltyCardCount(mDatabase));
 
  LoyaltyCard card = DBHelper.getLoyaltyCard(mDatabase, 1);