uniqueness test for param table, database merge scripts, README, some…

… small fix

README.md

@@ -33,6 +33,20 @@ The experiment recorder mainly consists of four parts, `experiment_start()`, `ex
 
 `detail_update()` saves the intermediate results. It's optional, and if you never use it and don't manually set the define dict, the detail table may not be created.
 
+
+## Scripts Introduction
+### export_xls 
+Export the content of a SQLite database to an Excel file
+```shell
+export_xls db_path(default ~/experiment.db) output_path(default ./experiment_record.xls)
+```
+### db_merge 
+Merge two SQLite databases.
+```shell
+db_merge db_path_destination db_path_source
+```
+
+
 ## Table Introduction
 
 ### Experiment Table

pyerm/dbbase.py

@@ -20,7 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-# Version: 0.1.3
+# Version: 0.1.6
 
 import sqlite3
 
@@ -72,7 +72,6 @@ def __init__(self, db:Database, table_name:str, columns:dict=None) -> None:
         assert table_name in db.table_names, f'Table {table_name} does not exist'
         self.db = db
         self.table_name = table_name
-        self._column = None
         if not table_exists(db, table_name):
             assert columns, 'Columns must be provided when creating a new table'
             columns_str = ', '.join([f'{key} {value}' for key, value in columns.items()])
@@ -83,7 +82,7 @@ def __init__(self, db:Database, table_name:str, columns:dict=None) -> None:
         else:
             assert columns.keys() == set([column[1] for column in self.db.cursor.execute(f'PRAGMA table_info({table_name})').fetchall()]), f'Columns do not match for table {table_name}, consider to check or change table name'
             print(f'Table {table_name} already exists')
-        self.primary_key = [column[5] for column in self.db.cursor.execute(f'PRAGMA table_info({table_name})').fetchall() if column[5] == 1]
+        self.primary_key = [column[1] for column in self.db.cursor.execute(f'PRAGMA table_info({table_name})').fetchall() if column[5] == 1]
 
     def insert(self, **kwargs) -> int:
         assert len(kwargs) == len(self.columns) - 1 or len(kwargs) == len(self.columns), 'Parameter number does not match'

pyerm/scripts/db_merge.py

@@ -0,0 +1,63 @@
+# MIT License
+
+# Copyright (c) 2024 Yuxuan Shao
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# Version: 0.1.6
+import sqlite3
+import argparse
+import os
+
+from dbbase import Table, Database
+
+def copy_table(db1:Database, db2:Database, table_name:str):
+    table = Table(db2, table_name)
+    columns = [info[1] for info in db2.cursor.execute(f"PRAGMA table_info({table_name})").fetchall() if info[1] not in table.primary_key]
+    columns_list_str = ', '.join(columns)  
+    placeholders = ', '.join(['?'] * len(columns))  
+    insert_stmt = f"INSERT INTO {table_name} ({columns_list_str}) VALUES ({placeholders})"
+    rows = db2.cursor.execute(f"SELECT {columns_list_str} FROM {table_name}").fetchall()
+    for row in rows:
+        try:
+            db1.cursor.execute(insert_stmt, row)
+        except sqlite3.IntegrityError:
+            pass
+    db1.conn.commit()
+
+def merge_db(db_path1:str, db_path2:str):
+    db1 = Database(db_path1)
+    db2 = Database(db_path2)
+    for table_name in db2.table_names:
+        copy_table(db1, db2, table_name)
+
+def main():
+    parser = argparse.ArgumentParser(description='Merge two SQLite databases.')
+    parser.add_argument('db_path_destination', type=str, help='Destination database file path.')
+    parser.add_argument('db_path_source', '', type=str, help='Source database file path.')
+    if not os.path.exists(args.db_path_destination):
+        raise FileNotFoundError(f"The database file {args.db_path_destination} does not exist")
+    if not os.path.exists(args.db_path_source):
+        raise FileNotFoundError(f"The database file {args.db_path_source} does not exist")
+    args = parser.parse_args()
+    merge_db(args.db_path_destination, args.db_path_source)
+
+
+if __name__ == "__main__":
+    main()

pyerm/scripts/export_xls.py

@@ -20,7 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-# Version: 0.1.4
+# Version: 0.1.6
 
 import pandas as pd
 import sqlite3
@@ -39,10 +39,14 @@ def export_xls(db_path:str, output_path:str):
     conn.close()
 
 def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('db_path', type=str, default= os.path.join(USER_HOME, 'experiment.db'), help='The path of the database file')
-    parser.add_argument('output_path', type=str, help='The path of the output excel file')
+    parser = argparse.ArgumentParser(description="Export the content of a SQLite database to an Excel file")
+    parser.add_argument('db_path', type=str, default=None, help='The path of the database file')
+    parser.add_argument('output_path', type=str, default="./experiment_record.xls", help='The path of the output excel file')
     args = parser.parse_args()
+    if args.db_path is None:
+        args.db_path = os.path.join(USER_HOME, 'experiment.db')
+    if not os.path.exists(args.db_path):
+        raise FileNotFoundError(f"The database file {args.db_path} does not exist")
     export_xls(args.db_path, args.output_path)
 
 if __name__ == "__main__":

pyerm/tables.py

@@ -20,7 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-# Version: 0.1.3
+# Version: 0.1.6
 
 from PIL import Image
 from io import BytesIO
@@ -81,10 +81,19 @@ def __init__(self, db: Database, data: str, param_def_dict: dict=None) -> None:
         else:
             assert param_def_dict, 'Data Parameter Dict must be provided when creating a new data table'
             columns = {
-                'data_id': 'INTEGER PRIMARY KEY',
+                'data_id': 'INTEGER PRIMARY KEY AUTOINCREMENT',
                 **param_def_dict,
             }
         super().__init__(db, table_name, columns)
+        self.db.cursor.execute(f"CREATE UNIQUE INDEX IF NOT EXISTS ON {table_name}({', '.join(param_def_dict.keys())}")
+
+    def insert(self, **kwargs):
+        id_list = self.db.cursor.execute(f"SELECT data_id FROM {self.table_name} WHERE {' AND '.join([f'{k}={v}' for k, v in kwargs.items()])}").fetchall()
+        if id_list == []:
+            return super().insert(**kwargs)
+        else:
+            return id_list[0][0] 
+
 
 class MethodTable(Table):
     def __init__(self, db: Database, method: str, param_def_dict: dict=None) -> None:
@@ -94,15 +103,23 @@ def __init__(self, db: Database, method: str, param_def_dict: dict=None) -> None
         else:
             assert param_def_dict, 'Method Parameter Dict must be provided when creating a new parameter table'
             columns = {
-                'method_id': 'INTEGER PRIMARY KEY',
+                'method_id': 'INTEGER PRIMARY KEY AUTOINCREMENT',
                 **param_def_dict,
             }
         super().__init__(db, table_name, columns)
+        self.db.cursor.execute(f"CREATE UNIQUE INDEX IF NOT EXISTS ON {table_name}({', '.join(param_def_dict.keys())}")
+
+    def insert(self, **kwargs):
+        id_list = self.db.cursor.execute(f"SELECT data_id FROM {self.table_name} WHERE {' AND '.join([f'{k}={v}' for k, v in kwargs.items()])}").fetchall()
+        if id_list == []:
+            return super().insert(**kwargs)
+        else:
+            return id_list[0][0] 
 
 class ResultTable(Table):
     def __init__(self, db: Database, task: str, rst_def_dict: dict=None, max_image_num: int=10) -> None:
         columns = {
-            'id': 'INTEGER PRIMARY KEY',
+            'id': 'INTEGER PRIMARY KEY AUTOINCREMENT',
             **{f'image_{i}': 'BLBO DEFAULT NULL' for i in range(max_image_num)},
             **rst_def_dict,
         }
@@ -127,7 +144,7 @@ def record_image(self, experiment_id:int, image_list:typing.List[typing.Union[Im
 class DetailTable(Table):
     def __init__(self, db: Database, method: str, detail_def_dict: dict=None) -> None:
         columns = {
-            'detail_id': 'INTEGER PRIMARY KEY',
+            'detail_id': 'INTEGER PRIMARY KEY AUTOINCREMENT',
             'experiment_id': 'INTEGER FOREIGN KEY REFERENCES experiment_list(id)',
             **detail_def_dict,
         }

setup.py

@@ -20,15 +20,15 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-# Version: 0.1.5
+# Version: 0.1.6
 from setuptools import setup, find_packages
 
 with open("README.md", "r", encoding="utf-8") as f:
     long_description = f.read()
 
 setup(
     name='pyerm',
-    version='0.1.5',
+    version='0.1.6',
     author='Yuxuan Shao',
     author_email='[email protected]',
     description='This project is an experiment record manager for python based on SQLite DMS, which can help you efficiently save your experiment settings and results for later analysis.',
@@ -45,6 +45,7 @@
     entry_points={
         'console_scripts': [
             'export_xls=pyerm.scripts.export_xls:main',
+            'db_merge=pyerm.scripts.db_merge:main',
         ],
     },
 )