delete failed record, refresh, fold fail reason

pyerm/database/dbbase.py

@@ -20,7 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-# Version: 0.2.5
+# Version: 0.2.6
 
 import sqlite3
 import re
@@ -96,7 +96,7 @@ def __init__(self, db:Database, table_name:str, columns:dict=None) -> None:
                 for column, definition in list(columns.items()):
                     if 'VIRTUAL' in definition:
                         del columns[column.replace(' ', '_')]
-            assert columns is None or list(columns.keys()) == \
+            assert columns is None or list([col for col in columns.keys() if not str(col).startswith("image_")]) == \
                 list([column[1] for column in self.db.cursor.execute(f'PRAGMA table_info({table_name})').fetchall() if not str(column[1]).startswith("image_")]), \
                     f'Columns(except images) do not match for table {table_name}, consider to check or change table name'
             if self.db.info:

pyerm/database/tables.py

@@ -20,7 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-# Version: 0.2.4
+# Version: 0.2.6
 
 from PIL import Image
 from io import BytesIO
@@ -140,14 +140,15 @@ def insert(self, **kwargs):
         else:
             return id_list[0][0] 
 
-# def image_def(i):
-#     return {f'image_{i}_name': 'TEXT DEFAULT NULL', f'image_{i}': 'BLOB DEFAULT NULL'}
+def image_def(i):
+    return {f'image_{i}_name': 'TEXT DEFAULT NULL', f'image_{i}': 'BLOB DEFAULT NULL'}
 
 class ResultTable(Table):
-    def __init__(self, db: Database, task: str, rst_def_dict: dict=None) -> None:
+    def __init__(self, db: Database, task: str, rst_def_dict: dict=None, default_image_num: int=2) -> None:
         columns = {
             'experiment_id': 'INTEGER PRIMARY KEY AUTOINCREMENT',
             **rst_def_dict,
+            **{k:v for i in range(default_image_num) for k,v in image_def(i).items()},
         }
         table_name = f"result_{task}"
         super().__init__(db, table_name, columns)
@@ -176,7 +177,9 @@ def record_image(self, experiment_id:int, **image_dict:typing.Dict[str, typing.U
                 image = open(image_dict[image_key], 'rb').read()
             # print(type(image_key))
             # print(type(image))
-            self.update(f"experiment_id={experiment_id}", **{f'image_{i}_name': image_key, f'image_{i}': image})
+            self.update(f"experiment_id={experiment_id}", **{f'image_{i}_name': image_key})
+            self.update(f"experiment_id={experiment_id}", **{f'image_{i}': image})
+
 
 
 class DetailTable(Table):

pyerm/database/utils.py

@@ -0,0 +1,35 @@
+# MIT License
+
+# Copyright (c) 2024 Yuxuan Shao
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# Version: 0.2.6
+
+from .dbbase import Database
+
+def delete_failed_experiments(db:Database):
+    experiment_table = db['experiment_list']
+    failed_experiments = experiment_table.select('id', 'task', where="status='failed'")
+    for experiment in failed_experiments:
+        experiment_id = experiment[0]
+        task = experiment[1]
+        task_table = db[f"result_{task}"]
+        task_table.delete(f"experiment_id={experiment_id}")
+        experiment_table.delete(f"id={experiment_id}")

pyerm/scripts/export_data.py

@@ -87,7 +87,7 @@ def zip_dir(dir_path:str, zip_path:str, remove_original=False):
                 file_path = os.path.join(root, file)
                 zipf.write(file_path, os.path.relpath(file_path, dir_path))
 
-                print(file_path)
+                # print(file_path)
     if remove_original:
         shutil.rmtree(dir_path)
 

pyerm/webUI/home.py

@@ -20,7 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-# Version: 0.2.4
+# Version: 0.2.6
 
 import streamlit as st
 import os
@@ -56,7 +56,8 @@ def title():
     st.markdown(f"**Disclaimer**: This is a demo version. The actual version is not available yet.")
 
 def load_db():
-    st.markdown('## Load Database (PyERM only supports local SQLite database for now)')
+    st.markdown('## Load Database')
+    st.markdown("### **(PyERM only supports local SQLite database for now)**")
     db_path = st.text_input("Database Path", value=st.session_state.db_path)
     if st.button('Change Database Path'):
         st.session_state.db_path = db_path

pyerm/webUI/tables.py

@@ -20,7 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-# Version: 0.2.4
+# Version: 0.2.6
 
 import pandas as pd
 from PIL import Image
@@ -29,7 +29,9 @@
 import streamlit as st
 import os
 import re
+import tempfile
 
+from pyerm.database.utils import delete_failed_experiments
 from pyerm.database.dbbase import Database
 
 def tables():
@@ -59,6 +61,12 @@ def make_image_clickable(image_name, image):
         img_str = image_to_base64(image)
         return f'<a href="data:image/jpeg;base64,{img_str}" target="_blank" title="{image_name}"><img src="data:image/jpeg;base64,{img_str}" width="100"></a>'
 
+    def fold_detail_row(row, col_name):
+        if row[col_name]:
+            return f'<details><summary>Details</summary>{row[col_name]}</details>'
+        else:
+            return 'None'
+
     db = Database(st.session_state.db_path, output_info=False)
     table_name = st.session_state.table_name
     if st.session_state.sql is not None:
@@ -74,6 +82,8 @@ def make_image_clickable(image_name, image):
         data = db[table_name].select()
         columns = [column[0] for column in db.cursor.description]
         df = pd.DataFrame(data, columns=columns)
+
+    # special process for image columns
     columns_keep = [col for col in df.columns if not col.startswith("image_")]
     pattern = re.compile(r'image_(\d+)')
     max_image_num = -1
@@ -85,8 +95,20 @@ def make_image_clickable(image_name, image):
         if f'image_{i}' in df.columns and not df[f'image_{i}_name'].isnull().all():
             df[f'image_{i}'] = df.apply(lambda x: make_image_clickable(x[f'image_{i}_name'], x[f'image_{i}']), axis=1)
             columns_keep.append(f'image_{i}')
+
+    if "failed_reason" in df.columns:
+        df['failed_reason'] = df.apply(lambda x: fold_detail_row(x, 'failed_reason'), axis=1)
     df = df[columns_keep]
+
+    if st.button('Refresh'):
+        st.session_state.table_name = table_name
     st.write('## Table:', table_name)
+    if table_name == 'experiment_list':
+        if st.checkbox('Delete all failed records'):
+            st.write('**Warning: This operation will delete all failed records and their results, which cannot be undone.**')
+            if st.button(f"Confirm"):
+                delete_failed_experiments(db)
+                st.session_state.table_name = table_name
     st.write(df.to_html(escape=False, columns=columns_keep), unsafe_allow_html=True)
 
     # st.dataframe(df[columns_keep])

setup.py

@@ -20,15 +20,15 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-# Version: 0.2.5
+# Version: 0.2.6
 from setuptools import setup, find_packages
 
 with open("README.md", "r", encoding="utf-8") as f:
     long_description = f.read()
 
 setup(
     name='pyerm',
-    version='0.2.5',
+    version='0.2.6',
     author='Yuxuan Shao',
     author_email='[email protected]',
     description='This project is an local experiment record manager for python based on SQLite DMS, suitable for recording experiment and analysing experiment data with a web UI, which can help you efficiently save your experiment settings and results for later analysis.',