Merge branch 'semantic-similarity' into main

BuilderIO · Dec 8, 2023 · b36324f · b36324f
2 parents 42d3daf + b63c9eb
commit b36324f
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 3 deletions.
diff --git a/src/conv_html_to_markdown.py b/src/conv_html_to_markdown.py
@@ -262,7 +262,6 @@ def main():
 
         chunks = list(chunk_dataset(original_data, chunk_size))
         formatted_contents = []
-
         logging.info("Processing and saving dataset in chunks.")
         with ThreadPoolExecutor(max_workers=max_threads) as executor:
             results = executor.map(process_chunk, chunks)
@@ -272,7 +271,6 @@ def main():
         output_file_name = "gpt-crawler-curated_markdown.md"
         save_output_in_chunks(output_file_name, formatted_contents)
         logging.info("Content formatted and saved in chunks successfully.")
-
         logging.info("\nConversion process successful. Exiting program.")
     except Exception as e:
         logging.error("An error occurred in the main function: %s", e)

diff --git a/tests/test_conv_html_to_markdown.py b/tests/test_conv_html_to_markdown.py
@@ -73,4 +73,4 @@ def test_process_chunk(self):
 
 
 if __name__ == "__main__":
-    unittest.main
+    unittest.main
Original file line number	Diff line number	Diff line change
Expand Up		@@ -73,4 +73,4 @@ def test_process_chunk(self):


		if __name__ == "__main__":
		unittest.main
		unittest.main