dev: 0.0.1

.github/workflows/tests.yaml

@@ -23,4 +23,4 @@ jobs:
       - name: Tests
         run: PYTHONPATH=. pytest --cov=data_flow --cov-report term
       - name: Lint
-        run: pflake8 data_flow/
+        run: pflake8 mysiar_data_flow/ tests/

.gitignore

@@ -162,5 +162,5 @@ cython_debug/
 
 
 # data_flow
-!data_flow/lib
+!mysiar_data_flow/lib
 tests/data/*.csv

Makefile

@@ -8,7 +8,20 @@ pip::
 	venv/bin/pip install -r requirements.dev.txt
 
 tests::
-	PYTHONPATH=. venv/bin/pytest --cov=data_flow --cov-report html --cov-report term -rP tests/ -vvv
+	PYTHONPATH=. venv/bin/pytest --cov=mysiar_data_flow --cov-report html --cov-report term -rP tests/ -vvv
 
 lint::
-	venv/bin/pflake8 data_flow/
+	venv/bin/pflake8 mysiar_data_flow/ tests/
+
+build::
+	rm -rf dist
+	venv/bin/poetry build
+
+
+upload-test::
+	$(MAKE) build
+	venv/bin/python -m twine upload -u $${PYPI_USER} -p $${PYPI_PASS_TEST} --verbose --repository testpypi dist/*
+
+upload::
+	$(MAKE) build
+	. venv/bin/activate && python -m twine upload -u $${PYPI_USER} -p $${PYPI_PASS} --verbose dist/*

README.md

@@ -8,5 +8,17 @@
 
 library to manipulate data
 
+## Installation instructions
+
+```sh
+pip install mysiar-data-flow
+```
+
 ## DataFlow.DataFrame
-work in progress
+
+### Usage
+For now check  [mysiar_data_flow/data_flow.py](mysiar_data_flow/data_flow.py) file for interface
+
+
+
+![work in progress](.github/5578703.png)

data_flow/data_flow.py → mysiar_data_flow/data_flow.py

@@ -7,31 +7,31 @@
 import polars as pl
 from pyarrow import feather
 
-from data_flow.lib import FileType, Operator
-from data_flow.lib.data_columns import (
+from mysiar_data_flow.lib import FileType, Operator
+from mysiar_data_flow.lib.data_columns import (
     data_get_columns,
     data_delete_columns,
     data_rename_columns,
     data_select_columns,
     data_filter_on_column,
 )
-from data_flow.lib.data_from import (
+from mysiar_data_flow.lib.data_from import (
     from_csv_2_file,
     from_feather_2_file,
     from_parquet_2_file,
     from_json_2_file,
     from_hdf_2_file,
 )
-from data_flow.lib.data_to import (
+from mysiar_data_flow.lib.data_to import (
     to_csv_from_file,
     to_feather_from_file,
     to_parquet_from_file,
     to_json_from_file,
     to_hdf_from_file,
 )
-from data_flow.lib.fireducks import from_fireducks_2_file, to_fireducks_from_file
-from data_flow.lib.pandas import from_pandas_2_file
-from data_flow.lib.tools import generate_temporary_filename, delete_file
+from mysiar_data_flow.lib.fireducks import from_fireducks_2_file, to_fireducks_from_file
+from mysiar_data_flow.lib.pandas import from_pandas_2_file
+from mysiar_data_flow.lib.tools import generate_temporary_filename, delete_file
 
 
 class DataFlow:

data_flow/lib/data_columns.py → mysiar_data_flow/lib/data_columns.py

@@ -2,8 +2,8 @@
 
 import fireducks.pandas as fd
 
-from data_flow.lib.FileType import FileType
-from data_flow.lib.Operator import Operator
+from mysiar_data_flow.lib.FileType import FileType
+from mysiar_data_flow.lib.Operator import Operator
 
 
 def data_get_columns(tmp_filename: str, file_type: FileType) -> list:

data_flow/lib/data_from.py → mysiar_data_flow/lib/data_from.py

@@ -1,7 +1,7 @@
 import fireducks.pandas as fd
 from pyarrow import feather
 
-from data_flow.lib.FileType import FileType
+from mysiar_data_flow.lib.FileType import FileType
 
 
 def from_csv_2_file(filename: str, tmp_filename: str, file_type: FileType) -> None:

data_flow/lib/data_to.py → mysiar_data_flow/lib/data_to.py

@@ -1,6 +1,6 @@
 import fireducks.pandas as fd
 
-from data_flow.lib.FileType import FileType
+from mysiar_data_flow.lib.FileType import FileType
 
 
 def to_csv_from_file(filename: str, tmp_filename: str, file_type: FileType) -> None:

data_flow/lib/fireducks.py → mysiar_data_flow/lib/fireducks.py

@@ -1,6 +1,6 @@
 import fireducks.pandas as fd
 
-from data_flow.lib.FileType import FileType
+from mysiar_data_flow.lib.FileType import FileType
 
 
 def from_fireducks_2_file(df: fd.DataFrame, tmp_filename: str, file_type: FileType) -> None:

data_flow/lib/pandas.py → mysiar_data_flow/lib/pandas.py

@@ -1,7 +1,7 @@
 import fireducks.pandas as fd
 import pandas as pd
 
-from data_flow.lib.FileType import FileType
+from mysiar_data_flow.lib.FileType import FileType
 
 
 def from_pandas_2_file(df: pd.DataFrame, tmp_filename: str, file_type: FileType) -> None:

pyproject.toml

@@ -1,3 +1,66 @@
+[tool.poetry]
+name = "mysiar-data-flow"
+version = "0.0.1rc2"
+readme = "README.md"
+description = "Python data manipulation library"
+authors = ["Piotr Synowiec <[email protected]>"]
+maintainers = ["Piotr Synowiec <[email protected]>"]
+keywords = ["dataframe", "pandas", "polars", "fireducks"]
+classifiers = [
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Utilities"
+]
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.13"
+fireducks = "*"
+tables = "*"
+pyarrow = "*"
+pandas = "*"
+polars = "*"
+black = "*"
+flake8 = "*"
+pyproject-flake8 = "*"
+pytest = "*"
+pytest-cov = "*"
+poetry = "*"
+
+
+[project.urls]
+Repository = "https://github.com/mysiar-org/python-data-flow"
+Issues = "https://github.com/mysiar-org/python-data-flow/issues"
+#Changelog=
+
+[tool.poetry.extras]
+all = [
+
+]
+dev = [
+    "black",
+    "flake8",
+    "pyproject-flake8",
+    "pytest",
+    "pytest-cov",
+    "poetry",
+]
+
+#[tool.poetry.packages]
+#packages = [
+#    { include = "data_flow" }
+#]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+
+
+
 [tool.black]
 line-length = 120
 

requirements.dev.txt

@@ -3,3 +3,5 @@ flake8
 pyproject-flake8
 pytest
 pytest-cov
+poetry
+twine

tests/BaseTestCase.py

@@ -4,8 +4,8 @@
 
 import pandas as pd
 
-from data_flow import DataFlow
-from data_flow.lib import Operator
+from mysiar_data_flow import DataFlow
+from mysiar_data_flow.lib import Operator
 
 
 class BaseTestCase(unittest.TestCase):

tests/test_data_flow_csv.py

@@ -1,8 +1,8 @@
 import unittest
 
-from data_flow import DataFlow
-from data_flow.lib import FileType
-from data_flow.lib.tools import delete_file
+from mysiar_data_flow import DataFlow
+from mysiar_data_flow.lib import FileType
+from mysiar_data_flow.lib.tools import delete_file
 from tests.BaseTestCase import BaseTestCase
 
 

tests/test_data_flow_feather.py

@@ -1,8 +1,8 @@
 import unittest
 
-from data_flow import DataFlow
-from data_flow.lib import FileType
-from data_flow.lib.tools import delete_file
+from mysiar_data_flow import DataFlow
+from mysiar_data_flow.lib import FileType
+from mysiar_data_flow.lib.tools import delete_file
 from tests.BaseTestCase import BaseTestCase
 
 

tests/test_data_flow_hdf.py

@@ -1,8 +1,8 @@
 import unittest
 
-from data_flow import DataFlow
-from data_flow.lib import FileType
-from data_flow.lib.tools import delete_file
+from mysiar_data_flow import DataFlow
+from mysiar_data_flow.lib import FileType
+from mysiar_data_flow.lib.tools import delete_file
 from tests.BaseTestCase import BaseTestCase
 
 

tests/test_data_flow_json.py

@@ -1,8 +1,8 @@
 import unittest
 
-from data_flow import DataFlow
-from data_flow.lib import FileType
-from data_flow.lib.tools import delete_file
+from mysiar_data_flow import DataFlow
+from mysiar_data_flow.lib import FileType
+from mysiar_data_flow.lib.tools import delete_file
 from tests.BaseTestCase import BaseTestCase
 
 

tests/test_data_flow_parquet.py

@@ -1,8 +1,8 @@
 import unittest
 
-from data_flow import DataFlow
-from data_flow.lib import FileType
-from data_flow.lib.tools import delete_file
+from mysiar_data_flow import DataFlow
+from mysiar_data_flow.lib import FileType
+from mysiar_data_flow.lib.tools import delete_file
 from tests.BaseTestCase import BaseTestCase