utils.files.pickle

  1"""
  2File operations with [pickles](https://docs.python.org/3/library/pickle.html).
  3"""
  4
  5import pathlib
  6import pandas
  7
  8from typing import Optional, List, Union
  9
 10from ..datasets import pandas as pnd
 11from ..files import file as fl
 12from ..logs.log import logger
 13
 14
 15def openPickleAsPandasTable(
 16    pickleFilePath: Union[str, pathlib.Path]
 17) -> pandas.DataFrame:
 18    """
 19    Read [Pandas](https://pandas.pydata.org) table from provided pickle file
 20    (*after checking that the file exists and that it is actually a file*).
 21
 22    Example:
 23
 24    ``` py
 25    from phab.utils.files import pickle
 26
 27    pnd = pickle.openPickleAsPandasTable("/path/to/some.pkl")
 28    #print(pnd.head(15))
 29    ```
 30    """
 31    pickleFile: Optional[pathlib.Path] = fl.fileExists(pickleFilePath)
 32    if pickleFile is None:
 33        raise ValueError(
 34            f"Provided path to [{pickleFilePath}] seems to be wrong"
 35        )
 36    else:
 37        return pandas.read_pickle(pickleFile)
 38
 39
 40def savePandasTableAsPickle(
 41    pandasTable: pandas.DataFrame,
 42    pickleFilePath: Union[str, pathlib.Path]
 43) -> None:
 44    """
 45    Save [Pandas](https://pandas.pydata.org) table to a pickle file.
 46
 47    Example:
 48
 49    ``` py
 50    from phab.utils.files import pickle
 51
 52    savePandasTableAsPickle(pnd, "/path/to/some.pkl")
 53    ```
 54    """
 55    filePath: pathlib.Path = pathlib.Path()
 56    if isinstance(pickleFilePath, str):
 57        filePath = pathlib.Path(pickleFilePath)
 58    else:
 59        filePath = pickleFilePath
 60    if filePath.exists():
 61        raise ValueError(f"The [{filePath}] file already exists")
 62    pandasTable.to_pickle(filePath)
 63
 64
 65def mergePickles(
 66    picklesToMergePath: Union[str, pathlib.Path],
 67    resultingPicklePath: Union[None, str, pathlib.Path]
 68) -> Optional[pandas.DataFrame]:
 69    """
 70    Merge several pickle files into one. Looks for pickle files (*`*.pkl`*)
 71    in the provided folder, reads them to [Pandas](https://pandas.pydata.org)
 72    tables (*with `utils.files.pickle.openPickleAsPandasTable`*)
 73    and concatenates those into one final Pandas table
 74    (*using `utils.datasets.pandas.mergeTables`*).
 75
 76    Saves resulting Pandas table to file (*if provided path is not `None`*)
 77    or just returns it.
 78
 79    Example:
 80
 81    ``` py
 82    from phab.utils.files import pickle
 83
 84    pickle.mergePickles(
 85        "/path/to/pickles/to/merge/",
 86        "/path/to/where/to/save/result.pkl"
 87    )
 88
 89    # or
 90
 91    tbl = pickle.mergePickles(
 92        "/path/to/pickles/to/merge/",
 93        None
 94    )
 95    #print(tbl.head(15))
 96    ```
 97    """
 98    picklesToMerge = None
 99    inputPath: Optional[pathlib.Path] = fl.directoryExists(picklesToMergePath)
100    if inputPath is None:
101        raise ValueError(
102            f"Provided path to [{picklesToMergePath}] seems to be wrong"
103        )
104    else:
105        picklesToMerge = list(inputPath.glob("**/*.pkl"))
106
107    frames = []
108
109    filesCount = len(picklesToMerge)
110    logger.debug(f"Found files: {filesCount}")
111    if filesCount == 0:
112        raise ValueError("There are no files in the provided folder")
113    # elif filesCount == 1:
114    #     raise ValueError(
115    #         "[ERROR] There is only one file in the provided folder"
116    #     )
117    else:
118        for p in picklesToMerge:
119            logger.info(f"Merging {p}...")
120            tbl = openPickleAsPandasTable(p)
121            logger.debug(f"Records in this pickle: {len(tbl)}")
122            frames.append(tbl)
123
124    mergedTable = pnd.mergeTables(frames)
125
126    if resultingPicklePath:
127        savePandasTableAsPickle(mergedTable, resultingPicklePath)
128        return None
129    else:
130        return mergedTable