getml.data.DataFrame

def add(
    self,
    col: Union[StringColumn, FloatColumn, np.ndarray],
    name: str,
    role: Optional[Role] = None,
    subroles: Optional[Union[Role, Iterable[str]]] = None,
    unit: str = "",
    time_formats: Optional[Iterable[str]] = None,
):
    """Adds a column to the current [`DataFrame`][getml.DataFrame].

    Args:
        col:
            The column or numpy.ndarray to be added.

        name:
            Name of the new column.

        role:
            Role of the new column. Must be from [`roles`][getml.data.roles].

        subroles:
            Subroles of the new column. Must be from [`subroles`][getml.data.subroles].

        unit:
            Unit of the column.

        time_formats:
            Formats to be used to parse the time stamps.

            This is only necessary, if an implicit conversion from
            a [`StringColumn`][getml.data.columns.StringColumn] to a time
            stamp is taking place.

            The formats are allowed to contain the following
            special characters:

            * %w - abbreviated weekday (Mon, Tue, ...)
            * %W - full weekday (Monday, Tuesday, ...)
            * %b - abbreviated month (Jan, Feb, ...)
            * %B - full month (January, February, ...)
            * %d - zero-padded day of month (01 .. 31)
            * %e - day of month (1 .. 31)
            * %f - space-padded day of month ( 1 .. 31)
            * %m - zero-padded month (01 .. 12)
            * %n - month (1 .. 12)
            * %o - space-padded month ( 1 .. 12)
            * %y - year without century (70)
            * %Y - year with century (1970)
            * %H - hour (00 .. 23)
            * %h - hour (00 .. 12)
            * %a - am/pm
            * %A - AM/PM
            * %M - minute (00 .. 59)
            * %S - second (00 .. 59)
            * %s - seconds and microseconds (equivalent to %S.%F)
            * %i - millisecond (000 .. 999)
            * %c - centisecond (0 .. 9)
            * %F - fractional seconds/microseconds (000000 - 999999)
            * %z - time zone differential in ISO 8601 format (Z or +NN.NN)
            * %Z - time zone differential in RFC format (GMT or +NNNN)
            * %% - percent sign
    """

    if isinstance(col, np.ndarray):
        self._add_numpy_array(col, name, role, subroles, unit)
        return

    col, role, subroles = _with_column(
        col, name, role, subroles, unit, time_formats
    )

    is_string = isinstance(col, (StringColumnView, StringColumn))

    if is_string:
        self._add_categorical_column(col, name, role, subroles, unit)
    else:
        self._add_column(col, name, role, subroles, unit)

def copy(self, name: str) -> DataFrame:
    """
    Creates a deep copy of the data frame under a new name.

    Args:
        name:
            The name of the new data frame.

    Returns:
            A handle to the deep copy.
    """

    if not isinstance(name, str):
        raise TypeError("'name' must be a string.")

    cmd: Dict[str, Any] = {}

    cmd["type_"] = "DataFrame.concat"
    cmd["name_"] = name

    cmd["data_frames_"] = [self._getml_deserialize()]

    comm.send(cmd)

    return DataFrame(name=name).refresh()

def delete(self):
    """
    Permanently deletes the data frame. `delete` first unloads the data frame
    from memory and then deletes it from disk.
    """
    # ------------------------------------------------------------

    self._delete()

def drop(
    self,
    cols: Union[
        FloatColumn,
        StringColumn,
        str,
        Union[Iterable[FloatColumn], Iterable[StringColumn], Iterable[str]],
    ],
) -> View:
    """Returns a new [`View`][getml.data.View] that has one or several columns removed.

    Args:
        cols:
            The columns or the names thereof.

    Returns:
        A new [`View`][getml.data.View] object with the specified columns removed.
    """

    names = _handle_cols(cols)

    return View(base=self, dropped=names)

def freeze(self):
    """Freezes the data frame.

    After you have frozen the data frame, the data frame is immutable
    and in-place operations are no longer possible. However, you can
    still create views. In other words, operations like
    [`set_role`][getml.DataFrame.set_role] are no longer possible,
    but operations like [`with_role`][getml.DataFrame.with_role] are.
    """
    cmd: Dict[str, Any] = {}
    cmd["type_"] = "DataFrame.freeze"
    cmd["name_"] = self.name
    comm.send(cmd)

@classmethod
def from_arrow(
    cls,
    table: pa.Table,
    name: str,
    roles: Optional[Union[Dict[Union[Role, str], Iterable[str]], Roles]] = None,
    ignore: bool = False,
    dry: bool = False,
) -> Union[DataFrame, Roles]:
    """Create a DataFrame from an Arrow Table.

    This is one of the fastest way to get data into the
    getML Engine.

    Args:
        table:
            The arrow tablelike to be read.

        name:
            Name of the data frame to be created.

        roles:
            Maps the [`roles`][getml.data.roles] to the
            column names (see [`colnames`][getml.DataFrame.colnames]).

            The `roles` dictionary is expected to have the following format:
            ```python
            roles = {getml.data.role.numeric: ["colname1", "colname2"],
                     getml.data.role.target: ["colname3"]}
            ```
            Otherwise, you can use the [`Roles`][getml.data.Roles] class.

        ignore:
            Only relevant when roles is not None.
            Determines what you want to do with any colnames not
            mentioned in roles. Do you want to ignore them (True)
            or read them in as unused columns (False)?

        dry:
            If set to True, the data will not be read. Instead, the method
            will return the inferred roles.

    Returns:
            Handler of the underlying data.
    """

    # ------------------------------------------------------------

    if not isinstance(name, str):
        raise TypeError("'name' must be str.")

    # The content of roles is checked in the class constructor called below.
    if roles is not None and not isinstance(roles, (dict, Roles)):
        raise TypeError("'roles' must be a geml.data.Roles object, a dict or None.")

    if not isinstance(ignore, bool):
        raise TypeError("'ignore' must be bool.")

    if not isinstance(dry, bool):
        raise TypeError("'dry' must be bool.")

    # ------------------------------------------------------------

    sniffed_roles = sniff_arrow(table)

    roles = _prepare_roles(roles, sniffed_roles, ignore_sniffed_roles=ignore)

    if dry:
        return roles

    data_frame = cls(name, roles)

    return data_frame.read_arrow(table=table, append=False)

@classmethod
def from_csv(
    cls,
    fnames: Union[str, Iterable[str]],
    name: str,
    num_lines_sniffed: None = None,
    num_lines_read: int = 0,
    quotechar: str = '"',
    sep: str = ",",
    skip: int = 0,
    colnames: Iterable[str] = (),
    roles: Optional[Union[Dict[Union[Role, str], Iterable[str]], Roles]] = None,
    ignore: bool = False,
    dry: bool = False,
    verbose: bool = True,
    block_size: int = DEFAULT_CSV_READ_BLOCK_SIZE,
    in_batches: bool = False,
) -> Union[DataFrame, Roles]:
    """Create a DataFrame from CSV files.

    The getML Engine will construct a data
    frame object in the Engine, fill it with the data read from
    the CSV file(s), and return a corresponding
    [`DataFrame`][getml.DataFrame] handle.

    Args:
        fnames:
            CSV file paths to be read.

        name:
            Name of the data frame to be created.

        num_lines_sniffed:
            Number of lines analyzed by the sniffer.

        num_lines_read:
            Number of lines read from each file.
            Set to 0 to read in the entire file.

        quotechar:
            The character used to wrap strings.

        sep:
            The separator used for separating fields.

        skip:
            Number of lines to skip at the beginning of each file.

        colnames: The first line of a CSV file
            usually contains the column names. When this is not the case,
            you need to explicitly pass them.

        roles:
            Maps the [`roles`][getml.data.roles] to the
            column names (see [`colnames`][getml.DataFrame.colnames]).

            The `roles` dictionary is expected to have the following format
            ```python
            roles = {getml.data.role.numeric: ["colname1", "colname2"],
                     getml.data.role.target: ["colname3"]}
            ```
            Otherwise, you can use the [`Roles`][getml.data.Roles] class.

        ignore:
            Only relevant when roles is not None.
            Determines what you want to do with any colnames not
            mentioned in roles. Do you want to ignore them (True)
            or read them in as unused columns (False)?

        dry:
            If set to True, the data will not be read. Instead, the method
            will return the inferred roles.

        verbose:
            If True, when fnames are urls, the filenames are
            printed to stdout during the download.

        block_size:
            The number of bytes read with each batch. Passed down to
            pyarrow.

        in_batches:
            If True, read blocks streamwise manner and send those batches to
            the engine. Blocks are read and sent to the engine sequentially.
            While more memory efficient, streaming in batches is slower as
            it is inherently single-threaded. If False (default) the data is
            read with multiple threads into arrow first and sent to the engine
            afterwards.

    Returns:
            Handler of the underlying data.


    Deprecated:
        1.5: The `num_lines_sniffed` parameter is deprecated.

    Note:
        It is assumed that the first line of each CSV file
        contains a header with the column names.

    ??? example
        Let's assume you have two CSV files - *file1.csv* and
        *file2.csv* - in the current working directory. You can
        import their data into the getML Engine using.
        ```python
        df_expd = data.DataFrame.from_csv(
            fnames=["file1.csv", "file2.csv"],
            name="MY DATA FRAME",
            sep=';',
            quotechar='"'
            )

        # However, the CSV format lacks type safety. If you want to
        # build a reliable pipeline, it is a good idea
        # to hard-code the roles:

        roles = {"categorical": ["col1", "col2"], "target": ["col3"]}

        df_expd = data.DataFrame.from_csv(
            fnames=["file1.csv", "file2.csv"],
            name="MY DATA FRAME",
            sep=';',
            quotechar='"',
            roles=roles
            )

        # If you think that typing out all the roles by hand is too
        # cumbersome, you can use a dry run:

        roles = data.DataFrame.from_csv(
            fnames=["file1.csv", "file2.csv"],
            name="MY DATA FRAME",
            sep=';',
            quotechar='"',
            dry=True
        )
        ```

        This will return the roles dictionary it would have used. You
        can now hard-code this.

    """

    if num_lines_sniffed is not None:
        warnings.warn(
            "The 'num_lines_sniffed' parameter is deprecated and will be ignored.",
            DeprecationWarning,
        )

    if isinstance(fnames, str):
        fnames = [fnames]

    if not _is_non_empty_typed_list(fnames, str):
        raise TypeError("'fnames' must be either a str or a list of str.")

    if not isinstance(name, str):
        raise TypeError("'name' must be str.")

    if not isinstance(num_lines_read, numbers.Real):
        raise TypeError("'num_lines_read' must be a real number")

    if not isinstance(quotechar, str):
        raise TypeError("'quotechar' must be str.")

    if not isinstance(sep, str):
        raise TypeError("'sep' must be str.")

    if not isinstance(skip, numbers.Real):
        raise TypeError("'skip' must be a real number")

    if roles is not None and not isinstance(roles, (dict, Roles)):
        raise TypeError("'roles' must be a geml.data.Roles object, a dict or None.")

    if not isinstance(ignore, bool):
        raise TypeError("'ignore' must be bool.")

    if not isinstance(ignore, bool):
        raise TypeError("'dry' must be bool.")

    if colnames:
        if not _is_iterable_not_str_of_type(colnames, str):
            raise TypeError("'colnames' must be an iterable of str")

    fnames = _retrieve_urls(fnames, verbose=verbose)

    sniffed_roles = sniff_csv(
        fnames=fnames,
        quotechar=quotechar,
        sep=sep,
        skip=int(skip),
        colnames=colnames,
    )

    roles = _prepare_roles(roles, sniffed_roles, ignore_sniffed_roles=ignore)

    if dry:
        return roles

    data_frame = cls(name, roles)

    return data_frame.read_csv(
        fnames=fnames,
        append=False,
        quotechar=quotechar,
        sep=sep,
        num_lines_read=num_lines_read,
        skip=skip,
        colnames=colnames,
        block_size=block_size,
        in_batches=in_batches,
    )

@classmethod
def from_db(
    cls,
    table_name: str,
    name: Optional[str] = None,
    roles: Optional[Union[Dict[Union[Role, str], Iterable[str]], Roles]] = None,
    ignore: bool = False,
    dry: bool = False,
    conn: Optional[Connection] = None,
) -> Union[DataFrame, Roles]:
    """Create a DataFrame from a table in a database.

    It will construct a data frame object in the Engine, fill it
    with the data read from table `table_name` in the connected
    database (see [`database`][getml.database]), and return a
    corresponding [`DataFrame`][getml.DataFrame] handle.

    Args:
        table_name:
            Name of the table to be read.

        name:
            Name of the data frame to be created. If not passed,
            then the *table_name* will be used.

        roles:
            Maps the [`roles`][getml.data.roles] to the
            column names (see [`colnames`][getml.DataFrame.colnames]).

            The `roles` dictionary is expected to have the following format:
            ```python
            roles = {getml.data.role.numeric: ["colname1", "colname2"],
                     getml.data.role.target: ["colname3"]}
            ```
            Otherwise, you can use the [`Roles`][getml.data.Roles] class.

        ignore:
            Only relevant when roles is not None.
            Determines what you want to do with any colnames not
            mentioned in roles. Do you want to ignore them (True)
            or read them in as unused columns (False)?

        dry:
            If set to True, the data will not be read. Instead, the method
            will return the inferred roles.

        conn:
            The database connection to be used.
            If you don't explicitly pass a connection, the Engine
            will use the default connection.

    Returns:
            Handler of the underlying data.

    ??? example
        ```python
        getml.database.connect_mysql(
            host="db.relational-data.org",
            port=3306,
            dbname="financial",
            user="guest",
            password="relational"
        )

        loan = getml.DataFrame.from_db(
            table_name='loan', name='data_frame_loan')
        ```
    """

    # -------------------------------------------

    name = name or table_name

    # -------------------------------------------

    if not isinstance(table_name, str):
        raise TypeError("'table_name' must be str.")

    if not isinstance(name, str):
        raise TypeError("'name' must be str.")

    # The content of roles is checked in the class constructor called below.
    if roles is not None and not isinstance(roles, (dict, Roles)):
        raise TypeError(
            "'roles' must be a getml.data.Roles object, a dict or None."
        )

    if not isinstance(ignore, bool):
        raise TypeError("'ignore' must be bool.")

    if not isinstance(dry, bool):
        raise TypeError("'dry' must be bool.")

    # -------------------------------------------

    conn = conn or database.Connection()

    # ------------------------------------------------------------

    sniffed_roles = _sniff_db(table_name, conn)

    roles = _prepare_roles(roles, sniffed_roles, ignore_sniffed_roles=ignore)

    if dry:
        return roles

    # ------------------------------------------------------------

    data_frame = cls(name, roles)

    return data_frame.read_db(table_name=table_name, append=False, conn=conn)

@classmethod
def from_dict(
    cls,
    data: Dict[Hashable, Iterable[Any]],
    name: str,
    roles: Optional[Union[Dict[Union[Role, str], Iterable[str]], Roles]] = None,
    ignore: bool = False,
    dry: bool = False,
) -> Union[DataFrame, Roles]:
    """Create a new DataFrame from a dict

    Args:
        data:
            The dict containing the data.
            The data should be in the following format:
            ```python
            data = {'col1': [1.0, 2.0, 1.0], 'col2': ['A', 'B', 'C']}
            ```
        name:
            Name of the data frame to be created.

        roles:
            Maps the [`roles`][getml.data.roles] to the
            column names (see [`colnames`][getml.DataFrame.colnames]).

            The `roles` dictionary is expected to have the following format:
            ```python
            roles = {getml.data.role.numeric: ["colname1", "colname2"],
                     getml.data.role.target: ["colname3"]}
            ```
            Otherwise, you can use the [`Roles`][getml.data.Roles] class.

        ignore:
            Only relevant when roles is not None.
            Determines what you want to do with any colnames not
            mentioned in roles. Do you want to ignore them (True)
            or read them in as unused columns (False)?

        dry:
            If set to True, the data will not be read. Instead, the method
            will return the inferred roles.
    Returns:
            Handler of the underlying data.
    """

    if not isinstance(data, dict):
        raise TypeError("'data' must be dict.")

    return cls.from_arrow(
        table=pa.Table.from_pydict(data),
        name=name,
        roles=roles,
        ignore=ignore,
        dry=dry,
    )

@classmethod
def from_json(
    cls,
    json_str: str,
    name: str,
    roles: Optional[Union[Dict[Union[Role, str], Iterable[str]], Roles]] = None,
    ignore: bool = False,
    dry: bool = False,
) -> Union[DataFrame, Roles]:
    """Create a new DataFrame from a JSON string.

    It will construct a data frame object in the Engine, fill it
    with the data read from the JSON string, and return a
    corresponding [`DataFrame`][getml.DataFrame] handle.

    Args:
        json_str:
            The JSON string containing the data.
            The json_str should be in the following format:
            ```python
            json_str = "{'col1': [1.0, 2.0, 1.0], 'col2': ['A', 'B', 'C']}"
            ```
        name:
            Name of the data frame to be created.

        roles:
            Maps the [`roles`][getml.data.roles] to the
            column names (see [`colnames`][getml.DataFrame.colnames]).

            The `roles` dictionary is expected to have the following format:
            ```python
            roles = {getml.data.role.numeric: ["colname1", "colname2"],
                     getml.data.role.target: ["colname3"]}
            ```
            Otherwise, you can use the [`Roles`][getml.data.Roles] class.

        ignore:
            Only relevant when roles is not None.
            Determines what you want to do with any colnames not
            mentioned in roles. Do you want to ignore them (True)
            or read them in as unused columns (False)?

        dry:
            If set to True, the data will not be read. Instead, the method
            will return the inferred roles.

    Returns:
        Handler of the underlying data.

    """

    if not isinstance(json_str, str):
        raise TypeError("'json_str' must be str.")

    return cls.from_dict(
        data=json.loads(json_str),
        name=name,
        roles=roles,
        ignore=ignore,
        dry=dry,
    )

@classmethod
def from_pandas(
    cls,
    pandas_df: pd.DataFrame,
    name: str,
    roles: Optional[Union[Dict[Union[Role, str], Iterable[str]], Roles]] = None,
    ignore: bool = False,
    dry: bool = False,
) -> Union[DataFrame, Roles]:
    """Create a DataFrame from a `pandas.DataFrame`.

    It will construct a data frame object in the Engine, fill it
    with the data read from the `pandas.DataFrame`, and
    return a corresponding [`DataFrame`][getml.DataFrame] handle.

    Args:
        pandas_df:
            The table to be read.

        name:
            Name of the data frame to be created.

        roles:
            Maps the [`roles`][getml.data.roles] to the
            column names (see [`colnames`][getml.DataFrame.colnames]).

            The `roles` dictionary is expected to have the following format:
            ```python
             roles = {getml.data.role.numeric: ["colname1", "colname2"],
                      getml.data.role.target: ["colname3"]}
            ```
            Otherwise, you can use the [`Roles`][getml.data.Roles] class.

        ignore:
            Only relevant when roles is not None.
            Determines what you want to do with any colnames not
            mentioned in roles. Do you want to ignore them (True)
            or read them in as unused columns (False)?

        dry:
            If set to True, the data will not be read. Instead, the method
            will return the inferred roles.

    Returns:
        Handler of the underlying data.
    """

    # ------------------------------------------------------------

    if not isinstance(pandas_df, pd.DataFrame):
        raise TypeError("'pandas_df' must be of type pandas.DataFrame.")

    if not isinstance(name, str):
        raise TypeError("'name' must be str.")

    # The content of roles is checked in the class constructor called below.
    if roles is not None and not isinstance(roles, (dict, Roles)):
        raise TypeError("'roles' must be a geml.data.Roles object, a dict or None.")

    if not isinstance(ignore, bool):
        raise TypeError("'ignore' must be bool.")

    if not isinstance(dry, bool):
        raise TypeError("'dry' must be bool.")

    # ------------------------------------------------------------

    sniffed_roles = _sniff_pandas(pandas_df)

    roles = _prepare_roles(roles, sniffed_roles, ignore_sniffed_roles=ignore)

    if dry:
        return roles

    data_frame = cls(name, roles)

    return data_frame.read_pandas(pandas_df=pandas_df, append=False)

@classmethod
def from_parquet(
    cls,
    fnames: Union[str, Iterable[str]],
    name: str,
    roles: Optional[Union[Dict[Union[Role, str], Iterable[str]], Roles]] = None,
    ignore: bool = False,
    dry: bool = False,
    colnames: Iterable[str] = (),
) -> Union[DataFrame, Roles]:
    """Create a DataFrame from parquet files.

    This is one of the fastest way to get data into the
    getML Engine.

    Args:
        fnames:
            The path of the parquet file(s) to be read.

        name:
            Name of the data frame to be created.

        roles:
            Maps the [`roles`][getml.data.roles] to the
            column names (see [`colnames`][getml.DataFrame.colnames]).

            The `roles` dictionary is expected to have the following format:
            ```python
            roles = {getml.data.role.numeric: ["colname1", "colname2"],
                     getml.data.role.target: ["colname3"]}
            ```
            Otherwise, you can use the [`Roles`][getml.data.Roles] class.

        ignore:
            Only relevant when roles is not None.
            Determines what you want to do with any colnames not
            mentioned in roles. Do you want to ignore them (True)
            or read them in as unused columns (False)?

        dry:
            If set to True, the data will not be read. Instead, the method
            will return the inferred roles.

    Returns:
        Handler of the underlying data.
    """

    # ------------------------------------------------------------

    if isinstance(fnames, str):
        fnames = [fnames]

    if not isinstance(name, str):
        raise TypeError("'name' must be str.")

    # The content of roles is checked in the class constructor called below.
    if roles is not None and not isinstance(roles, (dict, Roles)):
        raise TypeError("'roles' must be a geml.data.Roles object, a dict or None.")

    if not isinstance(ignore, bool):
        raise TypeError("'ignore' must be bool.")

    if not isinstance(dry, bool):
        raise TypeError("'dry' must be bool.")

    # ------------------------------------------------------------

    sniffed_roles = sniff_parquet(fnames, colnames)

    roles = _prepare_roles(roles, sniffed_roles, ignore_sniffed_roles=ignore)

    if dry:
        return roles

    data_frame = cls(name, roles)

    return data_frame.read_parquet(fnames=fnames, append=False, colnames=colnames)

@classmethod
def from_pyspark(
    cls,
    spark_df: pyspark.sql.DataFrame,
    name: str,
    roles: Optional[Union[Dict[Union[Role, str], Iterable[str]], Roles]] = None,
    ignore: bool = False,
    dry: bool = False,
) -> Union[DataFrame, Roles]:
    """Create a DataFrame from a `pyspark.sql.DataFrame`.

    It will construct a data frame object in the Engine, fill it
    with the data read from the `pyspark.sql.DataFrame`, and
    return a corresponding [`DataFrame`][getml.DataFrame] handle.

    Args:
        spark_df:
            The table to be read.

        name:
            Name of the data frame to be created.

        roles:
            Maps the [`roles`][getml.data.roles] to the
            column names (see [`colnames`][getml.DataFrame.colnames]).

            The `roles` dictionary is expected to have the following format:
            ```python
            roles = {getml.data.role.numeric: ["colname1", "colname2"],
                     getml.data.role.target: ["colname3"]}
            ```

            Otherwise, you can use the [`Roles`][getml.data.Roles] class.

        ignore:
            Only relevant when roles is not None.
            Determines what you want to do with any colnames not
            mentioned in roles. Do you want to ignore them (True)
            or read them in as unused columns (False)?

        dry:
            If set to True, the data will not be read. Instead, the method
            will return the inferred roles.

    Returns:
            Handler of the underlying data.
    """

    # ------------------------------------------------------------

    if not isinstance(name, str):
        raise TypeError("'name' must be str.")

    # The content of roles is checked in the class constructor called below.
    if roles is not None and not isinstance(roles, (dict, Roles)):
        raise TypeError("'roles' must be a geml.data.Roles object, a dict or None.")

    if not isinstance(ignore, bool):
        raise TypeError("'ignore' must be bool.")

    if not isinstance(dry, bool):
        raise TypeError("'dry' must be bool.")

    # ------------------------------------------------------------

    head = spark_df.limit(2).toPandas()

    sniffed_roles = _sniff_pandas(head)

    roles = _prepare_roles(roles, sniffed_roles, ignore_sniffed_roles=ignore)

    if dry:
        return roles

    data_frame = cls(name, roles)

    return data_frame.read_pyspark(spark_df=spark_df, append=False)