Propositionalization: Predicting air pollution in Beijing¶

# If we are in Colab, we need to fetch the utils folder from the repository
if os.getenv("COLAB_RELEASE_TAG"):
    !curl -L https://api.github.com/repos/getml/getml-demo/tarball/master | tar --wildcards --strip-components=1 -xz '*utils*'

parent = Path(os.getcwd()).parent.as_posix()

if parent not in sys.path:
    sys.path.append(parent)

from utils import Benchmark, FTTimeSeriesBuilder, TSFreshBuilder

FEATURETOOLS_FILES = ["featuretools_training.csv", "featuretools_test.csv"]

for fname in FEATURETOOLS_FILES:
    if not os.path.exists(fname):
        fname, res = request.urlretrieve(
            "https://static.getml.com/datasets/air_pollution/featuretools/" + fname,
            fname,
        )

TSFRESH_FILES = ["tsfresh_training.csv", "tsfresh_test.csv"]

for fname in TSFRESH_FILES:
    if not os.path.exists(fname):
        fname, res = request.urlretrieve(
            "https://static.getml.com/datasets/air_pollution/tsfresh/" + fname, fname
        )

fname = "PRSA_data_2010.1.1-2014.12.31.csv"

if not os.path.exists(fname):
    fname, res = request.urlretrieve(
        "https://archive.ics.uci.edu/ml/machine-learning-databases/00381/" + fname,
        fname,
    )

data_full_pandas = pd.read_csv(fname)

data_full_pandas = data_full_pandas[
    data_full_pandas["pm2.5"] == data_full_pandas["pm2.5"]
]

def add_leading_zero(val):
    if len(str(val)) == 1:
        return "0" + str(val)
    return str(val)


data_full_pandas["month"] = [add_leading_zero(val) for val in data_full_pandas["month"]]

data_full_pandas["day"] = [add_leading_zero(val) for val in data_full_pandas["day"]]

data_full_pandas["hour"] = [add_leading_zero(val) for val in data_full_pandas["hour"]]


def make_date(year, month, day, hour):
    return year + "-" + month + "-" + day + " " + hour + ":00:00"


data_full_pandas["date"] = [
    make_date(str(year), month, day, hour)
    for year, month, day, hour in zip(
        data_full_pandas["year"],
        data_full_pandas["month"],
        data_full_pandas["day"],
        data_full_pandas["hour"],
    )
]

data_full_pandas["id"] = 1

data_train_pandas = data_full_pandas[data_full_pandas["year"] < 2014]
data_test_pandas = data_full_pandas[data_full_pandas["year"] == 2014]
data_full_pandas = data_full_pandas

def remove_unwanted_columns(df):
    del df["cbwd"]
    del df["year"]
    del df["month"]
    del df["day"]
    del df["hour"]
    del df["No"]
    return df


data_full_pandas = remove_unwanted_columns(data_full_pandas)
data_train_pandas = remove_unwanted_columns(data_train_pandas)
data_test_pandas = remove_unwanted_columns(data_test_pandas)

df_full = getml.data.DataFrame.from_pandas(data_full_pandas, name="full")
df_full["date"] = df_full["date"].as_ts()

benchmark = Benchmark()

predictor = getml.predictors.XGBoostRegressor()

pipe_fp_pr = getml.pipeline.Pipeline(
    tags=["prediction", "fastprop"], predictors=[predictor]
)

ft_builder = FTTimeSeriesBuilder(
    num_features=200,
    horizon=pd.Timedelta(days=0),
    memory=pd.Timedelta(days=1),
    column_id="id",
    time_stamp="date",
    target="pm2.5",
)

df_featuretools_training = getml.data.DataFrame.from_pandas(
    featuretools_training, name="featuretools_training"
)
df_featuretools_test = getml.data.DataFrame.from_pandas(
    featuretools_test, name="featuretools_test"
)

def set_roles_featuretools(df):
    df["date"] = df["date"].as_ts()
    df.set_role(["pm2.5"], getml.data.roles.target)
    df.set_role(["date"], getml.data.roles.time_stamp)
    df.set_role(df.roles.unused, getml.data.roles.numerical)
    df.set_role(["id"], getml.data.roles.unused_float)
    return df


df_featuretools_training = set_roles_featuretools(df_featuretools_training)
df_featuretools_test = set_roles_featuretools(df_featuretools_test)

df_tsfresh_training = getml.data.DataFrame.from_pandas(
    tsfresh_training, name="tsfresh_training"
)
df_tsfresh_test = getml.data.DataFrame.from_pandas(tsfresh_test, name="tsfresh_test")

def set_roles_tsfresh(df):
    df["date"] = df["date"].as_ts()
    df.set_role(["pm2.5"], getml.data.roles.target)
    df.set_role(["date"], getml.data.roles.time_stamp)
    df.set_role(df.roles.unused, getml.data.roles.numerical)
    df.set_role(["id"], getml.data.roles.unused_float)
    return df


df_tsfresh_training = set_roles_tsfresh(df_tsfresh_training)
df_tsfresh_test = set_roles_tsfresh(df_tsfresh_test)

# export for further use
comparison.to_csv("comparisons/air_pollution.csv")

getml.engine.shutdown()