Add an introduction with some more handholding and mention NaN

polars/11_missing_data.py

@@ -20,7 +20,190 @@ def _(mo):
 
     _by [etrotta](https://github.com/etrotta)_
 
-    This notebook covers some common problems you may face when dealing with real datasets and techniques used to solve deal with them, providing an overview of polars functionalities to handle missing data.
+    This notebook covers some common problems you may face when dealing with real datasets and techniques used to solve deal with them, showcasing polars functionalities to handle missing data.
+
+    First we provide an overview of the methods available in polars, then we walk through a mini case study with real world data showing how to use it, and at last we provide some additional information in the 'Bonus Content' section.
+    You can navigate to skip around to each header using the menu on the right side
+    """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+    ## Methods for working with Nulls
+
+    We'll be using the following DataFrame to show the most important methods:
+    """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(pl):
+    df = pl.DataFrame(
+        [
+            {"species": "Dog", "name": "Millie", "height": None, "age": 4},
+            {"species": "Dog", "name": "Wally", "height": 60, "age": None},
+            {"species": "Dog", "name": None, "height": 50, "age": 12},
+            {"species": "Cat", "name": "Mini", "height": 15, "age": None},
+            {"species": "Cat", "name": None, "height": 25, "age": 6},
+            {"species": "Cat", "name": "Kazusa", "height": None, "age": 16},
+        ]
+    )
+    df
+    return (df,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+    ### Counting nulls
+
+    A simple yet convenient aggregation
+    """
+    )
+    return
+
+
+@app.cell
+def _(df):
+    df.null_count()
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+    ### Dropping Nulls
+
+    The simplest way of dealing with null values is throwing them away, but that is not always a good idea.
+    """
+    )
+    return
+
+
+@app.cell
+def _(df):
+    df.drop_nulls()
+    return
+
+
+@app.cell
+def _(df):
+    df.drop_nulls(subset="name")
+    return
+
+
+@app.cell
+def _(mo):
+    mo.md(
+        r"""
+    ### Filtering null values
+
+    To filter in polars, you'll typically use `df.filter(expression)` or `df.remove(expression)` methods.
+
+    Filter will only keep rows in which the expression evaluates to True.
+    It will remove not only rows in which it evalutes to False, but also those in which the expression evaluates to None.
+
+    Remove will only remove rows in which the expression evaluates to True.
+    It will keep rows in which it evalutes to None.
+    """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    df.filter(pl.col("age") > 10)
+    return
+
+
+@app.cell
+def _(df, pl):
+    df.remove(pl.col("age") < 10)
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+    You may also be tempted to use `== None` or `!= None`, but comparison operators in polars will generally pass null values through.
+
+    You can use `.eq_missing()` or `.ne_missing()` methods if you want to be strict about it, but there are also `.is_null()` and `.is_not_null()` methods you can use.
+    """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    df.select(
+        "name",
+        (pl.col("name") == None).alias("Name equals None"),
+        (pl.col("name") == "Mini").alias("Name equals Mini"),
+        (pl.col("name").eq_missing("Mini")).alias("Name eq_missing Mini"),
+        (pl.col("name").is_null()).alias("Name is null"),
+        (pl.col("name").is_not_null()).alias("Name is not null"),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+    ### Filling Null values
+
+    You can also fill in the values with constants, calculations or by consulting external data sources.
+
+    Be careful not to treat estimated or guessed values as if they a ground truth however, otherwise you may end up making conclusions about a reality that does not exists.
+    """
+    )
+    return
+
+
+@app.cell
+def _(df, mo, pl):
+    guesstimates = df.with_columns(
+        pl.col("height").fill_null(pl.col("height").mean().over("species")),
+        pl.col("age").fill_null(0),
+    )
+    guesstimates = mo.ui.data_editor(
+        guesstimates,
+        editable_columns=["name"],
+        label="Let's guess some values to fill in nulls, then try giving names to the animals with `null` by editing the cells",
+    )
+    guesstimates
+    return (guesstimates,)
+
+
+@app.cell
+def _(guesstimates):
+    guesstimates.value
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+    ### TL;DR
+
+    Before we head into the mini case study, a brief review of what we have covered:
+
+    - use `df.null_counts()` or `expr.is_null()` to count and identify missing values
+    - you could just drop rows with values missing in any columns or a subset of them with `df.drop_nulls()`, but for most cases you'll want to be more careful about it
+    - take into consideration whenever you want to preserve null values or remove them when choosing between `df.filter()` or `df.remove()`
+    - if you don't want to propagate null values, use `_missing` variations of methods such as `eq` vs `eq_missing`
+    - you may want to fill in missing values based on calculations via `fill_null`, or manually edit the data based on external documents
+
+    Whichever approach you take, remember to document how you handled them!
     """
     )
     return
@@ -30,7 +213,9 @@ def _(mo):
 def _(mo):
     mo.md(
         r"""
-    We will be using a dataset about the weather in Rio de Janeiro, originally available in Google Big Query under `datario.clima_pluviometro`. What you need to know about it: 
+    # Mini Case Study
+
+    We will be using a dataset from `alertario` about the weather in Rio de Janeiro, originally available in Google Big Query under `datario.clima_pluviometro`. What you need to know about it: 
 
     - Contains multiple stations covering the Municipality of Rio de Janeiro
     - Measures the precipitation as milimeters, with a granularity of 15 minutes
@@ -133,7 +318,7 @@ def _(dirty_weather, mo, rain):
     _missing_count = dirty_weather.select(rain.is_null().sum()).item()
 
     mo.md(
-        f"As you can see, there are {_missing_count:,} rows missing the accumulated rain for a period.\n\nThat could be cause due to sensor malfunctions, maintenance, bobby tables or a myriad of other reasons. While it may be a small percentage of the data ({_missing_count / len(dirty_weather):.3%}), it is still important to take it in consideration, one way or the other."
+        f"As you can see, there are {_missing_count:,} rows missing the accumulated rain for a period.\n\nThat could be caused by sensor malfunctions, maintenance, bobby tables or a myriad of other reasons. While it may be a small percentage of the data ({_missing_count / len(dirty_weather):.3%}), it is still important to take it in consideration, one way or the other."
     )
     return
 
@@ -160,7 +345,7 @@ def _(mo):
 
     ### Last option to fixing it: Acquire the correct values from elsewhere.
 
-    We will not explore this option in this notebook, but you could try finding approximate values from another dataset or in some cases manually input the correct values.
+    Like manually adding names to the animals in the introduction, but you could try finding approximate values from another dataset or in some cases manually input the correct values.
 
     ### However
 
@@ -207,6 +392,8 @@ def _(mo):
 
     That difference could be caused by the same factors as null values, or even by someone dropping null values along the way, but for the purposes of this notebook let's say that we want to have values for each combination with no exceptions, so we'll have to make reasonable assumptions to interpolate and extrapolate them.
 
+    ### Upsampling
+
     Given that we are working with time series data, we will [upsample](https://docs.pola.rs/api/python/stable/reference/dataframe/api/polars.DataFrame.upsample.html) the data, but you could also create a DataFrame containing all expected rows then use `join(how="...")`
 
     However, that will give us _even more_ null values, so we will want to fill them in afterwards. For this case, we will just use a forward fill followed by a backwards fill.
@@ -245,52 +432,63 @@ def _(dirty_weather, mo, pl, rain):
 
 @app.cell(hide_code=True)
 def _(mo):
-    mo.md(r"""Now that we finally have a clean dataset, let's play around with it a little""")
+    mo.md(
+        r"""
+    Now that we finally have a clean dataset, let's play around with it a little.
+
+    ### Example App
+
+    Let's display the amount of precipitation each station measured within a timeframe, aggregated to a lower granularity.
+
+    First we'll filter by day
+    """
+    )
     return
 
 
 @app.cell(hide_code=True)
 def _(mo):
-    year_picker = mo.ui.dropdown(options=[2020, 2021, 2022], value=2022, label="Year")
-    day_slider = mo.ui.range_slider(1, 365, show_value=True, label="Day of the year", full_width=True, value=[87, 94])
-    hour_slider = mo.ui.range_slider(0, 24, 0.25, show_value=True, label="Hour of the day", full_width=True)
-    interval = mo.ui.dropdown(
-        options=["15m", "30m", "1h", "2h", "4h", "6h", "1d"], value="4h", label="Aggregation Granularity"
+    filters = (
+        mo.md(
+            """Filters for the example
+
+        Year: {year}
+        Days of the year: {day}
+        Hours of each day: {hour}
+        Aggregation granularity: {interval}
+        """
+        )
+        .batch(
+            year=mo.ui.dropdown([2020, 2021, 2022], value=2022),
+            day=mo.ui.range_slider(1, 365, show_value=True, full_width=True, value=[87, 94]),
+            hour=mo.ui.range_slider(0, 24, 0.25, show_value=True, full_width=True),
+            interval=mo.ui.dropdown(["15m", "30m", "1h", "2h", "4h", "6h", "1d", "7d", "30d"], value="4h"),
+        )
+        .form()
     )
 
-    mo.vstack(
-        [
-            year_picker,
-            day_slider,
-            hour_slider,
-            interval,
-        ]
-    )
-    return day_slider, hour_slider, interval, year_picker
+    # Note: You could use `mo.ui.date_range` instead, but I just don't like it myself
+    # mo.ui.date_range(start="2020-01-01", stop="2022-12-31", value=["2022-03-28", "2022-04-03"], label="Display range")
+
+    filters
+    return (filters,)
 
 
 @app.cell
-def _(
-    day_slider,
-    hour_slider,
-    interval,
-    pl,
-    rain,
-    stations,
-    weather,
-    year_picker,
-):
-    _range_seconds = map(lambda hour: hour * 3600, hour_slider.value)
+def _(filters, mo, pl, rain, stations, weather):
+    mo.stop(filters.value is None)
+
+    _range_seconds = map(lambda hour: hour * 3600, filters.value["hour"])
     _df_seconds = pl.col("datetime").dt.hour() + pl.col("datetime").dt.minute().mul(60)
 
     animation_data = (
         weather.lazy()
         .filter(
-            pl.col("datetime").dt.year() == year_picker.value,
-            pl.col("datetime").dt.ordinal_day().is_between(*day_slider.value),
+            pl.col("datetime").dt.year() == filters.value["year"],
+            pl.col("datetime").dt.ordinal_day().is_between(*filters.value["day"]),
             _df_seconds.is_between(*_range_seconds),
         )
-        .group_by_dynamic("datetime", group_by="station", every=interval.value)
+        .group_by_dynamic("datetime", group_by="station", every=filters.value["interval"])
         .agg(rain.sum().alias("precipitation"))
         .remove(pl.col("precipitation").eq(0).all().over("station"))
         .join(stations.lazy(), on="station")
@@ -369,13 +567,13 @@ def _(mo):
 def _(mo):
     mo.md(
         r"""
-    ## Bonus Content
+    # Bonus Content
 
-    ### Appendix A: Missing Time Zones
+    ## Appendix A: Missing Time Zones
 
     The original dataset contained naive datetimes instead of timezone-aware, but we can infer whenever it refers to UTC time or local time (for this case, -03:00 UTC) based on the measurements.
 
-    For example, we can select one specific interval during which we know rained a lot, or graph the average amount of precipitation for each hour of the day, then compare the data timestamps with a ground truth.
+    For example, we can select one specific interval during which we know that rained a lot, or graph the average amount of precipitation for each hour of the day, then compare the data timestamps with a ground truth.
     """
     )
     return
@@ -438,7 +636,7 @@ def _(mo):
         r"""
     By externally researching the expected distribution and looking up some of the extreme weather events, we can come to a conclusion about whenever it is aligned with the local time or with UTC.
 
-    In this case, the distribution matches the normal weather for this region and we can see that the hours with the most precipitation match those of historical events, so it is safe to say it is using Americas/São Paulo time zone.
+    In this case, the distribution matches the normal weather for this region and we can see that the hours with the most precipitation match those of historical events, so it is safe to say it is using local time (equivalent to the Americas/São Paulo time zone).
     """
     )
     return
@@ -460,7 +658,45 @@ def _(dirty_weather_naive, pl):
 def _(mo):
     mo.md(
         r"""
-    ### Utilities
+    ## Appendix B: Not a Number
+
+    While some other tools without proper support for missing values, as well as datasets made to work with them, may use `NaN` as a way to indicate a value is missing, in polars it is treated exclusively as a float value, much like `0.0`, `1.0` or `infinity`.
+
+    You can use `.fill_null(float('nan'))` if you need to convert to a format such tools accept, or use `.fill_nan(None)` if you are importing data from them, assuming that there are no values which really are supposed to be the float NaN.
+
+    Remember that many calculations can result in NaN, for example dividing by zero:
+    """
+    )
+    return
+
+
+@app.cell
+def _(dirty_weather, pl, rain):
+    day_perc = dirty_weather.select(
+        "datetime",
+        (rain / rain.sum().over("station", pl.col("datetime").dt.date())).alias("percentage_of_day_precipitation"),
+    )
+    perc_col = pl.col("percentage_of_day_precipitation")
+
+    day_perc
+    return day_perc, perc_col
+
+
+@app.cell(hide_code=True)
+def _(day_perc, mo, perc_col):
+    mo.md(f"""It is null for {day_perc.select(perc_col.is_null().mean()).item():.4%} of the rows, but is NaN for {day_perc.select(perc_col.is_nan().mean()).item():.4%} of them.
+    If we use the cleaned weather dataframe to calculate it instead of the dirty_weather, we will have no Nulls, but note how for this calculation we can end up with both, with each having a different meaning.
+
+    In this case it makes sense to fill in NaNs as 0 to indicate there was no rain during that period, but treating the nulls the same could lead to a different interpretation of the data, so remember to handle NaNs and nulls separately.
+    """)
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+    ## Utilities
 
     Loading data and imports
     """