Skip to content

Commit 3d4e741

Browse files
NickCrewsNickCrews
authored
docs: clarify NULL/NaN/inf distinction in docstring and examples (#11077)
1 parent 4c74ddd commit 3d4e741

File tree

2 files changed

+127
-55
lines changed

2 files changed

+127
-55
lines changed

‎ibis/expr/types/generic.py

Lines changed: 67 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -429,7 +429,10 @@ def typeof(self) -> ir.StringValue:
429429
return ops.TypeOf(self).to_expr()
430430

431431
def fill_null(self, fill_value: Scalar, /) -> Value:
432-
"""Replace any null values with the indicated fill value.
432+
"""Replace `NULL`s with the given value. Does NOT affect `NaN` and `inf` values.
433+
434+
This only replaces genuine `NULL` values, it does NOT affect
435+
`NaN` and `inf` values for floating point types.
433436
434437
Parameters
435438
----------
@@ -440,36 +443,45 @@ def fill_null(self, fill_value: Scalar, /) -> Value:
440443
--------
441444
[`Value.coalesce()`](./expression-generic.qmd#ibis.expr.types.generic.Value.coalesce)
442445
[`ibis.coalesce()`](./expression-generic.qmd#ibis.coalesce)
446+
[`Value.isnull()`](./expression-generic.qmd#ibis.expr.types.generic.Value.isnull)
447+
[`FloatingValue.isnan()`](./expression-numeric.qmd#ibis.expr.types.numeric.FloatingValue.isnan)
448+
[`FloatingValue.isinf()`](./expression-numeric.qmd#ibis.expr.types.numeric.FloatingValue.isinf)
443449
444450
Examples
445451
--------
446452
>>> import ibis
447453
>>> ibis.options.interactive = True
448-
>>> t = ibis.examples.penguins.fetch().limit(5)
449-
>>> t.sex
450-
┏━━━━━━━━┓
451-
┃ sex ┃
452-
┡━━━━━━━━┩
453-
│ string │
454-
├────────┤
455-
│ male │
456-
│ female │
457-
│ female │
458-
│ NULL │
459-
│ female │
460-
└────────┘
461-
>>> t.sex.fill_null("unrecorded").name("sex")
462-
┏━━━━━━━━━━━━┓
463-
┃ sex ┃
464-
┡━━━━━━━━━━━━┩
465-
│ string │
466-
├────────────┤
467-
│ male │
468-
│ female │
469-
│ female │
470-
│ unrecorded │
471-
│ female │
472-
└────────────┘
454+
>>> t = ibis.memtable({"f": [None, "-inf", "3.0", "inf", "nan"]})
455+
>>> t = t.mutate(f=ibis._.f.cast(float))
456+
>>> t = t.mutate(filled=t.f.fill_null(99))
457+
>>> t
458+
┏━━━━━━━━━┳━━━━━━━━━┓
459+
┃ f ┃ filled ┃
460+
┡━━━━━━━━━╇━━━━━━━━━┩
461+
│ float64 │ float64 │
462+
├─────────┼─────────┤
463+
│ NULL │ 99.0 │
464+
│ -inf │ -inf │
465+
│ 3.0 │ 3.0 │
466+
│ inf │ inf │
467+
│ nan │ nan │
468+
└─────────┴─────────┘
469+
470+
If you want to fill all `NaN` and `inf` values as well, use something like
471+
the following:
472+
473+
>>> t.mutate(filled2=ibis.or_(t.f.isnull(), t.f.isnan(), t.f.isinf()).ifelse(99, t.f))
474+
┏━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓
475+
┃ f ┃ filled ┃ filled2 ┃
476+
┡━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩
477+
│ float64 │ float64 │ float64 │
478+
├─────────┼─────────┼─────────┤
479+
│ NULL │ 99.0 │ 99.0 │
480+
│ -inf │ -inf │ 99.0 │
481+
│ 3.0 │ 3.0 │ 3.0 │
482+
│ inf │ inf │ 99.0 │
483+
│ nan │ nan │ 99.0 │
484+
└─────────┴─────────┴─────────┘
473485
474486
Returns
475487
-------
@@ -480,7 +492,7 @@ def fill_null(self, fill_value: Scalar, /) -> Value:
480492

481493
@deprecated(as_of="9.1", instead="use fill_null instead")
482494
def fillna(self, fill_value: Scalar, /) -> Value:
483-
"""DEPRECATED: use `fill_null` instead."""
495+
"""DEPRECATED: use `fill_null` instead, which acts exactly the same."""
484496
return self.fill_null(fill_value)
485497

486498
def nullif(self, null_if_expr: Value, /) -> Value:
@@ -879,37 +891,39 @@ def bind(table):
879891
return bind(_)
880892

881893
def isnull(self) -> ir.BooleanValue:
882-
"""Return whether this expression is NULL.
894+
"""Whether this expression is `NULL`. Does NOT detect `NaN` and `inf` values.
895+
896+
For FloatingValue types, use [`FloatingValue.isnan()`](./expression-numeric.qmd#ibis.expr.types.numeric.FloatingValue.isnan)
897+
and [`FloatingValue.isinf()`](./expression-numeric.qmd#ibis.expr.types.numeric.FloatingValue.isinf) to detect `NaN` and `inf` values.
898+
899+
See Also
900+
--------
901+
[`Value.fill_null()`](./expression-generic.qmd#ibis.expr.types.generic.Value.fill_null)
902+
[`FloatingValue.isnan()`](./expression-numeric.qmd#ibis.expr.types.numeric.FloatingValue.isnan)
903+
[`FloatingValue.isinf()`](./expression-numeric.qmd#ibis.expr.types.numeric.FloatingValue.isinf)
883904
884905
Examples
885906
--------
886907
>>> import ibis
887908
>>> ibis.options.interactive = True
888-
>>> t = ibis.examples.penguins.fetch().limit(5)
889-
>>> t.bill_depth_mm
890-
┏━━━━━━━━━━━━━━━┓
891-
┃ bill_depth_mm ┃
892-
┡━━━━━━━━━━━━━━━┩
893-
│ float64 │
894-
├───────────────┤
895-
│ 18.7 │
896-
│ 17.4 │
897-
│ 18.0 │
898-
│ NULL │
899-
│ 19.3 │
900-
└───────────────┘
901-
>>> t.bill_depth_mm.isnull()
902-
┏━━━━━━━━━━━━━━━━━━━━━━━┓
903-
┃ IsNull(bill_depth_mm) ┃
904-
┡━━━━━━━━━━━━━━━━━━━━━━━┩
905-
│ boolean │
906-
├───────────────────────┤
907-
│ False │
908-
│ False │
909-
│ False │
910-
│ True │
911-
│ False │
912-
└───────────────────────┘
909+
>>> t = ibis.memtable({"f": [None, "-inf", "3.0", "inf", "nan"]})
910+
>>> t = t.mutate(f=ibis._.f.cast(float))
911+
>>> t.mutate(
912+
... isnull=t.f.isnull(),
913+
... isnan=t.f.isnan(),
914+
... isinf=t.f.isinf(),
915+
... )
916+
┏━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓
917+
┃ f ┃ isnull ┃ isnan ┃ isinf ┃
918+
┡━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩
919+
│ float64 │ boolean │ boolean │ boolean │
920+
├─────────┼─────────┼─────────┼─────────┤
921+
│ NULL │ True │ NULL │ NULL │
922+
│ -inf │ False │ False │ True │
923+
│ 3.0 │ False │ False │ False │
924+
│ inf │ False │ False │ True │
925+
│ nan │ False │ True │ False │
926+
└─────────┴─────────┴─────────┴─────────┘
913927
"""
914928
return ops.IsNull(self).to_expr()
915929

‎ibis/expr/types/numeric.py

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1662,11 +1662,69 @@ def bit_xor(self, *, where: ir.BooleanValue | None = None) -> IntegerScalar:
16621662
@public
16631663
class FloatingValue(NumericValue):
16641664
def isnan(self) -> ir.BooleanValue:
1665-
"""Return whether the value is NaN."""
1665+
"""Return whether the value is NaN. Does NOT detect `NULL` and `inf` values.
1666+
1667+
See Also
1668+
--------
1669+
[`Value.isnull()`](./expression-generic.qmd#ibis.expr.types.generic.Value.fill_null)
1670+
[`FloatingValue.isinf()`](./expression-numeric.qmd#ibis.expr.types.numeric.FloatingValue.isinf)
1671+
1672+
Examples
1673+
--------
1674+
>>> import ibis
1675+
>>> ibis.options.interactive = True
1676+
>>> t = ibis.memtable({"f": [None, "-inf", "3.0", "inf", "nan"]})
1677+
>>> t = t.mutate(f=ibis._.f.cast(float))
1678+
>>> t.mutate(
1679+
... isnull=t.f.isnull(),
1680+
... isnan=t.f.isnan(),
1681+
... isinf=t.f.isinf(),
1682+
... )
1683+
┏━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓
1684+
┃ f ┃ isnull ┃ isnan ┃ isinf ┃
1685+
┡━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩
1686+
│ float64 │ boolean │ boolean │ boolean │
1687+
├─────────┼─────────┼─────────┼─────────┤
1688+
│ NULL │ True │ NULL │ NULL │
1689+
│ -inf │ False │ False │ True │
1690+
│ 3.0 │ False │ False │ False │
1691+
│ inf │ False │ False │ True │
1692+
│ nan │ False │ True │ False │
1693+
└─────────┴─────────┴─────────┴─────────┘
1694+
"""
16661695
return ops.IsNan(self).to_expr()
16671696

16681697
def isinf(self) -> ir.BooleanValue:
1669-
"""Return whether the value is infinity."""
1698+
"""Return whether the value is +/-inf. Does NOT detect `NULL` and `inf` values.
1699+
1700+
See Also
1701+
--------
1702+
[`Value.isnull()`](./expression-generic.qmd#ibis.expr.types.generic.Value.fill_null)
1703+
[`FloatingValue.isnan()`](./expression-numeric.qmd#ibis.expr.types.numeric.FloatingValue.isnan)
1704+
1705+
Examples
1706+
--------
1707+
>>> import ibis
1708+
>>> ibis.options.interactive = True
1709+
>>> t = ibis.memtable({"f": [None, "-inf", "3.0", "inf", "nan"]})
1710+
>>> t = t.mutate(f=ibis._.f.cast(float))
1711+
>>> t.mutate(
1712+
... isnull=t.f.isnull(),
1713+
... isnan=t.f.isnan(),
1714+
... isinf=t.f.isinf(),
1715+
... )
1716+
┏━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓
1717+
┃ f ┃ isnull ┃ isnan ┃ isinf ┃
1718+
┡━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩
1719+
│ float64 │ boolean │ boolean │ boolean │
1720+
├─────────┼─────────┼─────────┼─────────┤
1721+
│ NULL │ True │ NULL │ NULL │
1722+
│ -inf │ False │ False │ True │
1723+
│ 3.0 │ False │ False │ False │
1724+
│ inf │ False │ False │ True │
1725+
│ nan │ False │ True │ False │
1726+
└─────────┴─────────┴─────────┴─────────┘
1727+
"""
16701728
return ops.IsInf(self).to_expr()
16711729

16721730

0 commit comments

Comments
 (0)