add not_in filter for tables and dataframes (#7439)

## 📝 Summary

<!--
Provide a concise summary of what this pull request is addressing.

If this PR fixes any issues, list them here by number (e.g., Fixes
#123).
-->


https://github.com/user-attachments/assets/4ed2c47c-d8e2-4df4-9dd3-d15d28a2420e

## 🔍 Description of Changes

<!--
Detail the specific changes made in this pull request. Explain the
problem addressed and how it was resolved. If applicable, provide before
and after comparisons, screenshots, or any relevant details to help
reviewers understand the changes easily.
-->

## 📋 Checklist

- [x] I have read the [contributor
guidelines](https://github.com/marimo-team/marimo/blob/main/CONTRIBUTING.md).
- [ ] For large changes, or changes that affect the public API: this
change was discussed or approved through an issue, on
[Discord](https://marimo.io/discord?ref=pr), or the community
[discussions](https://github.com/marimo-team/marimo/discussions) (Please
provide a link if applicable).
- [x] I have added tests for the changes made.
- [x] I have run the code and verified that it works as expected.

Author: Light2Dark

frontend/src/components/data-table/context-menu.tsx

@@ -95,11 +95,11 @@ export const CellContextMenu = <TData,>({
   const column = cell.column;
   const canFilter = column.getCanFilter() && column.columnDef.meta?.filterType;
 
-  const handleFilterCell = () => {
+  const handleFilterCell = (operator: "in" | "not_in") => {
     column.setFilterValue(
       Filter.select({
         options: [cell.getValue()],
-        operator: "in",
+        operator,
       }),
     );
   };
@@ -119,10 +119,14 @@ export const CellContextMenu = <TData,>({
       {canFilter && (
         <>
           <ContextMenuSeparator />
-          <ContextMenuItem onClick={handleFilterCell}>
+          <ContextMenuItem onClick={() => handleFilterCell("in")}>
             <FilterIcon className="mo-dropdown-icon h-3 w-3" />
             Filter by this value
           </ContextMenuItem>
+          <ContextMenuItem onClick={() => handleFilterCell("not_in")}>
+            <FilterIcon className="mo-dropdown-icon h-3 w-3" />
+            Remove rows with this value
+          </ContextMenuItem>
         </>
       )}
     </ContextMenuContent>

frontend/src/components/data-table/filter-pills.tsx

@@ -96,7 +96,8 @@ function formatValue(value: ColumnFilterValue, timeFormatter: DateFormatter) {
     const stringifiedOptions = value.options.map((o) =>
       stringifyUnknownValue({ value: o }),
     );
-    return `is in [${stringifiedOptions.join(", ")}]`;
+    const operator = value.operator === "in" ? "is in" : "not in";
+    return `${operator} [${stringifiedOptions.join(", ")}]`;
   }
   if (value.type === "text") {
     return `contains "${value.text}"`;

frontend/src/components/data-table/filters.ts

@@ -7,6 +7,7 @@ import type { ConditionType } from "@/plugins/impl/data-frames/schema";
 import type { ColumnId } from "@/plugins/impl/data-frames/types";
 import type { OperatorType } from "@/plugins/impl/data-frames/utils/operators";
 import { assertNever } from "@/utils/assertNever";
+import { Logger } from "@/utils/Logger";
 
 declare module "@tanstack/react-table" {
   //allows us to define custom properties for our columns
@@ -192,12 +193,20 @@ export function filterToFilterCondition(
       }
 
       return [];
-    case "select":
+    case "select": {
+      let operator = filter.operator;
+      if (filter.operator !== "in" && filter.operator !== "not_in") {
+        Logger.warn("Invalid operator for select filter", {
+          operator: filter.operator,
+        });
+        operator = "in"; // default to in operator
+      }
       return {
         column_id: columnId,
-        operator: "in",
+        operator,
         value: filter.options,
       };
+    }
 
     default:
       assertNever(filter);

frontend/src/plugins/impl/data-frames/utils/__tests__/operators.test.ts

@@ -100,12 +100,14 @@ describe("isConditionValueValid", () => {
     );
     expect(isConditionValueValid("contains", "test")).toBe(true);
     expect(isConditionValueValid("in", ["test"])).toBe(true);
+    expect(isConditionValueValid("not_in", ["test"])).toBe(true);
   });
 
   it("should return false if the value is not valid according to the schema for the given operator", () => {
     expect(isConditionValueValid("==", "not a number")).toBe(false);
     expect(isConditionValueValid("contains", 123)).toBe(false);
     expect(isConditionValueValid("in", "not an array")).toBe(false);
+    expect(isConditionValueValid("not_in", "not an array")).toBe(false);
   });
 
   it("should return true if the operator does not require a value", () => {

frontend/src/plugins/impl/data-frames/utils/operators.ts

@@ -58,6 +58,7 @@ export const STRING_OPERATORS = {
   starts_with: [Schema.string],
   ends_with: [Schema.string],
   in: [Schema.stringMultiColumnValues],
+  not_in: [Schema.stringMultiColumnValues],
   is_null: [],
   is_not_null: [],
 };

marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py

@@ -223,6 +223,14 @@ def convert_value(v: Any, converter: Callable[[str], Any]) -> Any:
                     condition_expr = column.is_in(value) | column.is_null()
                 else:
                     condition_expr = column.is_in(value or [])
+            elif condition.operator == "not_in":
+                # ~is_in returns null for null values, so we need to explicitly include/exclude nulls
+                if value is not None and None in value:
+                    condition_expr = ~column.is_in(value) & ~column.is_null()
+                else:
+                    condition_expr = (
+                        ~column.is_in(value or []) | column.is_null()
+                    )
             else:
                 assert_never(condition.operator)
 

marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py

@@ -56,8 +56,9 @@ def generate_where_clause(df_name: str, where: Condition) -> str:
             return f"{df_name}[{_as_literal(column_id)}].str.startswith({_as_literal(value)})"  # noqa: E501
         elif operator == "ends_with":
             return f"{df_name}[{_as_literal(column_id)}].str.endswith({_as_literal(value)})"  # noqa: E501
-        elif operator == "in":
-            return f"{df_name}[{_as_literal(column_id)}].isin({_list_of_strings(value)})"  # noqa: E501
+        elif operator == "in" or operator == "not_in":
+            result = f"{df_name}[{_as_literal(column_id)}].isin({_list_of_strings(value)})"  # noqa: E501
+            return result if operator == "in" else f"~{result}"
         elif operator == "!=":
             return (
                 f"{df_name}[{_as_literal(column_id)}].ne({_as_literal(value)})"
@@ -237,8 +238,9 @@ def generate_where_clause_polars(where: Condition) -> str:
             return f"pl.col({_as_literal(column_id)}).str.starts_with({_as_literal(value)})"  # noqa: E501
         elif operator == "ends_with":
             return f"pl.col({_as_literal(column_id)}).str.ends_with({_as_literal(value)})"  # noqa: E501
-        elif operator == "in":
-            return f"pl.col({_as_literal(column_id)}).is_in({_list_of_strings(value)})"  # noqa: E501
+        elif operator == "in" or operator == "not_in":
+            result = f"pl.col({_as_literal(column_id)}).is_in({_list_of_strings(value)})"  # noqa: E501
+            return result if operator == "in" else f"~{result}"
         elif operator in [">", ">=", "<", "<="]:
             return f"pl.col({_as_literal(column_id)}) {operator} {_as_literal(value)}"  # noqa: E501
         elif operator == "is_null":
@@ -397,8 +399,9 @@ def generate_where_clause(df_name: str, where: Condition) -> str:
             return f"({df_name}[{_as_literal(column_id)}].startswith({_as_literal(value)}))"  # noqa: E501
         elif operator == "ends_with":
             return f"({df_name}[{_as_literal(column_id)}].endswith({_as_literal(value)}))"  # noqa: E501
-        elif operator == "in":
-            return f"({df_name}[{_as_literal(column_id)}].isin({_list_of_strings(value)}))"  # noqa: E501
+        elif operator == "in" or operator == "not_in":
+            result = f"({df_name}[{_as_literal(column_id)}].isin({_list_of_strings(value)}))"  # noqa: E501
+            return result if operator == "in" else f"~{result}"
         elif operator in [">", ">=", "<", "<="]:
             return f"({df_name}[{_as_literal(column_id)}] {operator} {_as_literal(value)})"  # noqa: E501
         elif operator == "is_null":

marimo/_plugins/ui/_impl/dataframes/transforms/types.py

@@ -40,6 +40,7 @@
     "starts_with",
     "ends_with",
     "in",
+    "not_in",
 ]
 Aggregation = Literal[
     "count",
@@ -77,7 +78,7 @@ def __hash__(self) -> int:
         return hash((self.column_id, self.operator, self.value))
 
     def __post_init__(self) -> None:
-        if self.operator == "in":
+        if self.operator == "in" or self.operator == "not_in":
             if isinstance(self.value, list):
                 # Hack to convert to tuple for frozen dataclass
                 # Only tuples can be hashed
@@ -86,7 +87,7 @@ def __post_init__(self) -> None:
                 pass
             else:
                 raise ValueError(
-                    "value must be a list or tuple for 'in' operator"
+                    "value must be a list or tuple for 'in' or 'not_in' operator"
                 )
 
 

tests/_plugins/ui/_impl/dataframes/test_handlers.py

@@ -584,6 +584,111 @@ def test_filter_rows_in_operator_null_rows(
         result = apply(df, transform)
         assert_frame_equal(result, expected)
 
+    @staticmethod
+    @pytest.mark.parametrize(
+        ("df", "expected"),
+        list(
+            zip(
+                create_test_dataframes({"A": [1, 2, 3], "B": [4, 5, 6]}),
+                create_test_dataframes({"A": [3], "B": [6]}),
+            )
+        ),
+    )
+    def test_filter_rows_not_in_operator(
+        df: DataFrameType, expected: DataFrameType
+    ) -> None:
+        transform = FilterRowsTransform(
+            type=TransformType.FILTER_ROWS,
+            operation="keep_rows",
+            where=[Condition(column_id="A", operator="not_in", value=[1, 2])],
+        )
+        result = apply(df, transform)
+        assert_frame_equal(result, expected)
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        ("df", "expected"),
+        list(
+            zip(
+                create_test_dataframes(
+                    {"A": ["foo", "bar", "baz"], "B": [1, 2, 3]}
+                ),
+                create_test_dataframes({"A": ["baz"], "B": [3]}),
+            )
+        ),
+    )
+    def test_filter_rows_not_in_operator_strings(
+        df: DataFrameType, expected: DataFrameType
+    ) -> None:
+        transform = FilterRowsTransform(
+            type=TransformType.FILTER_ROWS,
+            operation="keep_rows",
+            where=[
+                Condition(
+                    column_id="A", operator="not_in", value=["foo", "bar"]
+                )
+            ],
+        )
+        result = apply(df, transform)
+        assert_frame_equal(result, expected)
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        ("df", "expected"),
+        [
+            *zip(
+                create_test_dataframes(
+                    {"A": [1, 2, 3, None], "B": [4, 5, 6, 7]},
+                    exclude=["ibis"],
+                ),
+                create_test_dataframes({"A": [3], "B": [6]}, exclude=["ibis"]),
+            ),
+        ],
+    )
+    def test_filter_rows_not_in_operator_with_nulls(
+        df: DataFrameType, expected: DataFrameType
+    ) -> None:
+        # not_in with None in value should exclude rows where A is 1, 2, or null
+        transform = FilterRowsTransform(
+            type=TransformType.FILTER_ROWS,
+            operation="keep_rows",
+            where=[
+                Condition(column_id="A", operator="not_in", value=[1, 2, None])
+            ],
+        )
+        result = apply(df, transform)
+        assert_frame_equal(result, expected)
+
+    @staticmethod
+    @pytest.mark.parametrize(
+        ("df", "expected"),
+        [
+            *zip(
+                create_test_dataframes(
+                    {"A": [1, 2, 3, None], "B": [4, 5, 6, 7]},
+                    exclude=["ibis"],
+                ),
+                create_test_dataframes(
+                    {"A": [3, None], "B": [6, 7]}, exclude=["ibis"]
+                ),
+            ),
+        ],
+    )
+    def test_filter_rows_not_in_operator_keep_nulls(
+        df: DataFrameType, expected: DataFrameType
+    ) -> None:
+        # not_in WITHOUT None in value should keep null rows (only exclude 1 and 2)
+        transform = FilterRowsTransform(
+            type=TransformType.FILTER_ROWS,
+            operation="keep_rows",
+            where=[Condition(column_id="A", operator="not_in", value=[1, 2])],
+        )
+        result = apply(df, transform)
+        if nw.dependencies.is_pandas_dataframe(result):
+            assert_frame_equal_with_nans(result, expected)
+        else:
+            assert_frame_equal(result, expected)
+
     @staticmethod
     @pytest.mark.parametrize(
         ("df", "expected"),

tests/_plugins/ui/_impl/dataframes/test_print_code.py

@@ -131,7 +131,7 @@ def create_transform_strategy(
     list_condition_strategy = st.builds(
         Condition,
         column_id=list_column_id,
-        operator=st.just("in"),
+        operator=st.sampled_from(["in", "not_in"]),
         value=st.lists(st.one_of(st.text()), min_size=1),
     )