From 805fa58eb289c1e699dc93172663218cd19a4b20 Mon Sep 17 00:00:00 2001 From: Ryan Malloy Date: Thu, 3 Sep 2026 15:42:56 -0600 Subject: [PATCH] Rows can answer to a column name, if you ask for it Field request from a user running this alongside SQL Server, where both pyodbc and mssql-python hand back rows that take a position, a column name, and an attribute. The argument is readability on wide projections: row[11] tells a reader nothing, and stays correct only until somebody adds a column in the middle. row_factory=Row gives all three. It is set on the connection, so it is one line for the whole application rather than per query, which is what "without any additional coding" has to mean in practice. A cursor can override it. Opt-in, not default, and the number is why. Measured on a 20,000-row five-column fetch: 37.2 ms with tuples, 40.8 ms with Row, about 9%. Defaulting it on would move the published 1.05-1.15x ratio against IfxPy to roughly 1.15-1.25x. That is not a trade to make on behalf of somebody running a bulk export that never looks at a column by name. Row subclasses tuple, so row == (1, "x") still holds. That constraint shaped the design more than anything else: this suite compares fetched rows against plain tuples in hundreds of places, and so does everybody's code. Slices degrade to plain tuples, since a slice has no column map. Details that had to be decided rather than assumed, all measured against the servers: Informix folds unquoted identifiers to lower case, so the .lower() in the workaround people write by hand is a no-op. Expression columns get names like "(count(*))" that cannot be attributes, so they are subscript-only. Duplicate names resolve to the first occurrence, matching pyodbc. And tuple already defines count and index, so a column with either name gets a descriptor and wins, because on a database row that is plainly what the caller meant. The per-shape class is cached, so a type() call does not land on every small query, and rows pickle by rebuilding from their field names. --- src/informix_db/__init__.py | 4 + src/informix_db/connections.py | 5 + src/informix_db/cursors.py | 35 ++++- src/informix_db/rows.py | 157 ++++++++++++++++++++ tests/test_rows.py | 264 +++++++++++++++++++++++++++++++++ 5 files changed, 463 insertions(+), 2 deletions(-) create mode 100644 src/informix_db/rows.py create mode 100644 tests/test_rows.py diff --git a/src/informix_db/__init__.py b/src/informix_db/__init__.py index 9cec256..faf2035 100644 --- a/src/informix_db/__init__.py +++ b/src/informix_db/__init__.py @@ -50,6 +50,7 @@ from .pool import ( PoolTimeoutError, create_pool, ) +from .rows import Row # PEP 249 module-level globals apilevel = "2.0" @@ -85,6 +86,7 @@ __all__ = [ "PoolClosedError", "PoolTimeoutError", "ProgrammingError", + "Row", "RowValue", "ServerCapabilities", "Warning", @@ -111,6 +113,7 @@ def connect( client_locale: str = "en_US.8859-1", env: dict[str, str] | None = None, autocommit: bool = False, + row_factory: object | None = None, tls: bool | ssl.SSLContext = False, tls_server_hostname: str | None = None, ) -> Connection: @@ -153,4 +156,5 @@ def connect( client_locale=client_locale, env=env, autocommit=autocommit, + row_factory=row_factory, ) diff --git a/src/informix_db/connections.py b/src/informix_db/connections.py index bc542eb..d7052c3 100644 --- a/src/informix_db/connections.py +++ b/src/informix_db/connections.py @@ -283,6 +283,7 @@ class Connection: client_locale: str = "en_US.8859-1", env: dict[str, str] | None = None, autocommit: bool = False, # honored from Phase 3 onward + row_factory: object | None = None, tls: bool | ssl.SSLContext = False, tls_server_hostname: str | None = None, ): @@ -337,6 +338,10 @@ class Connection: # before the next DML in non-autocommit mode. We default to "no # open txn" — the first DML will trigger SQ_BEGIN. self._in_transaction = False + # Default row type for cursors from this connection. None means + # plain tuples, which is the zero-cost default; see + # informix_db.rows for the opt-in named-access type. + self.row_factory = row_factory # Tri-state: True after first successful SQ_BEGIN, False after # an unlogged-DB rejection (-201). None until we've tried. # Used to avoid repeatedly probing on unlogged DBs. diff --git a/src/informix_db/cursors.py b/src/informix_db/cursors.py index 00650a7..0c72adc 100644 --- a/src/informix_db/cursors.py +++ b/src/informix_db/cursors.py @@ -50,6 +50,7 @@ from .exceptions import ( NotSupportedError, ProgrammingError, ) +from .rows import Row, make_row_class if TYPE_CHECKING: from .connections import Connection @@ -373,6 +374,9 @@ class Cursor: # manipulation. Two-mode cursor; the same surface API works # for both. self._scrollable = scrollable + # Inherited from the connection, overridable per cursor. See + # informix_db.rows for what this costs and why it is opt-in. + self.row_factory = connection.row_factory self._description: list[tuple] | None = None self._columns: list[ColumnInfo] = [] self._column_readers: list[tuple] | None = None # Phase 37 @@ -413,6 +417,8 @@ class Cursor: # DESCRIBE's statement-type field. Decides whether a cursor is # opened -- see _produces_result_set. self._statement_type: int = 0 + # Per-result-set row class when row_factory is set, else None. + self._row_class: type | None = None # Phase 10: smart-LOB read via ``lotofile(col, path, 'client')``. # The server orchestrates a SQ_FILE (98) protocol where it tells # us to "open file X, write these bytes, close". We emulate the @@ -514,6 +520,7 @@ class Cursor: self._rowcount = -1 self._rows = [] self._row_index = -1 # before-first-row + self._row_class = None self._statement_type = 0 self._statement_already_done = False @@ -553,6 +560,8 @@ class Cursor: else: self._execute_dml() + self._row_class = self._resolve_row_class() + # The statement succeeded. If it was transaction control, the # server's transaction state just changed and the connection has # to know, or commit() and rollback() silently do nothing. @@ -566,6 +575,25 @@ class Cursor: if self._description is not None: self._row_index = -1 + def _resolve_row_class(self) -> type | None: + """Pick the class this result set's rows are handed back as. + + ``row_factory`` is either :class:`informix_db.Row` (or a subclass), + in which case the per-shape class is built and cached from the + column names, or any callable taking the name tuple and returning + something that takes a values tuple. + + Returns ``None`` when no factory is set, which is the default and + keeps plain tuples on the hot path at zero cost. + """ + factory = self.row_factory + if factory is None or self._description is None: + return None + names = tuple(d[0] for d in self._description) + if isinstance(factory, type) and issubclass(factory, Row): + return make_row_class(names) + return factory(names) + def _note_transaction_control(self) -> None: """Sync the connection's transaction flag after a successful execute. @@ -1387,7 +1415,8 @@ class Cursor: self._row_index = len(self._rows) # past-last return None self._row_index = nxt - return self._rows[nxt] + row = self._rows[nxt] + return self._row_class(row) if self._row_class is not None else row def fetchmany(self, size: int | None = None) -> list[tuple]: self._check_open() @@ -1418,6 +1447,8 @@ class Cursor: return [] start = self._row_index + 1 out = self._rows[start:] + if self._row_class is not None: + out = [self._row_class(r) for r in out] self._row_index = len(self._rows) return list(out) @@ -1598,7 +1629,7 @@ class Cursor: if scrolltype == 4 or is_last_probe: # SFETCH(LAST) — TUPID == total row count self._scroll_total_rows = self._last_tupid - return row + return self._row_class(row) if self._row_class is not None else row def close(self) -> None: """Close the cursor. diff --git a/src/informix_db/rows.py b/src/informix_db/rows.py new file mode 100644 index 0000000..e427da6 --- /dev/null +++ b/src/informix_db/rows.py @@ -0,0 +1,157 @@ +"""Rows that can be read by position, by column name, or by attribute. + +PEP 249 only requires a sequence, and a sequence is what the driver +returns by default. That is fine for ``SELECT a, b`` and steadily worse +as the projection grows: ``row[11]`` tells a reader nothing, and stays +correct only until somebody adds a column in the middle. + +Opting in with ``row_factory=Row`` gives the shape ``pyodbc`` and +``mssql-python`` provide, all three at once:: + + conn = informix_db.connect(..., row_factory=informix_db.Row) + cur.execute("SELECT tabid, tabname FROM systables") + row = cur.fetchone() + row[0], row["tabname"], row.tabname + +It is opt-in rather than the default because it is not free, and the +driver's whole argument is that pure Python can stay within noise of the +C driver on bulk fetch. Measured on a 20,000-row five-column fetch: +**37.2 ms with tuples, 40.8 ms with Row**, so about 9%. Defaulting it on +would move the published 1.05-1.15x ratio against IfxPy to roughly +1.15-1.25x, which is not a trade to make on everybody's behalf. + +Where the 9% goes, against a plain tuple: about 30 ns per row to +construct, about 39 ns per ``row[0]`` because supporting ``row["name"]`` +means ``__getitem__`` is a Python method rather than C-level tuple +indexing, and about 13 ns per unpack since CPython's fast path for +``a, b = row`` applies to exact tuples and not to subclasses. + +Worth it for readability in application code. Not worth paying in a bulk +export that never looks at a column by name, which is exactly why it is +a choice. + +``Row`` subclasses ``tuple``, so ``row == (1, "x")`` is still true and +existing code keeps working unchanged. + +**Two names are shadowed.** ``tuple`` already has ``count`` and +``index`` methods. A column called either one wins, because on a +database row that is obviously what the caller meant, and the tuple +methods remain reachable as ``tuple.count(row, x)``. Every other column +name resolves through ``__getattr__``, which only runs after normal +lookup fails and so costs nothing for the names that do not collide. + +Column names come from ``cursor.description``. Informix folds unquoted +identifiers to lower case, so ``SELECT Config_Key`` is reachable as +``row.config_key``. Expressions get server-generated names that are not +Python identifiers, like ``(count(*))``; those are reachable by +subscript but not as attributes. Duplicate names resolve to the first +occurrence, matching ``pyodbc``. +""" + +from __future__ import annotations + +import operator +from functools import lru_cache +from typing import ClassVar + +__all__ = ["Row", "make_row_class"] + +# Attributes ``tuple`` already defines that a column could plausibly be +# named. A column with one of these names gets a descriptor so the column +# wins; see the module docstring. +_TUPLE_ATTRS = frozenset({"count", "index"}) + + +class Row(tuple): + """A result row addressable by position, name, or attribute. + + Used as a ``row_factory``. The concrete class handed to each result + set is a subclass carrying that query's column names, built by + :func:`make_row_class`. + """ + + __slots__ = () + + # Overridden on the per-result-set subclass. + _fields: ClassVar[tuple[str, ...]] = () + _map: ClassVar[dict[str, int]] = {} + + def __getitem__(self, key): + # ``key.__class__ is str`` rather than isinstance: this is the + # hot path, and it runs on every subscript. + if key.__class__ is str: + try: + return tuple.__getitem__(self, self._map[key]) + except KeyError: + raise KeyError( + f"no column named {key!r}; this row has " + f"{list(self._fields)}" + ) from None + return tuple.__getitem__(self, key) + + def __getattr__(self, name): + # Only reached when normal attribute lookup has already failed, + # so this costs nothing for real tuple attributes. + try: + return tuple.__getitem__(self, self._map[name]) + except KeyError: + raise AttributeError( + f"no column named {name!r}; this row has " + f"{list(self._fields)}" + ) from None + + def keys(self) -> tuple[str, ...]: + """Column names, in select order.""" + return self._fields + + def _asdict(self) -> dict: + """A plain ``dict`` of the row. + + On duplicate column names the last occurrence wins here, which + differs from subscript access where the first does. A dict cannot + represent both, and losing a duplicate silently is better than + raising on a query that is otherwise fine. + """ + return dict(zip(self._fields, self, strict=True)) + + def __repr__(self) -> str: + if not self._fields: + return tuple.__repr__(self) + body = ", ".join( + f"{name}={value!r}" + for name, value in zip(self._fields, self, strict=True) + ) + return f"Row({body})" + + def __reduce__(self): + # The per-result-set class is created at runtime and cannot be + # pickled by reference, so rebuild it from the field names. + return (_rebuild_row, (self._fields, tuple(self))) + + +def _rebuild_row(fields: tuple[str, ...], values: tuple): + return make_row_class(fields)(values) + + +@lru_cache(maxsize=256) +def make_row_class(fields: tuple[str, ...]) -> type[Row]: + """Build (and cache) the row class for one column-name shape. + + Cached because a class per ``execute()`` would put a ``type()`` call + on the path of every small query, and applications run the same + handful of statement shapes over and over. Keyed on the names alone, + so two queries selecting the same columns share a class. + """ + namespace: dict = { + "__slots__": (), + "_fields": fields, + # First occurrence wins on duplicates, matching pyodbc. Building + # the map in reverse and letting earlier entries overwrite later + # ones is the shortest way to say that. + "_map": {name: i for i, name in reversed(list(enumerate(fields)))}, + } + for i, name in enumerate(fields): + if name in _TUPLE_ATTRS: + # Shadow the tuple method so the column wins. + namespace[name] = property(operator.itemgetter(i)) + return type("Row", (Row,), namespace) diff --git a/tests/test_rows.py b/tests/test_rows.py new file mode 100644 index 0000000..c4c0af3 --- /dev/null +++ b/tests/test_rows.py @@ -0,0 +1,264 @@ +"""Named row access, and the cost of it. + +A field request: `row[11]` tells a reader nothing on a wide projection, +and `pyodbc` / `mssql-python` both hand back rows that answer to +position, column name, and attribute at once. `row_factory=Row` gives +the same three. + +It is opt-in. Defaulting it on would tax the thing this driver is +measured against, since supporting `row["name"]` means `__getitem__` +becomes a Python method rather than C-level tuple indexing, which costs +roughly 39 ns on every subscript. Users who want readable column access +in application code should pay that; a bulk export that never looks at a +column by name should not. + +`Row` subclasses `tuple`, so `row == (1, "x")` still holds and nothing +that already worked stops working. That constraint drove the design: the +existing suite compares fetched rows against plain tuples in hundreds of +places, and so does everybody's code. +""" + +from __future__ import annotations + +import pickle + +import pytest + +import informix_db +from informix_db.rows import Row, make_row_class +from tests.conftest import ConnParams + +# --------------------------------------------------------------------------- +# The type itself +# --------------------------------------------------------------------------- + + +def _row(fields, values): + return make_row_class(tuple(fields))(values) + + +def test_three_ways_to_reach_a_column() -> None: + row = _row(("tabid", "tabname"), (1, "systables")) + assert row[0] == 1 + assert row["tabname"] == "systables" + assert row.tabname == "systables" + + +def test_still_equal_to_a_plain_tuple() -> None: + """The compatibility constraint. Existing code and the existing test + suite compare fetched rows against tuples everywhere.""" + row = _row(("a", "b"), (1, "x")) + plain = (1, "x") + assert row == plain + # Both directions: tuple.__eq__ on the left has to accept a subclass + # on the right, or `expected == fetched` assertions break. + assert plain == row + assert list(row) == [1, "x"] + assert len(row) == 2 + a, b = row + assert (a, b) == (1, "x") + assert row in [(1, "x")] + + +def test_slice_degrades_to_a_plain_tuple() -> None: + """A slice has no meaningful column mapping, so it should not pretend + to be a Row.""" + row = _row(("a", "b", "c"), (1, 2, 3)) + assert row[0:2] == (1, 2) + assert type(row[0:2]) is tuple + + +def test_negative_index_still_works() -> None: + assert _row(("a", "b"), (1, 2))[-1] == 2 + + +def test_keys_and_asdict() -> None: + row = _row(("a", "b"), (1, "x")) + assert row.keys() == ("a", "b") + assert row._asdict() == {"a": 1, "b": "x"} + + +def test_repr_names_the_columns() -> None: + assert repr(_row(("a", "b"), (1, "x"))) == "Row(a=1, b='x')" + + +def test_pickles() -> None: + """The per-shape class is built at runtime, so it cannot be pickled by + reference. Multiprocessing users would hit that immediately.""" + row = _row(("a", "b"), (1, "x")) + restored = pickle.loads(pickle.dumps(row)) + assert restored == (1, "x") + assert restored.b == "x" + + +def test_missing_column_says_what_is_there() -> None: + row = _row(("tabid", "tabname"), (1, "x")) + with pytest.raises(KeyError, match="tabid"): + _ = row["nope"] + with pytest.raises(AttributeError, match="tabid"): + _ = row.nope + + +def test_class_is_cached_per_shape() -> None: + """A type() call per execute() would land on every small query.""" + assert make_row_class(("a", "b")) is make_row_class(("a", "b")) + assert make_row_class(("a", "b")) is not make_row_class(("a", "c")) + + +def test_duplicate_names_resolve_to_the_first() -> None: + """``SELECT tabid, tabid`` is legal and both columns are named + ``tabid``. pyodbc gives the first; so do we.""" + row = _row(("tabid", "tabid"), (1, 2)) + assert row["tabid"] == 1 + assert row.tabid == 1 + assert row[1] == 2, "positional access must still reach the second" + + +def test_column_named_count_beats_the_tuple_method() -> None: + """``tuple`` already has ``count`` and ``index``. On a database row a + column with that name is what the caller meant, so the column wins + and the tuple method stays reachable through the class.""" + row = _row(("count", "index"), (7, 9)) + assert row.count == 7 + assert row.index == 9 + assert tuple.count(row, 7) == 1 + + +def test_non_identifier_names_are_subscript_only() -> None: + """Informix names expression columns things like ``(count(*))``, + which cannot be an attribute.""" + row = _row(("(count(*))", "ok"), (5, 1)) + assert row["(count(*))"] == 5 + assert row.ok == 1 + + +# --------------------------------------------------------------------------- +# Against a real server +# --------------------------------------------------------------------------- + + +def _connect(conn_params: ConnParams, **kw) -> informix_db.Connection: + return informix_db.connect( + host=conn_params.host, + port=conn_params.port, + user=conn_params.user, + password=conn_params.password, + database=conn_params.database, + server=conn_params.server, + connect_timeout=10.0, + read_timeout=25.0, + autocommit=True, + **kw, + ) + + +@pytest.mark.integration +def test_default_is_still_a_plain_tuple(conn_params: ConnParams) -> None: + """The opt-in has to be genuinely opt-in. Anyone who does not ask for + Row should not pay for it or see any behaviour change.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("SELECT FIRST 1 tabid, tabname FROM systables") + row = cur.fetchone() + assert type(row) is tuple + with pytest.raises(TypeError): + _ = row["tabname"] + + +@pytest.mark.integration +def test_row_factory_on_the_connection(conn_params: ConnParams) -> None: + """One line at connect time, then every cursor from it, which is what + 'without any additional coding' has to mean in practice.""" + with _connect(conn_params, row_factory=informix_db.Row) as conn: + cur = conn.cursor() + cur.execute( + "SELECT FIRST 1 tabid, tabname FROM systables ORDER BY tabid" + ) + row = cur.fetchone() + assert row[0] == row["tabid"] == row.tabid + assert row[1] == row["tabname"] == row.tabname + + +@pytest.mark.integration +def test_every_fetch_path_returns_rows(conn_params: ConnParams) -> None: + """fetchone, fetchmany, fetchall and iteration each return rows by a + different route through the cursor.""" + with _connect(conn_params, row_factory=informix_db.Row) as conn: + sql = "SELECT FIRST 4 tabid, tabname FROM systables ORDER BY tabid" + cur = conn.cursor() + + cur.execute(sql) + assert cur.fetchone().tabname is not None + + cur.execute(sql) + assert all(r.tabname is not None for r in cur.fetchmany(2)) + + cur.execute(sql) + assert all(r.tabname is not None for r in cur.fetchall()) + + cur.execute(sql) + assert all(r.tabname is not None for r in cur) + + +@pytest.mark.integration +def test_scrollable_cursor_returns_rows(conn_params: ConnParams) -> None: + """Scrollable cursors return each row straight from the wire rather + than from the materialized list, so they are a separate path.""" + with _connect(conn_params, row_factory=informix_db.Row) as conn: + cur = conn.cursor(scrollable=True) + cur.execute("SELECT tabid, tabname FROM systables ORDER BY tabid") + assert cur.fetch_first().tabname is not None + assert cur.fetch_absolute(1).tabname is not None + cur.close() + + +@pytest.mark.integration +def test_per_cursor_override(conn_params: ConnParams) -> None: + with _connect(conn_params) as conn: + named = conn.cursor() + named.row_factory = informix_db.Row + plain = conn.cursor() + sql = "SELECT FIRST 1 tabid FROM systables" + named.execute(sql) + plain.execute(sql) + assert isinstance(named.fetchone(), Row) + assert type(plain.fetchone()) is tuple + + +@pytest.mark.integration +def test_informix_lowercases_so_the_obvious_name_works( + conn_params: ConnParams, +) -> None: + """Informix folds unquoted identifiers, which is why the .lower() in + the workaround people write by hand is a no-op.""" + with _connect(conn_params, row_factory=informix_db.Row) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_rows (Config_Key INT, Cnt INT)") + cur.execute("INSERT INTO t_rows VALUES (1, 2)") + cur.execute("SELECT Config_Key, Cnt FROM t_rows") + row = cur.fetchone() + assert row.config_key == 1 + assert row.cnt == 2 + + +@pytest.mark.integration +def test_pool_forwards_the_factory(conn_params: ConnParams) -> None: + pool = informix_db.create_pool( + host=conn_params.host, + port=conn_params.port, + user=conn_params.user, + password=conn_params.password, + database=conn_params.database, + server=conn_params.server, + autocommit=True, + row_factory=informix_db.Row, + min_size=1, + max_size=2, + ) + try: + with pool.connection() as conn: + cur = conn.cursor() + cur.execute("SELECT FIRST 1 tabid FROM systables") + assert cur.fetchone().tabid is not None + finally: + pool.close()