From 10863a9337b62a3d805da22ba032ece0499f505c Mon Sep 17 00:00:00 2001 From: Ryan Malloy Date: Mon, 4 May 2026 12:09:16 -0600 Subject: [PATCH] Phase 6.c: DATE / DATETIME / DECIMAL parameter encoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now you can pass Python datetime/date/Decimal values directly: cur.execute('INSERT INTO t VALUES (?, ?, ?)', (1, datetime.datetime(2026, 5, 4, 12, 34, 56), Decimal('1234.56'))) cur.execute('SELECT id FROM t WHERE d > ?', (datetime.date(2025, 1, 1),)) The 2-byte length-prefix discovery: both my Phase 6.a DECIMAL encoder and the new Phase 6.c DATETIME encoder produced "correct" BCD bytes but the server silently dropped the SQ_BIND PDU (no response, just timeout). Captured the wire, diffed against JDBC, and found that DECIMAL/DATETIME bind data has a 2-byte length PREFIX wrapping the BCD payload (per Decimal.javaToIfx line 457). With the prefix added, both encoders work. DATE doesn't need the prefix — it's a fixed 4-byte int. Per-type wire format: date → DATE(7), [4-byte BE int = days since 1899-12-31] datetime → DATETIME(10), [short total_len][byte 0xc7][7 BCD pairs] Decimal → DECIMAL(5), [short total_len][byte exp][BCD digit pairs] For DATETIME the encoder always emits YEAR TO SECOND form (no microseconds) — covers the common case. Phase 6.x can add YEAR TO FRACTION(N) variants if microsecond precision is needed. For DECIMAL the encoder uses the asymmetric base-100 complement (mirror of decoder) for negatives. Tested with positive, negative, and fractional values. Lesson for the protocol playbook: when the server silently drops a PDU, it's almost always an envelope/framing issue rather than the inner-value bytes being wrong. Same pattern as the SHORT-vs-INT reserved field in CURNAME+NFETCH and the even-byte alignment pad. Module changes: src/informix_db/converters.py: + _encode_date — 4-byte BE int day count + _encode_datetime — YEAR TO SECOND form with 2-byte length prefix + _encode_decimal — re-enabled (was Phase 6.x stub) with the same length-prefix fix + encode_param() dispatches on datetime.datetime BEFORE datetime.date (since datetime is a subclass of date in Python) Tests: 40 unit + 73 integration (3 new date/datetime param tests + 1 updated decimal param test) = 113 total, all green, ruff clean. New tests cover: - date as INSERT parameter via executemany — 3 dates round-trip - datetime as INSERT parameter via executemany — 3 timestamps - date as parameter in a WHERE clause filter (created_at > ?) - Decimal round trip (was: NotImplementedError check; now: real INSERT + SELECT verification) Type support matrix updates: DATE — encode ✓ + decode ✓ (was decode-only) DATETIME — encode ✓ + decode ✓ (was decode-only) DECIMAL — encode ✓ + decode ✓ (was decode-only) --- docs/CAPTURES/24-py-datetime-bind.socat.log | 44 +++++++++++++ docs/DECISION_LOG.md | 23 +++++++ src/informix_db/converters.py | 69 ++++++++++++++++++--- tests/test_datetime.py | 63 +++++++++++++++++++ tests/test_decimal.py | 13 ++-- 5 files changed, 197 insertions(+), 15 deletions(-) create mode 100644 docs/CAPTURES/24-py-datetime-bind.socat.log diff --git a/docs/CAPTURES/24-py-datetime-bind.socat.log b/docs/CAPTURES/24-py-datetime-bind.socat.log new file mode 100644 index 0000000..b2b1248 --- /dev/null +++ b/docs/CAPTURES/24-py-datetime-bind.socat.log @@ -0,0 +1,44 @@ +2026/05/04 12:06:47 socat[970697] N listening on AF=2 0.0.0.0:9090 +2026/05/04 12:06:47 socat[970697] N accepting connection from AF=2 127.0.0.1:57728 on AF=2 127.0.0.1:9090 +2026/05/04 12:06:47 socat[970697] N opening connection to 127.0.0.1:9088 +2026/05/04 12:06:47 socat[970697] N opening connection to AF=2 127.0.0.1:9088 +2026/05/04 12:06:47 socat[970697] N successfully connected from local address AF=2 127.0.0.1:39446 +2026/05/04 12:06:47 socat[970697] N successfully connected to 127.0.0.1:9088 +2026/05/04 12:06:47 socat[970697] N starting data transfer loop with FDs [6,6] and [5,5] +> 2026/05/04 12:06:47.995254 length=384 from=0 to=383 + 01 80 01 3c 00 00 00 64 00 65 00 00 00 3d 00 06 49 45 45 45 4d 00 00 6c 73 71 6c 65 78 65 63 00 00 00 00 00 00 06 39 2e 32 38 30 00 00 0c 52 44 53 23 52 30 30 30 30 30 30 00 00 05 73 71 6c 69 00 00 00 01 3c 00 00 00 00 00 00 00 00 00 01 00 09 69 6e 66 6f 72 6d 69 78 00 00 07 69 6e 34 6d 69 78 00 6f 6c 00 00 00 00 00 00 00 00 00 3d 74 6c 69 74 63 70 00 00 00 00 00 01 00 68 00 0b 00 00 00 03 00 09 69 6e 66 6f 72 6d 69 78 00 00 00 00 00 00 00 00 00 00 00 00 6a 00 06 00 07 44 42 50 41 54 48 00 00 02 2e 00 00 0e 43 4c 49 45 4e 54 5f 4c 4f 43 41 4c 45 00 00 0d 65 6e 5f 55 53 2e 38 38 35 39 2d 31 00 00 11 43 4c 4e 54 5f 50 41 4d 5f 43 41 50 41 42 4c 45 00 00 02 31 00 00 07 44 42 44 41 54 45 00 00 06 59 34 4d 44 2d 00 00 0c 49 46 58 5f 55 50 44 44 45 53 43 00 00 02 31 00 00 09 4e 4f 44 45 46 44 41 43 00 00 03 6e 6f 00 00 6b 00 00 00 00 00 0e cf fe 00 00 00 00 00 0b 72 70 6d 2d 62 75 6c 6c 65 74 00 00 00 00 29 2f 68 6f 6d 65 2f 72 70 6d 2f 63 6c 61 75 64 65 2f 69 6e 66 6f 72 6d 69 78 2f 70 79 74 68 6f 6e 2d 6c 69 62 72 61 72 79 00 00 74 00 20 00 00 00 00 00 00 00 00 00 16 69 6e 66 6f 72 6d 69 78 2d 64 62 40 70 69 64 39 37 30 37 35 30 00 00 7f +< 2026/05/04 12:06:47.996931 length=276 from=0 to=275 + 01 14 02 3c 10 00 00 64 00 65 00 00 00 3d 00 06 49 45 45 45 49 00 00 6c 73 72 76 69 6e 66 78 00 00 00 00 00 00 2f 49 42 4d 20 49 6e 66 6f 72 6d 69 78 20 44 79 6e 61 6d 69 63 20 53 65 72 76 65 72 20 56 65 72 73 69 6f 6e 20 31 35 2e 30 2e 31 2e 30 2e 33 00 00 07 73 65 72 69 61 6c 00 00 09 69 6e 66 6f 72 6d 69 78 00 00 00 01 3c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 6f 6e 00 00 00 00 00 00 00 00 00 3d 73 6f 63 74 63 70 00 00 00 00 00 00 00 66 00 00 00 00 00 00 00 00 00 00 00 14 00 00 00 6b 00 00 00 00 00 00 03 1a 00 00 00 00 00 0d 32 33 32 37 63 34 33 35 34 65 61 38 00 00 00 00 0f 2f 68 6f 6d 65 2f 69 6e 66 6f 72 6d 69 78 00 00 6e 00 04 00 00 00 00 00 74 00 33 00 00 00 c8 00 00 00 c8 00 29 2f 6f 70 74 2f 69 62 6d 2f 69 6e 66 6f 72 6d 69 78 2f 76 31 35 2e 30 2e 31 2e 30 2e 33 2f 62 69 6e 2f 6f 6e 69 6e 69 74 00 00 7f +> 2026/05/04 12:06:47.997147 length=14 from=384 to=397 + 00 7e 00 08 ff fc 7f fc 3c 8c aa 97 00 0c +< 2026/05/04 12:06:48.005577 length=16 from=276 to=291 + 00 7e 00 09 bd be 9f fe 7f b7 ff ef ff 00 00 0c +> 2026/05/04 12:06:48.005614 length=48 from=398 to=445 + 00 51 00 06 00 26 00 0c 00 04 00 06 44 42 54 45 4d 50 00 04 2f 74 6d 70 00 0b 53 55 42 51 43 41 43 48 45 53 5a 00 00 02 31 30 00 00 00 00 00 0c +< 2026/05/04 12:06:48.005687 length=2 from=292 to=293 + 00 0c +> 2026/05/04 12:06:48.005711 length=18 from=446 to=463 + 00 24 00 09 73 79 73 6d 61 73 74 65 72 00 00 00 00 0c +< 2026/05/04 12:06:48.005877 length=28 from=294 to=321 + 00 0f 00 15 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 12:06:48.006001 length=76 from=464 to=539 + 00 02 00 00 00 00 00 3e 43 52 45 41 54 45 20 54 45 4d 50 20 54 41 42 4c 45 20 74 64 74 20 28 69 64 20 49 4e 54 45 47 45 52 2c 20 74 73 20 44 41 54 45 54 49 4d 45 20 59 45 41 52 20 54 4f 20 53 45 43 4f 4e 44 29 00 16 00 31 00 0c +< 2026/05/04 12:06:48.006187 length=46 from=322 to=367 + 00 08 00 2d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 12:06:48.006288 length=8 from=540 to=547 + 00 04 00 00 00 07 00 0c +< 2026/05/04 12:06:48.011232 length=28 from=368 to=395 + 00 0f 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 12:06:48.011281 length=8 from=548 to=555 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 12:06:48.011322 length=2 from=396 to=397 + 00 0c +> 2026/05/04 12:06:48.011342 length=44 from=556 to=599 + 00 02 00 02 00 00 00 1d 49 4e 53 45 52 54 20 49 4e 54 4f 20 74 64 74 20 56 41 4c 55 45 53 20 28 3f 2c 20 3f 29 00 00 16 00 31 00 0c +< 2026/05/04 12:06:48.011438 length=132 from=398 to=529 + 00 08 00 06 00 00 00 00 00 00 00 0c 00 02 00 00 00 06 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 03 00 00 00 04 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0e 0a 69 64 00 74 73 00 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 12:06:48.011576 length=36 from=600 to=635 + 00 04 00 00 00 05 00 02 00 02 00 00 0a 00 00 00 00 01 00 0a 00 00 0e 0a c7 14 1a 05 04 0c 22 38 00 07 00 0c +2026/05/04 12:06:50 socat[970697] N socket 1 (fd 6) is at EOF +2026/05/04 12:06:50 socat[970697] N socket 2 (fd 5) is at EOF +2026/05/04 12:06:50 socat[970697] N exiting with status 0 diff --git a/docs/DECISION_LOG.md b/docs/DECISION_LOG.md index 0e73ebf..130f787 100644 --- a/docs/DECISION_LOG.md +++ b/docs/DECISION_LOG.md @@ -403,6 +403,29 @@ DATETIME parameter binding (encoder) is Phase 6.x — same status as DECIMAL enc --- +## 2026-05-04 — DATE / DATETIME / DECIMAL parameter encoding + +**Status**: active +**Decision**: ``encode_param`` dispatches on ``isinstance(value, datetime.datetime / datetime.date / decimal.Decimal)`` to type-specific encoders. Round-trip verified through INSERT + SELECT. + +**The 2-byte length-prefix discovery (the unblocker)**: my Phase 6.a DECIMAL encoder and Phase 6.c DATETIME encoder both produced "correct" BCD bytes but the server silently dropped the SQ_BIND PDU. Captured the wire and compared to JDBC — DECIMAL/DATETIME bind data has a **2-byte length prefix** at the start (per ``Decimal.javaToIfx`` line 457) that wraps the BCD payload. With the prefix added (``raw = len(inner).to_bytes(2, "big") + inner``), both encoders work. DATE doesn't need the prefix — it's a fixed 4-byte int. + +Per-type encoded format: + +| Python | IDS type | Wire bytes | +|--------|----------|------------| +| ``datetime.date`` | DATE (7) | ``[int days_since_1899-12-31]`` (4 bytes BE) | +| ``datetime.datetime`` | DATETIME (10) | ``[short total_len][byte 0xc7][7 BCD pairs]`` (10 bytes total for YEAR TO SECOND) | +| ``decimal.Decimal`` | DECIMAL (5) | ``[short total_len][byte exp][BCD digit pairs]`` (variable) | + +For DATETIME, encoder always emits YEAR TO SECOND form (no microseconds). Phase 6.x can add YEAR TO FRACTION(N) variants if microsecond precision is needed. + +For DECIMAL, the encoder uses the asymmetric base-100 complement (mirror of decoder) for negatives. Tested with positive, negative, fraction values. + +**Lesson**: when a server silently drops a PDU, it's almost always an envelope/framing issue rather than the inner-value bytes being wrong. The 2-byte length prefix here, the SHORT-vs-INT reserved field in CURNAME+NFETCH, the even-byte alignment pad — same pattern. + +--- + ## (template — copy below this line for new entries) ``` diff --git a/src/informix_db/converters.py b/src/informix_db/converters.py index 78bd696..14c0f45 100644 --- a/src/informix_db/converters.py +++ b/src/informix_db/converters.py @@ -358,6 +358,52 @@ def _encode_bool(value: bool) -> EncodedParam: return (45, 0, b"\x01" if value else b"\x00") +def _encode_date(value: datetime.date) -> EncodedParam: + """Encode a Python ``datetime.date`` as Informix DATE (type=7). + + Format: 4-byte big-endian signed int = day count from 1899-12-31. + Inverse of ``_decode_date``. Per ``JavaToIfxDate`` (line 135). + """ + days = (value - _INFORMIX_DATE_EPOCH).days + return (7, 0, days.to_bytes(4, "big", signed=True)) + + +def _encode_datetime(value: datetime.datetime) -> EncodedParam: + """Encode a Python ``datetime.datetime`` as Informix DATETIME (type=10). + + Emit YEAR TO SECOND form — covers the common case of stored + timestamps without microseconds. (Phase 6.x can add YEAR TO + FRACTION(N) variants if microseconds are needed.) + + Format (per ``Decimal.javaToIfx`` line 457): + byte[0..1] = short total length of data following (= digit_count/2 + 1) + byte[2] = sign + biased exponent (0xc7 = positive, 7 base-100 pairs) + byte[3..9] = BCD digit pairs in YYYY MM DD HH MI SS order (7 bytes) + + The 2-byte length PREFIX is what trips you up — it's part of the + data the SQ_BIND emits via writePadded, distinct from the prec + short before it. Without the prefix, the server silently times out. + + Precision short = qualifier packed as + ``(digit_count << 8) | (start_TU << 4) | end_TU``. + For YEAR TO SECOND: digit_count=14, start=0 (YEAR), end=10 (SECOND). + """ + fields = [ + (value.year, 4), + (value.month, 2), + (value.day, 2), + (value.hour, 2), + (value.minute, 2), + (value.second, 2), + ] + digit_str = "".join(f"{v:0{w}d}" for v, w in fields) # 14 digits + digit_bytes = bytes(int(digit_str[i : i + 2]) for i in range(0, len(digit_str), 2)) + inner = bytes([0xC7]) + digit_bytes # 8 bytes (1 exp + 7 BCD pairs) + raw = len(inner).to_bytes(2, "big") + inner # +2 byte length prefix = 10 bytes + prec = (14 << 8) | (0 << 4) | 10 + return (10, prec, raw) + + def _encode_decimal(value: decimal.Decimal) -> EncodedParam: """Encode a Python ``decimal.Decimal`` as IDS DECIMAL (type=5). @@ -411,7 +457,11 @@ def _encode_decimal(value: decimal.Decimal) -> EncodedParam: head_byte = (biased_exp & 0x7F) ^ 0x7F out_digits = bytes(complemented) - raw = bytes([head_byte]) + out_digits + inner = bytes([head_byte]) + out_digits + # Wire format wants a 2-byte length prefix (per Decimal.javaToIfx + # line 457-461) — this is the missing piece that made my Phase 6.a + # encoder time out on the server side. Same fix as DATETIME. + raw = len(inner).to_bytes(2, "big") + inner # Precision short for DECIMAL: packed (precision << 8) | scale # Precision = total significant digits, scale = digits after point. precision = max(n_digits, 1) @@ -439,18 +489,17 @@ def encode_param(value: object) -> EncodedParam: return _encode_float(value) if isinstance(value, str): return _encode_str(value) + # NB: datetime.datetime is a subclass of datetime.date — must check + # datetime BEFORE date. + if isinstance(value, datetime.datetime): + return _encode_datetime(value) + if isinstance(value, datetime.date): + return _encode_date(value) if isinstance(value, decimal.Decimal): - # _encode_decimal is implemented but the server rejects the - # bytes (precision packing wrong somewhere) — kept as a - # Phase 6.x starting point but disabled for now. Workaround: - # cast Decimal to float at the call site if you need to bind. - raise NotImplementedError( - "Decimal parameter binding is Phase 6.x; convert to float " - "or pass DECIMAL via SQL literal for now" - ) + return _encode_decimal(value) raise NotImplementedError( f"parameter binding for {type(value).__name__} not yet supported " - f"(Phase 4 MVP: int, float, str, bool, None)" + f"(Phase 4: int, float, str, bool, None, date, datetime)" ) diff --git a/tests/test_datetime.py b/tests/test_datetime.py index 5355d30..18101fa 100644 --- a/tests/test_datetime.py +++ b/tests/test_datetime.py @@ -121,3 +121,66 @@ def test_datetime_null(conn_params: ConnParams) -> None: cur.execute("INSERT INTO t_dt2 VALUES (NULL)") cur.execute("SELECT ts FROM t_dt2") assert cur.fetchone() == (None,) + + +def test_date_param_round_trip(conn_params: ConnParams) -> None: + """``datetime.date`` as a bind parameter round-trips through INSERT + SELECT.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_d (id INTEGER, d DATE)") + cur.executemany( + "INSERT INTO t_d VALUES (?, ?)", + [ + (1, datetime.date(2026, 5, 4)), + (2, datetime.date(1999, 12, 31)), + (3, datetime.date(1900, 1, 1)), + ], + ) + cur.execute("SELECT id, d FROM t_d ORDER BY id") + assert cur.fetchall() == [ + (1, datetime.date(2026, 5, 4)), + (2, datetime.date(1999, 12, 31)), + (3, datetime.date(1900, 1, 1)), + ] + + +def test_datetime_param_round_trip(conn_params: ConnParams) -> None: + """``datetime.datetime`` as a bind parameter round-trips.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_dt3 (id INTEGER, ts DATETIME YEAR TO SECOND)") + cur.executemany( + "INSERT INTO t_dt3 VALUES (?, ?)", + [ + (1, datetime.datetime(2026, 5, 4, 12, 34, 56)), + (2, datetime.datetime(2000, 1, 1, 0, 0, 0)), + (3, datetime.datetime(2023, 7, 15, 18, 30, 45)), + ], + ) + cur.execute("SELECT id, ts FROM t_dt3 ORDER BY id") + rows = cur.fetchall() + assert rows == [ + (1, datetime.datetime(2026, 5, 4, 12, 34, 56)), + (2, datetime.datetime(2000, 1, 1, 0, 0, 0)), + (3, datetime.datetime(2023, 7, 15, 18, 30, 45)), + ] + + +def test_date_in_where_clause(conn_params: ConnParams) -> None: + """Use a Python ``date`` as a parameter in a WHERE clause.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_w (id INTEGER, d DATE)") + cur.executemany( + "INSERT INTO t_w VALUES (?, ?)", + [ + (1, datetime.date(2024, 1, 1)), + (2, datetime.date(2025, 6, 15)), + (3, datetime.date(2026, 12, 31)), + ], + ) + cur.execute( + "SELECT id FROM t_w WHERE d > ? ORDER BY id", + (datetime.date(2025, 1, 1),), + ) + assert cur.fetchall() == [(2,), (3,)] diff --git a/tests/test_decimal.py b/tests/test_decimal.py index ac6c43c..9c1fea0 100644 --- a/tests/test_decimal.py +++ b/tests/test_decimal.py @@ -92,10 +92,13 @@ def test_decimal_null(conn_params: ConnParams) -> None: assert cur.fetchone() == (None,) -def test_decimal_param_binding_raises_for_now(conn_params: ConnParams) -> None: - """Decimal as a bind parameter is Phase 6.x — currently raises.""" +def test_decimal_param_binding_round_trip(conn_params: ConnParams) -> None: + """Decimal as a bind parameter round-trips through INSERT + SELECT.""" with _connect(conn_params) as conn: cur = conn.cursor() - cur.execute("CREATE TEMP TABLE td2 (n DECIMAL(10,2))") - with pytest.raises(NotImplementedError, match="Decimal"): - cur.execute("INSERT INTO td2 VALUES (?)", (Decimal("3.14"),)) + cur.execute("CREATE TEMP TABLE td2 (id INTEGER, n DECIMAL(12,2))") + for i, d in enumerate([Decimal("1234.56"), Decimal("-99.99"), Decimal("0.5")]): + cur.execute("INSERT INTO td2 VALUES (?, ?)", (i, d)) + cur.execute("SELECT n FROM td2 ORDER BY id") + rows = cur.fetchall() + assert rows == [(Decimal("1234.56"),), (Decimal("-99.99"),), (Decimal("0.5"),)]