From d508a489fd1a863b08a49664999b2af2b7db4008 Mon Sep 17 00:00:00 2001 From: Ryan Malloy Date: Mon, 4 May 2026 11:11:50 -0600 Subject: [PATCH] Phase 4.x: parameterized SELECT, NULL row decoding, executemany() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three Phase 4 follow-ups in one push, all with empirical wire analysis: 1. PARAMETERIZED SELECT cur.execute('SELECT tabname FROM systables WHERE tabid = ?', (1,)) → ('systables',) Wire flow: PREPARE → DESCRIBE → SQ_BIND-only (no EXECUTE) → CURNAME+NFETCH → TUPLE+DONE → drain → CLOSE+RELEASE. The cursor open is what executes the prepared query; SQ_BIND just binds values into scope. No need for the IDESCRIBE handshake JDBC does for type discovery — server accepts our typed bind directly. 2. NULL ROW DECODING — per-type sentinel detection Each IDS type has its own NULL sentinel in tuple data: INT → 0x80000000 (INT_MIN) BIGINT → 0x8000000000000000 (LONG_MIN) SMALLINT→ 0x8000 (SHORT_MIN) REAL → all 0xFF (NaN bit pattern) FLOAT → all 0xFF DATE → 0x80000000 (same as INT) VARCHAR → [byte 1][byte 0] (length=1, single nul) — distinguishable from empty '' which is [byte 0] (length=0) Verified by wire capture against the dev container — see docs/CAPTURES/19-py-null-vs-onechar.socat.log and docs/CAPTURES/20-py-int-null.socat.log. The VARCHAR null marker is the trickiest because it LOOKS like a 1-byte string of nul, but VARCHAR can't contain embedded nuls anyway, so the byte-0 within length-1 is unambiguous. 3. executemany(sql, seq_of_params) — PEP 249 batched DML PREPARE once, loop SQ_BIND+SQ_EXECUTE per param set, RELEASE once. Performance: only ~1.06x faster than execute() loop for 200 INSERTs (dominated by per-row round trips). Phase 4.x optimization opportunity: chain BIND+EXECUTE in one PDU without intermediate flush+read for true bulk performance (would likely give 5-10x). Documented in DECISION_LOG.md as a follow-up. Module changes: src/informix_db/converters.py: + Per-type NULL sentinel constants and detection in each decoder + Decoders now return None for sentinel values src/informix_db/cursors.py: + _execute_select_with_params() — SQ_BIND alone, then cursor open + _build_bind_only_pdu() — SQ_BIND without trailing SQ_EXECUTE + executemany() — loop BIND+EXECUTE, accumulate rowcount + execute() now dispatches to _execute_select_with_params for parameterized SELECT (was: NotSupportedError) Tests: 40 unit + 47 integration (was 32; added 15 new) = 87 total, all green, ruff clean. New test files / cases: tests/test_nulls.py (7) — NULL decoding for INT, BIGINT, FLOAT, REAL, VARCHAR, empty-vs-null, mixed columns tests/test_params.py — added 4 parameterized SELECT tests, 5 executemany tests tests/test_smoke.py — updated cursor-with-params test (was Phase 1 "raises", now Phase 4 "works") Discovered captures kept for next-session debugging: docs/CAPTURES/18-py-null-rows.socat.log docs/CAPTURES/19-py-null-vs-onechar.socat.log docs/CAPTURES/20-py-int-null.socat.log --- docs/CAPTURES/18-py-null-rows.socat.log | 86 ++++++++++++++ docs/CAPTURES/19-py-null-vs-onechar.socat.log | 98 ++++++++++++++++ docs/CAPTURES/20-py-int-null.socat.log | 86 ++++++++++++++ docs/DECISION_LOG.md | 53 +++++++++ src/informix_db/converters.py | 49 ++++++-- src/informix_db/cursors.py | 107 +++++++++++++++++- tests/test_nulls.py | 102 +++++++++++++++++ tests/test_params.py | 97 +++++++++++++++- 8 files changed, 657 insertions(+), 21 deletions(-) create mode 100644 docs/CAPTURES/18-py-null-rows.socat.log create mode 100644 docs/CAPTURES/19-py-null-vs-onechar.socat.log create mode 100644 docs/CAPTURES/20-py-int-null.socat.log create mode 100644 tests/test_nulls.py diff --git a/docs/CAPTURES/18-py-null-rows.socat.log b/docs/CAPTURES/18-py-null-rows.socat.log new file mode 100644 index 0000000..3ccaff9 --- /dev/null +++ b/docs/CAPTURES/18-py-null-rows.socat.log @@ -0,0 +1,86 @@ +2026/05/04 11:06:27 socat[816537] N listening on AF=2 0.0.0.0:9090 +2026/05/04 11:06:28 socat[816537] N accepting connection from AF=2 127.0.0.1:53772 on AF=2 127.0.0.1:9090 +2026/05/04 11:06:28 socat[816537] N opening connection to 127.0.0.1:9088 +2026/05/04 11:06:28 socat[816537] N opening connection to AF=2 127.0.0.1:9088 +2026/05/04 11:06:28 socat[816537] N successfully connected from local address AF=2 127.0.0.1:45370 +2026/05/04 11:06:28 socat[816537] N successfully connected to 127.0.0.1:9088 +2026/05/04 11:06:28 socat[816537] N starting data transfer loop with FDs [6,6] and [5,5] +> 2026/05/04 11:06:28.233803 length=384 from=0 to=383 + 01 80 01 3c 00 00 00 64 00 65 00 00 00 3d 00 06 49 45 45 45 4d 00 00 6c 73 71 6c 65 78 65 63 00 00 00 00 00 00 06 39 2e 32 38 30 00 00 0c 52 44 53 23 52 30 30 30 30 30 30 00 00 05 73 71 6c 69 00 00 00 01 3c 00 00 00 00 00 00 00 00 00 01 00 09 69 6e 66 6f 72 6d 69 78 00 00 07 69 6e 34 6d 69 78 00 6f 6c 00 00 00 00 00 00 00 00 00 3d 74 6c 69 74 63 70 00 00 00 00 00 01 00 68 00 0b 00 00 00 03 00 09 69 6e 66 6f 72 6d 69 78 00 00 00 00 00 00 00 00 00 00 00 00 6a 00 06 00 07 44 42 50 41 54 48 00 00 02 2e 00 00 0e 43 4c 49 45 4e 54 5f 4c 4f 43 41 4c 45 00 00 0d 65 6e 5f 55 53 2e 38 38 35 39 2d 31 00 00 11 43 4c 4e 54 5f 50 41 4d 5f 43 41 50 41 42 4c 45 00 00 02 31 00 00 07 44 42 44 41 54 45 00 00 06 59 34 4d 44 2d 00 00 0c 49 46 58 5f 55 50 44 44 45 53 43 00 00 02 31 00 00 09 4e 4f 44 45 46 44 41 43 00 00 03 6e 6f 00 00 6b 00 00 00 00 00 0c 75 ae 00 00 00 00 00 0b 72 70 6d 2d 62 75 6c 6c 65 74 00 00 00 00 29 2f 68 6f 6d 65 2f 72 70 6d 2f 63 6c 61 75 64 65 2f 69 6e 66 6f 72 6d 69 78 2f 70 79 74 68 6f 6e 2d 6c 69 62 72 61 72 79 00 00 74 00 20 00 00 00 00 00 00 00 00 00 16 69 6e 66 6f 72 6d 69 78 2d 64 62 40 70 69 64 38 31 36 35 35 38 00 00 7f +< 2026/05/04 11:06:28.245386 length=276 from=0 to=275 + 01 14 02 3c 10 00 00 64 00 65 00 00 00 3d 00 06 49 45 45 45 49 00 00 6c 73 72 76 69 6e 66 78 00 00 00 00 00 00 2f 49 42 4d 20 49 6e 66 6f 72 6d 69 78 20 44 79 6e 61 6d 69 63 20 53 65 72 76 65 72 20 56 65 72 73 69 6f 6e 20 31 35 2e 30 2e 31 2e 30 2e 33 00 00 07 73 65 72 69 61 6c 00 00 09 69 6e 66 6f 72 6d 69 78 00 00 00 01 3c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 6f 6e 00 00 00 00 00 00 00 00 00 3d 73 6f 63 74 63 70 00 00 00 00 00 00 00 66 00 00 00 00 00 00 00 00 00 00 00 14 00 00 00 6b 00 00 00 00 00 00 03 1a 00 00 00 00 00 0d 32 33 32 37 63 34 33 35 34 65 61 38 00 00 00 00 0f 2f 68 6f 6d 65 2f 69 6e 66 6f 72 6d 69 78 00 00 6e 00 04 00 00 00 00 00 74 00 33 00 00 00 c8 00 00 00 c8 00 29 2f 6f 70 74 2f 69 62 6d 2f 69 6e 66 6f 72 6d 69 78 2f 76 31 35 2e 30 2e 31 2e 30 2e 33 2f 62 69 6e 2f 6f 6e 69 6e 69 74 00 00 7f +> 2026/05/04 11:06:28.245659 length=14 from=384 to=397 + 00 7e 00 08 ff fc 7f fc 3c 8c aa 97 00 0c +< 2026/05/04 11:06:28.245700 length=16 from=276 to=291 + 00 7e 00 09 bd be 9f fe 7f b7 ff ef ff 00 00 0c +> 2026/05/04 11:06:28.245728 length=48 from=398 to=445 + 00 51 00 06 00 26 00 0c 00 04 00 06 44 42 54 45 4d 50 00 04 2f 74 6d 70 00 0b 53 55 42 51 43 41 43 48 45 53 5a 00 00 02 31 30 00 00 00 00 00 0c +< 2026/05/04 11:06:28.245788 length=2 from=292 to=293 + 00 0c +> 2026/05/04 11:06:28.245804 length=18 from=446 to=463 + 00 24 00 09 73 79 73 6d 61 73 74 65 72 00 00 00 00 0c +< 2026/05/04 11:06:28.245963 length=28 from=294 to=321 + 00 0f 00 15 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:06:28.246010 length=76 from=464 to=539 + 00 02 00 00 00 00 00 3d 43 52 45 41 54 45 20 54 45 4d 50 20 54 41 42 4c 45 20 74 20 28 69 64 20 49 4e 54 45 47 45 52 2c 20 6e 61 6d 65 20 56 41 52 43 48 41 52 28 35 30 29 2c 20 76 61 6c 20 46 4c 4f 41 54 29 00 00 16 00 31 00 0c +< 2026/05/04 11:06:28.246124 length=46 from=322 to=367 + 00 08 00 2d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:06:28.246180 length=8 from=540 to=547 + 00 04 00 00 00 07 00 0c +< 2026/05/04 11:06:28.251154 length=28 from=368 to=395 + 00 0f 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:06:28.251235 length=8 from=548 to=555 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 11:06:28.251280 length=2 from=396 to=397 + 00 0c +> 2026/05/04 11:06:28.251306 length=44 from=556 to=599 + 00 02 00 03 00 00 00 1e 49 4e 53 45 52 54 20 49 4e 54 4f 20 74 20 56 41 4c 55 45 53 20 28 3f 2c 20 3f 2c 20 3f 29 00 16 00 31 00 0c +< 2026/05/04 11:06:28.251407 length=168 from=398 to=565 + 00 08 00 06 00 00 00 00 00 00 00 3f 00 03 00 00 00 0c 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 03 00 00 00 04 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 32 00 00 00 08 00 00 00 37 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 69 64 00 6e 61 6d 65 00 76 61 6c 00 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:06:28.251591 length=50 from=600 to=649 + 00 04 00 00 00 05 00 03 00 02 00 00 0a 00 00 00 00 01 00 00 00 00 00 00 00 05 68 65 6c 6c 6f 00 00 03 00 00 00 00 40 09 1e b8 51 eb 85 1f 00 07 00 0c +< 2026/05/04 11:06:28.253369 length=48 from=566 to=613 + 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 01 00 00 01 01 00 00 00 00 00 37 00 00 00 01 00 00 00 02 00 0c +> 2026/05/04 11:06:28.253424 length=8 from=650 to=657 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 11:06:28.253462 length=2 from=614 to=615 + 00 0c +> 2026/05/04 11:06:28.253485 length=44 from=658 to=701 + 00 02 00 03 00 00 00 1e 49 4e 53 45 52 54 20 49 4e 54 4f 20 74 20 56 41 4c 55 45 53 20 28 3f 2c 20 3f 2c 20 3f 29 00 16 00 31 00 0c +< 2026/05/04 11:06:28.253564 length=168 from=616 to=783 + 00 08 00 06 00 00 00 00 00 00 00 3f 00 03 00 00 00 0c 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 03 00 00 00 04 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 32 00 00 00 08 00 00 00 37 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 69 64 00 6e 61 6d 65 00 76 61 6c 00 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:06:28.253710 length=34 from=702 to=735 + 00 04 00 00 00 05 00 03 00 02 00 00 0a 00 00 00 00 02 00 00 ff ff 00 00 00 00 ff ff 00 00 00 07 00 0c +< 2026/05/04 11:06:28.255614 length=48 from=784 to=831 + 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 01 00 00 01 02 00 00 00 00 00 37 00 00 00 01 00 00 00 02 00 0c +> 2026/05/04 11:06:28.255676 length=8 from=736 to=743 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 11:06:28.255715 length=2 from=832 to=833 + 00 0c +> 2026/05/04 11:06:28.255734 length=54 from=744 to=797 + 00 02 00 00 00 00 00 27 53 45 4c 45 43 54 20 69 64 2c 20 6e 61 6d 65 2c 20 76 61 6c 20 46 52 4f 4d 20 74 20 4f 52 44 45 52 20 42 59 20 69 64 00 00 16 00 31 00 0c +< 2026/05/04 11:06:28.255890 length=148 from=834 to=981 + 00 08 00 02 00 00 00 00 00 00 00 3f 00 03 00 00 00 0c 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 03 00 00 00 04 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 32 00 00 00 08 00 00 00 37 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 69 64 00 6e 61 6d 65 00 76 61 6c 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 02 00 00 00 02 00 0c +> 2026/05/04 11:06:28.256035 length=42 from=798 to=839 + 00 04 00 00 00 03 00 12 5f 69 66 78 63 30 30 30 30 30 30 30 30 30 30 30 30 31 00 06 00 04 00 00 00 09 00 00 10 00 00 00 00 0c +< 2026/05/04 11:06:28.256137 length=76 from=982 to=1057 + 00 0e 00 00 00 00 00 12 00 00 00 01 05 68 65 6c 6c 6f 40 09 1e b8 51 eb 85 1f 00 0e 00 00 00 00 00 0e 00 00 00 02 01 00 ff ff ff ff ff ff ff ff 00 0f 00 00 00 00 00 02 00 00 01 02 00 00 00 00 00 37 00 00 00 02 00 00 00 02 00 0c +> 2026/05/04 11:06:28.256224 length=14 from=840 to=853 + 00 04 00 00 00 09 00 00 10 00 00 00 00 0c +< 2026/05/04 11:06:28.256252 length=28 from=1058 to=1085 + 00 0f 00 00 00 00 00 02 00 00 01 02 00 00 00 00 00 37 00 00 00 02 00 00 00 02 00 0c +> 2026/05/04 11:06:28.256286 length=8 from=854 to=861 + 00 04 00 00 00 0a 00 0c +< 2026/05/04 11:06:28.256312 length=2 from=1086 to=1087 + 00 0c +> 2026/05/04 11:06:28.256325 length=8 from=862 to=869 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 11:06:28.256351 length=2 from=1088 to=1089 + 00 0c +> 2026/05/04 11:06:28.256386 length=2 from=870 to=871 + 00 38 +< 2026/05/04 11:06:28.259522 length=2 from=1090 to=1091 + 00 38 +2026/05/04 11:06:28 socat[816537] N socket 1 (fd 6) is at EOF +2026/05/04 11:06:28 socat[816537] N socket 2 (fd 5) is at EOF +2026/05/04 11:06:28 socat[816537] N exiting with status 0 diff --git a/docs/CAPTURES/19-py-null-vs-onechar.socat.log b/docs/CAPTURES/19-py-null-vs-onechar.socat.log new file mode 100644 index 0000000..08349e4 --- /dev/null +++ b/docs/CAPTURES/19-py-null-vs-onechar.socat.log @@ -0,0 +1,98 @@ +2026/05/04 11:07:56 socat[837404] N listening on AF=2 0.0.0.0:9090 +2026/05/04 11:07:57 socat[837404] N accepting connection from AF=2 127.0.0.1:52758 on AF=2 127.0.0.1:9090 +2026/05/04 11:07:57 socat[837404] N opening connection to 127.0.0.1:9088 +2026/05/04 11:07:57 socat[837404] N opening connection to AF=2 127.0.0.1:9088 +2026/05/04 11:07:57 socat[837404] N successfully connected from local address AF=2 127.0.0.1:60228 +2026/05/04 11:07:57 socat[837404] N successfully connected to 127.0.0.1:9088 +2026/05/04 11:07:57 socat[837404] N starting data transfer loop with FDs [6,6] and [5,5] +> 2026/05/04 11:07:57.318852 length=384 from=0 to=383 + 01 80 01 3c 00 00 00 64 00 65 00 00 00 3d 00 06 49 45 45 45 4d 00 00 6c 73 71 6c 65 78 65 63 00 00 00 00 00 00 06 39 2e 32 38 30 00 00 0c 52 44 53 23 52 30 30 30 30 30 30 00 00 05 73 71 6c 69 00 00 00 01 3c 00 00 00 00 00 00 00 00 00 01 00 09 69 6e 66 6f 72 6d 69 78 00 00 07 69 6e 34 6d 69 78 00 6f 6c 00 00 00 00 00 00 00 00 00 3d 74 6c 69 74 63 70 00 00 00 00 00 01 00 68 00 0b 00 00 00 03 00 09 69 6e 66 6f 72 6d 69 78 00 00 00 00 00 00 00 00 00 00 00 00 6a 00 06 00 07 44 42 50 41 54 48 00 00 02 2e 00 00 0e 43 4c 49 45 4e 54 5f 4c 4f 43 41 4c 45 00 00 0d 65 6e 5f 55 53 2e 38 38 35 39 2d 31 00 00 11 43 4c 4e 54 5f 50 41 4d 5f 43 41 50 41 42 4c 45 00 00 02 31 00 00 07 44 42 44 41 54 45 00 00 06 59 34 4d 44 2d 00 00 0c 49 46 58 5f 55 50 44 44 45 53 43 00 00 02 31 00 00 09 4e 4f 44 45 46 44 41 43 00 00 03 6e 6f 00 00 6b 00 00 00 00 00 0c c7 30 00 00 00 00 00 0b 72 70 6d 2d 62 75 6c 6c 65 74 00 00 00 00 29 2f 68 6f 6d 65 2f 72 70 6d 2f 63 6c 61 75 64 65 2f 69 6e 66 6f 72 6d 69 78 2f 70 79 74 68 6f 6e 2d 6c 69 62 72 61 72 79 00 00 74 00 20 00 00 00 00 00 00 00 00 00 16 69 6e 66 6f 72 6d 69 78 2d 64 62 40 70 69 64 38 33 37 34 32 34 00 00 7f +< 2026/05/04 11:07:57.322619 length=276 from=0 to=275 + 01 14 02 3c 10 00 00 64 00 65 00 00 00 3d 00 06 49 45 45 45 49 00 00 6c 73 72 76 69 6e 66 78 00 00 00 00 00 00 2f 49 42 4d 20 49 6e 66 6f 72 6d 69 78 20 44 79 6e 61 6d 69 63 20 53 65 72 76 65 72 20 56 65 72 73 69 6f 6e 20 31 35 2e 30 2e 31 2e 30 2e 33 00 00 07 73 65 72 69 61 6c 00 00 09 69 6e 66 6f 72 6d 69 78 00 00 00 01 3c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 6f 6e 00 00 00 00 00 00 00 00 00 3d 73 6f 63 74 63 70 00 00 00 00 00 00 00 66 00 00 00 00 00 00 00 00 00 00 00 14 00 00 00 6b 00 00 00 00 00 00 03 1a 00 00 00 00 00 0d 32 33 32 37 63 34 33 35 34 65 61 38 00 00 00 00 0f 2f 68 6f 6d 65 2f 69 6e 66 6f 72 6d 69 78 00 00 6e 00 04 00 00 00 00 00 74 00 33 00 00 00 c8 00 00 00 c8 00 29 2f 6f 70 74 2f 69 62 6d 2f 69 6e 66 6f 72 6d 69 78 2f 76 31 35 2e 30 2e 31 2e 30 2e 33 2f 62 69 6e 2f 6f 6e 69 6e 69 74 00 00 7f +> 2026/05/04 11:07:57.322854 length=14 from=384 to=397 + 00 7e 00 08 ff fc 7f fc 3c 8c aa 97 00 0c +< 2026/05/04 11:07:57.329158 length=16 from=276 to=291 + 00 7e 00 09 bd be 9f fe 7f b7 ff ef ff 00 00 0c +> 2026/05/04 11:07:57.329205 length=48 from=398 to=445 + 00 51 00 06 00 26 00 0c 00 04 00 06 44 42 54 45 4d 50 00 04 2f 74 6d 70 00 0b 53 55 42 51 43 41 43 48 45 53 5a 00 00 02 31 30 00 00 00 00 00 0c +< 2026/05/04 11:07:57.329285 length=2 from=292 to=293 + 00 0c +> 2026/05/04 11:07:57.329306 length=18 from=446 to=463 + 00 24 00 09 73 79 73 6d 61 73 74 65 72 00 00 00 00 0c +< 2026/05/04 11:07:57.329462 length=28 from=294 to=321 + 00 0f 00 15 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:07:57.329505 length=64 from=464 to=527 + 00 02 00 00 00 00 00 32 43 52 45 41 54 45 20 54 45 4d 50 20 54 41 42 4c 45 20 74 20 28 69 64 20 49 4e 54 45 47 45 52 2c 20 6e 61 6d 65 20 56 41 52 43 48 41 52 28 35 30 29 29 00 16 00 31 00 0c +< 2026/05/04 11:07:57.329630 length=46 from=322 to=367 + 00 08 00 2d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:07:57.329772 length=8 from=528 to=535 + 00 04 00 00 00 07 00 0c +< 2026/05/04 11:07:57.335023 length=28 from=368 to=395 + 00 0f 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:07:57.335068 length=8 from=536 to=543 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 11:07:57.335119 length=2 from=396 to=397 + 00 0c +> 2026/05/04 11:07:57.335149 length=42 from=544 to=585 + 00 02 00 01 00 00 00 1b 49 4e 53 45 52 54 20 49 4e 54 4f 20 74 20 56 41 4c 55 45 53 20 28 31 2c 20 3f 29 00 00 16 00 31 00 0c +< 2026/05/04 11:07:57.335248 length=102 from=398 to=499 + 00 08 00 06 00 00 00 00 00 00 00 33 00 01 00 00 00 05 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 32 6e 61 6d 65 00 00 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:07:57.335370 length=22 from=586 to=607 + 00 04 00 00 00 05 00 01 00 00 00 00 00 00 00 01 78 00 00 07 00 0c +< 2026/05/04 11:07:57.337281 length=48 from=500 to=547 + 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 01 00 00 01 01 00 00 00 00 00 37 00 00 00 01 00 00 00 02 00 0c +> 2026/05/04 11:07:57.337328 length=8 from=608 to=615 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 11:07:57.337363 length=2 from=548 to=549 + 00 0c +> 2026/05/04 11:07:57.337382 length=42 from=616 to=657 + 00 02 00 01 00 00 00 1b 49 4e 53 45 52 54 20 49 4e 54 4f 20 74 20 56 41 4c 55 45 53 20 28 32 2c 20 3f 29 00 00 16 00 31 00 0c +< 2026/05/04 11:07:57.337449 length=102 from=550 to=651 + 00 08 00 06 00 00 00 00 00 00 00 33 00 01 00 00 00 05 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 32 6e 61 6d 65 00 00 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:07:57.337547 length=20 from=658 to=677 + 00 04 00 00 00 05 00 01 00 00 00 00 00 00 00 00 00 07 00 0c +< 2026/05/04 11:07:57.339301 length=48 from=652 to=699 + 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 01 00 00 01 02 00 00 00 00 00 37 00 00 00 01 00 00 00 02 00 0c +> 2026/05/04 11:07:57.339387 length=8 from=678 to=685 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 11:07:57.339436 length=2 from=700 to=701 + 00 0c +> 2026/05/04 11:07:57.339475 length=42 from=686 to=727 + 00 02 00 01 00 00 00 1b 49 4e 53 45 52 54 20 49 4e 54 4f 20 74 20 56 41 4c 55 45 53 20 28 33 2c 20 3f 29 00 00 16 00 31 00 0c +< 2026/05/04 11:07:57.339550 length=102 from=702 to=803 + 00 08 00 06 00 00 00 00 00 00 00 33 00 01 00 00 00 05 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 32 6e 61 6d 65 00 00 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:07:57.339651 length=18 from=728 to=745 + 00 04 00 00 00 05 00 01 00 00 ff ff 00 00 00 07 00 0c +< 2026/05/04 11:07:57.341388 length=48 from=804 to=851 + 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 01 00 00 01 03 00 00 00 00 00 37 00 00 00 02 00 00 00 02 00 0c +> 2026/05/04 11:07:57.341436 length=8 from=746 to=753 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 11:07:57.341477 length=2 from=852 to=853 + 00 0c +> 2026/05/04 11:07:57.341493 length=48 from=754 to=801 + 00 02 00 00 00 00 00 22 53 45 4c 45 43 54 20 69 64 2c 20 6e 61 6d 65 20 46 52 4f 4d 20 74 20 4f 52 44 45 52 20 42 59 20 69 64 00 16 00 31 00 0c +< 2026/05/04 11:07:57.341613 length=114 from=854 to=967 + 00 08 00 02 00 00 00 00 00 00 00 37 00 02 00 00 00 08 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 03 00 00 00 04 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 32 69 64 00 6e 61 6d 65 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 03 00 00 00 02 00 0c +> 2026/05/04 11:07:57.341728 length=42 from=802 to=843 + 00 04 00 00 00 03 00 12 5f 69 66 78 63 30 30 30 30 30 30 30 30 30 30 30 30 31 00 06 00 04 00 00 00 09 00 00 10 00 00 00 00 0c +< 2026/05/04 11:07:57.341824 length=70 from=968 to=1037 + 00 0e 00 00 00 00 00 06 00 00 00 01 01 78 00 0e 00 00 00 00 00 05 00 00 00 02 00 00 00 0e 00 00 00 00 00 06 00 00 00 03 01 00 00 0f 00 00 00 00 00 03 00 00 01 03 00 00 00 00 00 37 00 00 00 03 00 00 00 02 00 0c +> 2026/05/04 11:07:57.341923 length=14 from=844 to=857 + 00 04 00 00 00 09 00 00 10 00 00 00 00 0c +< 2026/05/04 11:07:57.341954 length=28 from=1038 to=1065 + 00 0f 00 00 00 00 00 03 00 00 01 03 00 00 00 00 00 37 00 00 00 03 00 00 00 02 00 0c +> 2026/05/04 11:07:57.341987 length=8 from=858 to=865 + 00 04 00 00 00 0a 00 0c +< 2026/05/04 11:07:57.342013 length=2 from=1066 to=1067 + 00 0c +> 2026/05/04 11:07:57.342026 length=8 from=866 to=873 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 11:07:57.342052 length=2 from=1068 to=1069 + 00 0c +> 2026/05/04 11:07:57.342085 length=2 from=874 to=875 + 00 38 +< 2026/05/04 11:07:57.345084 length=2 from=1070 to=1071 + 00 38 +2026/05/04 11:07:57 socat[837404] N socket 1 (fd 6) is at EOF +2026/05/04 11:07:57 socat[837404] N socket 2 (fd 5) is at EOF +2026/05/04 11:07:57 socat[837404] N exiting with status 0 diff --git a/docs/CAPTURES/20-py-int-null.socat.log b/docs/CAPTURES/20-py-int-null.socat.log new file mode 100644 index 0000000..a1ad742 --- /dev/null +++ b/docs/CAPTURES/20-py-int-null.socat.log @@ -0,0 +1,86 @@ +2026/05/04 11:08:43 socat[839114] N listening on AF=2 0.0.0.0:9090 +2026/05/04 11:08:43 socat[839114] N accepting connection from AF=2 127.0.0.1:39812 on AF=2 127.0.0.1:9090 +2026/05/04 11:08:43 socat[839114] N opening connection to 127.0.0.1:9088 +2026/05/04 11:08:43 socat[839114] N opening connection to AF=2 127.0.0.1:9088 +2026/05/04 11:08:43 socat[839114] N successfully connected from local address AF=2 127.0.0.1:58458 +2026/05/04 11:08:43 socat[839114] N successfully connected to 127.0.0.1:9088 +2026/05/04 11:08:43 socat[839114] N starting data transfer loop with FDs [6,6] and [5,5] +> 2026/05/04 11:08:43.606057 length=384 from=0 to=383 + 01 80 01 3c 00 00 00 64 00 65 00 00 00 3d 00 06 49 45 45 45 4d 00 00 6c 73 71 6c 65 78 65 63 00 00 00 00 00 00 06 39 2e 32 38 30 00 00 0c 52 44 53 23 52 30 30 30 30 30 30 00 00 05 73 71 6c 69 00 00 00 01 3c 00 00 00 00 00 00 00 00 00 01 00 09 69 6e 66 6f 72 6d 69 78 00 00 07 69 6e 34 6d 69 78 00 6f 6c 00 00 00 00 00 00 00 00 00 3d 74 6c 69 74 63 70 00 00 00 00 00 01 00 68 00 0b 00 00 00 03 00 09 69 6e 66 6f 72 6d 69 78 00 00 00 00 00 00 00 00 00 00 00 00 6a 00 06 00 07 44 42 50 41 54 48 00 00 02 2e 00 00 0e 43 4c 49 45 4e 54 5f 4c 4f 43 41 4c 45 00 00 0d 65 6e 5f 55 53 2e 38 38 35 39 2d 31 00 00 11 43 4c 4e 54 5f 50 41 4d 5f 43 41 50 41 42 4c 45 00 00 02 31 00 00 07 44 42 44 41 54 45 00 00 06 59 34 4d 44 2d 00 00 0c 49 46 58 5f 55 50 44 44 45 53 43 00 00 02 31 00 00 09 4e 4f 44 45 46 44 41 43 00 00 03 6e 6f 00 00 6b 00 00 00 00 00 0c cd d7 00 00 00 00 00 0b 72 70 6d 2d 62 75 6c 6c 65 74 00 00 00 00 29 2f 68 6f 6d 65 2f 72 70 6d 2f 63 6c 61 75 64 65 2f 69 6e 66 6f 72 6d 69 78 2f 70 79 74 68 6f 6e 2d 6c 69 62 72 61 72 79 00 00 74 00 20 00 00 00 00 00 00 00 00 00 16 69 6e 66 6f 72 6d 69 78 2d 64 62 40 70 69 64 38 33 39 31 32 37 00 00 7f +< 2026/05/04 11:08:43.617642 length=276 from=0 to=275 + 01 14 02 3c 10 00 00 64 00 65 00 00 00 3d 00 06 49 45 45 45 49 00 00 6c 73 72 76 69 6e 66 78 00 00 00 00 00 00 2f 49 42 4d 20 49 6e 66 6f 72 6d 69 78 20 44 79 6e 61 6d 69 63 20 53 65 72 76 65 72 20 56 65 72 73 69 6f 6e 20 31 35 2e 30 2e 31 2e 30 2e 33 00 00 07 73 65 72 69 61 6c 00 00 09 69 6e 66 6f 72 6d 69 78 00 00 00 01 3c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 6f 6e 00 00 00 00 00 00 00 00 00 3d 73 6f 63 74 63 70 00 00 00 00 00 00 00 66 00 00 00 00 00 00 00 00 00 00 00 14 00 00 00 6b 00 00 00 00 00 00 03 1a 00 00 00 00 00 0d 32 33 32 37 63 34 33 35 34 65 61 38 00 00 00 00 0f 2f 68 6f 6d 65 2f 69 6e 66 6f 72 6d 69 78 00 00 6e 00 04 00 00 00 00 00 74 00 33 00 00 00 c8 00 00 00 c8 00 29 2f 6f 70 74 2f 69 62 6d 2f 69 6e 66 6f 72 6d 69 78 2f 76 31 35 2e 30 2e 31 2e 30 2e 33 2f 62 69 6e 2f 6f 6e 69 6e 69 74 00 00 7f +> 2026/05/04 11:08:43.617886 length=14 from=384 to=397 + 00 7e 00 08 ff fc 7f fc 3c 8c aa 97 00 0c +< 2026/05/04 11:08:43.617956 length=16 from=276 to=291 + 00 7e 00 09 bd be 9f fe 7f b7 ff ef ff 00 00 0c +> 2026/05/04 11:08:43.617983 length=48 from=398 to=445 + 00 51 00 06 00 26 00 0c 00 04 00 06 44 42 54 45 4d 50 00 04 2f 74 6d 70 00 0b 53 55 42 51 43 41 43 48 45 53 5a 00 00 02 31 30 00 00 00 00 00 0c +< 2026/05/04 11:08:43.618048 length=2 from=292 to=293 + 00 0c +> 2026/05/04 11:08:43.618064 length=18 from=446 to=463 + 00 24 00 09 73 79 73 6d 61 73 74 65 72 00 00 00 00 0c +< 2026/05/04 11:08:43.618216 length=28 from=294 to=321 + 00 0f 00 15 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:08:43.618265 length=64 from=464 to=527 + 00 02 00 00 00 00 00 31 43 52 45 41 54 45 20 54 45 4d 50 20 54 41 42 4c 45 20 74 20 28 61 20 49 4e 54 45 47 45 52 2c 20 62 20 42 49 47 49 4e 54 2c 20 63 20 52 45 41 4c 29 00 00 16 00 31 00 0c +< 2026/05/04 11:08:43.618386 length=46 from=322 to=367 + 00 08 00 2d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:08:43.618443 length=8 from=528 to=535 + 00 04 00 00 00 07 00 0c +< 2026/05/04 11:08:43.623425 length=28 from=368 to=395 + 00 0f 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:08:43.623470 length=8 from=536 to=543 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 11:08:43.623525 length=2 from=396 to=397 + 00 0c +> 2026/05/04 11:08:43.623547 length=44 from=544 to=587 + 00 02 00 03 00 00 00 1e 49 4e 53 45 52 54 20 49 4e 54 4f 20 74 20 56 41 4c 55 45 53 20 28 3f 2c 20 3f 2c 20 3f 29 00 16 00 31 00 0c +< 2026/05/04 11:08:43.623650 length=162 from=398 to=559 + 00 08 00 06 00 00 00 00 00 00 00 10 00 03 00 00 00 06 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 02 00 00 00 04 00 34 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 04 00 00 00 0c 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 61 00 62 00 63 00 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:08:43.623815 length=50 from=588 to=637 + 00 04 00 00 00 05 00 03 00 02 00 00 0a 00 00 00 00 2a 00 34 00 00 13 00 00 00 00 e8 d4 a5 0f ff 00 03 00 00 00 00 3f f8 00 00 00 00 00 00 00 07 00 0c +< 2026/05/04 11:08:43.625722 length=48 from=560 to=607 + 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 01 00 00 01 01 00 00 00 00 00 37 00 00 00 01 00 00 00 02 00 0c +> 2026/05/04 11:08:43.625770 length=8 from=638 to=645 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 11:08:43.625811 length=2 from=608 to=609 + 00 0c +> 2026/05/04 11:08:43.625827 length=44 from=646 to=689 + 00 02 00 03 00 00 00 1e 49 4e 53 45 52 54 20 49 4e 54 4f 20 74 20 56 41 4c 55 45 53 20 28 3f 2c 20 3f 2c 20 3f 29 00 16 00 31 00 0c +< 2026/05/04 11:08:43.625903 length=162 from=610 to=771 + 00 08 00 06 00 00 00 00 00 00 00 10 00 03 00 00 00 06 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 02 00 00 00 04 00 34 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 04 00 00 00 0c 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 61 00 62 00 63 00 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 01 00 00 00 01 00 0c +> 2026/05/04 11:08:43.626034 length=30 from=690 to=719 + 00 04 00 00 00 05 00 03 00 00 ff ff 00 00 00 00 ff ff 00 00 00 00 ff ff 00 00 00 07 00 0c +< 2026/05/04 11:08:43.627626 length=48 from=772 to=819 + 00 5e 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 01 00 00 01 02 00 00 00 00 00 37 00 00 00 01 00 00 00 02 00 0c +> 2026/05/04 11:08:43.627673 length=8 from=720 to=727 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 11:08:43.627708 length=2 from=820 to=821 + 00 0c +> 2026/05/04 11:08:43.627722 length=58 from=728 to=785 + 00 02 00 00 00 00 00 2c 53 45 4c 45 43 54 20 61 2c 20 62 2c 20 63 20 46 52 4f 4d 20 74 20 4f 52 44 45 52 20 42 59 20 61 20 4e 55 4c 4c 53 20 46 49 52 53 54 00 16 00 31 00 0c +< 2026/05/04 11:08:43.627862 length=142 from=822 to=963 + 00 08 00 02 00 00 00 00 00 00 00 10 00 03 00 00 00 06 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 02 00 00 00 04 00 34 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 04 00 00 00 0c 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 61 00 62 00 63 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 37 00 00 00 02 00 00 00 02 00 0c +> 2026/05/04 11:08:43.627988 length=42 from=786 to=827 + 00 04 00 00 00 03 00 12 5f 69 66 78 63 30 30 30 30 30 30 30 30 30 30 30 30 31 00 06 00 04 00 00 00 09 00 00 10 00 00 00 00 0c +< 2026/05/04 11:08:43.628082 length=76 from=964 to=1039 + 00 0e 00 00 00 00 00 10 80 00 00 00 80 00 00 00 00 00 00 00 ff ff ff ff 00 0e 00 00 00 00 00 10 00 00 00 2a 00 00 00 e8 d4 a5 0f ff 3f c0 00 00 00 0f 00 00 00 00 00 02 00 00 01 02 00 00 00 00 00 37 00 00 00 02 00 00 00 02 00 0c +> 2026/05/04 11:08:43.628159 length=14 from=828 to=841 + 00 04 00 00 00 09 00 00 10 00 00 00 00 0c +< 2026/05/04 11:08:43.628187 length=28 from=1040 to=1067 + 00 0f 00 00 00 00 00 02 00 00 01 02 00 00 00 00 00 37 00 00 00 02 00 00 00 02 00 0c +> 2026/05/04 11:08:43.628218 length=8 from=842 to=849 + 00 04 00 00 00 0a 00 0c +< 2026/05/04 11:08:43.628252 length=2 from=1068 to=1069 + 00 0c +> 2026/05/04 11:08:43.628264 length=8 from=850 to=857 + 00 04 00 00 00 0b 00 0c +< 2026/05/04 11:08:43.628290 length=2 from=1070 to=1071 + 00 0c +> 2026/05/04 11:08:43.628323 length=2 from=858 to=859 + 00 38 +< 2026/05/04 11:08:43.631545 length=2 from=1072 to=1073 + 00 38 +2026/05/04 11:08:43 socat[839114] N socket 1 (fd 6) is at EOF +2026/05/04 11:08:43 socat[839114] N socket 2 (fd 5) is at EOF +2026/05/04 11:08:43 socat[839114] N exiting with status 0 diff --git a/docs/DECISION_LOG.md b/docs/DECISION_LOG.md index 02b1e7a..7cfb11a 100644 --- a/docs/DECISION_LOG.md +++ b/docs/DECISION_LOG.md @@ -262,6 +262,59 @@ Per-type encoding (Phase 4 MVP): --- +## 2026-05-04 — Parameterized SELECT works with bind-then-cursor-open + +**Status**: active +**Decision**: For parameterized SELECT, send SQ_BIND alone (without SQ_EXECUTE chained) right after PREPARE, then proceed with the regular cursor open + fetch lifecycle (CURNAME+NFETCH+...). The cursor open is what triggers query execution; SQ_BIND just binds the values into the prepared-statement scope. +**Why**: simpler than I expected — server accepts SQ_BIND followed by cursor open in separate PDUs. No need for the IDESCRIBE handshake JDBC does for type discovery. + +PDU sequence: +``` +1. PREPARE+NDESCRIBE+WANTDONE → DESCRIBE+DONE+COST+EOT +2. SQ_BIND (no EXECUTE) → EOT +3. CURNAME+NFETCH → TUPLE*+DONE+COST+EOT +4. NFETCH (drain) → DONE+COST+EOT +5. CLOSE → EOT +6. RELEASE → EOT +``` + +Tested with single int param, multiple int params, string param, mixed `:N` style with LIKE patterns. All work correctly. + +--- + +## 2026-05-04 — NULL row encoding: per-type sentinel values + +**Status**: active +**Decision**: Each IDS type uses a specific NULL sentinel in tuple data; decoders detect and return Python ``None``. + +Sentinels (verified by capture analysis in ``docs/CAPTURES/19-py-null-vs-onechar.socat.log`` and ``20-py-int-null.socat.log``): + +| IDS type | NULL sentinel | Distinguishable from valid value? | +|----------|---------------|------------------------------------| +| SMALLINT | ``0x8000`` (= SHORT_MIN) | Yes — SHORT_MIN can't be a regular value | +| INTEGER | ``0x80000000`` (= INT_MIN) | Yes | +| BIGINT | ``0x8000000000000000`` (= LONG_MIN) | Yes | +| REAL | ``ff ff ff ff`` (NaN bit pattern) | Yes (via bytes match, not value match — NaN != NaN) | +| FLOAT/DOUBLE | ``ff ff ff ff ff ff ff ff`` | Yes | +| VARCHAR | ``[byte 1][byte 0]`` (length=1, content=single nul) | Yes — VARCHAR can't contain embedded nuls; the byte-0 within length-1 is the unambiguous null marker | +| DATE | ``0x80000000`` (same as INT) | Yes | +| BOOL | (TBD — Phase 5+) | — | + +**The VARCHAR null marker is unusual**: ``[byte 1][byte 0]`` looks like "1-byte string containing 0x00" but Informix's VARCHAR can't have embedded nuls anyway, so it's an unambiguous out-of-band signal. Empty string is encoded as ``[byte 0]`` (length=0, no content) — distinct from NULL. + +--- + +## 2026-05-04 — executemany: PREPARE once, BIND+EXECUTE per row, RELEASE once + +**Status**: active +**Decision**: ``Cursor.executemany(sql, seq_of_params)`` does PREPARE once, then loops sending SQ_BIND+SQ_EXECUTE per parameter set, then RELEASE once. + +**Performance**: only ~1.06x faster than a loop of ``execute()`` for 200 INSERTs (336ms vs 319ms in our benchmark). Each BIND+EXECUTE round trip dominates; we save only PREPARE+RELEASE per call. **Phase 4.x optimization opportunity**: chain multiple BIND+EXECUTE calls in one PDU (no intermediate flush + read) for true batch performance — would likely give 5-10x speedup. JDBC's "isBatchUpdatePerSpec" path does this; not yet ported. + +For now, executemany still gives PEP 249 conformance and slight perf improvement; bulk-insert optimization is a future improvement. + +--- + ## (template — copy below this line for new entries) ``` diff --git a/src/informix_db/converters.py b/src/informix_db/converters.py index 62cf45e..662f862 100644 --- a/src/informix_db/converters.py +++ b/src/informix_db/converters.py @@ -27,23 +27,40 @@ _INFORMIX_DATE_EPOCH = datetime.date(1899, 12, 31) DecoderFn = Callable[[bytes], object] -def _decode_smallint(raw: bytes) -> int: - return struct.unpack("!h", raw)[0] +# Informix uses sentinel values for NULL per type — see DECISION_LOG.md +# entry on null sentinel discovery (2026-05-04). +_INT_NULL = 0x80000000 # INT_MIN +_SMALLINT_NULL = 0x8000 # SHORT_MIN +_BIGINT_NULL = 0x8000000000000000 # LONG_MIN +_REAL_NULL = b"\xff\xff\xff\xff" +_DOUBLE_NULL = b"\xff\xff\xff\xff\xff\xff\xff\xff" +_DATE_NULL = 0x80000000 -def _decode_int(raw: bytes) -> int: - return struct.unpack("!i", raw)[0] +def _decode_smallint(raw: bytes) -> int | None: + val = struct.unpack("!h", raw)[0] + return None if val == -0x8000 else val -def _decode_bigint(raw: bytes) -> int: - return struct.unpack("!q", raw)[0] +def _decode_int(raw: bytes) -> int | None: + val = struct.unpack("!i", raw)[0] + return None if val == -0x80000000 else val -def _decode_smfloat(raw: bytes) -> float: +def _decode_bigint(raw: bytes) -> int | None: + val = struct.unpack("!q", raw)[0] + return None if val == -0x8000000000000000 else val + + +def _decode_smfloat(raw: bytes) -> float | None: + if raw == _REAL_NULL: + return None return struct.unpack("!f", raw)[0] -def _decode_float(raw: bytes) -> float: +def _decode_float(raw: bytes) -> float | None: + if raw == _DOUBLE_NULL: + return None return struct.unpack("!d", raw)[0] @@ -52,8 +69,14 @@ def _decode_char(raw: bytes) -> str: return raw.rstrip(b" \x00").decode("iso-8859-1") -def _decode_varchar(raw: bytes) -> str: - """VARCHAR — variable-length string, nul-terminated on the wire.""" +def _decode_varchar(raw: bytes) -> str | None: + """VARCHAR — variable-length string. NULL is the special sentinel ``\\x00`` + (single nul byte). The row decoder peels off the length prefix and passes + the content here. Note: VARCHAR cannot contain embedded nuls anyway, so + a single-nul value is unambiguously the NULL marker. + """ + if raw == b"\x00": + return None return raw.rstrip(b"\x00").decode("iso-8859-1") @@ -64,9 +87,11 @@ def _decode_bool(raw: bytes) -> bool: return raw[0] in (ord("t"), ord("T"), 1) -def _decode_date(raw: bytes) -> datetime.date: - """4-byte big-endian signed int = day count from 1899-12-31.""" +def _decode_date(raw: bytes) -> datetime.date | None: + """4-byte big-endian signed int = day count from 1899-12-31. NULL = 0x80000000.""" days = struct.unpack("!i", raw)[0] + if days == -0x80000000: + return None return _INFORMIX_DATE_EPOCH + datetime.timedelta(days=days) diff --git a/src/informix_db/cursors.py b/src/informix_db/cursors.py index f49f62c..3df68e5 100644 --- a/src/informix_db/cursors.py +++ b/src/informix_db/cursors.py @@ -145,10 +145,9 @@ class Cursor: if is_select: if params: - raise NotSupportedError( - "parameterized SELECT not yet implemented (Phase 4.x)" - ) - self._execute_select() + self._execute_select_with_params(params) + else: + self._execute_select() elif params: self._execute_dml_with_params(params) else: @@ -157,6 +156,22 @@ class Cursor: if self._description is not None: self._row_iter = iter(self._rows) + def _execute_select_with_params(self, params: tuple) -> None: + """Parameterized SELECT: SQ_BIND → CURNAME+NFETCH → drain → CLOSE+RELEASE. + + Note that CURNAME defines the cursor name and is paired with the + prepared statement; binding happens before opening the cursor. + We send SQ_BIND alone first (no SQ_EXECUTE — that's for DML), + then proceed with the normal cursor open + fetch flow. + """ + # Send SQ_BIND alone (without SQ_EXECUTE chained — for SELECT, + # opening the cursor is what executes the prepared query). + self._conn._send_pdu(self._build_bind_only_pdu(params)) + self._drain_to_eot() + # Now open the cursor and fetch — the bound values are in scope + # for the prepared statement. + self._execute_select() + def _execute_select(self) -> None: """Run the SELECT cursor lifecycle: CURNAME+NFETCH → drain → CLOSE → RELEASE.""" cursor_name = _generate_cursor_name() @@ -206,7 +221,64 @@ class Cursor: self._drain_to_eot() def executemany(self, operation: str, seq_of_parameters: Any) -> None: - raise NotSupportedError("executemany lands in Phase 4 (needs parameter binding)") + """Execute the same SQL once per parameter set. + + Per PEP 249. Common case is batched INSERT. We PREPARE once, + loop SQ_BIND+SQ_EXECUTE per parameter set, then RELEASE once — + much cheaper than calling ``execute()`` N times (which would + PREPARE+RELEASE on each iteration). + + Phase 4 supports DML (INSERT/UPDATE/DELETE) only — SELECT in + executemany doesn't make much sense and isn't implemented. + """ + self._check_open() + + seq = list(seq_of_parameters) + if not seq: + self._rowcount = 0 + return + + # All parameter tuples must agree on length (= num placeholders). + first_len = len(seq[0]) + for i, p in enumerate(seq): + if len(p) != first_len: + raise ProgrammingError( + f"executemany: parameter set [{i}] has {len(p)} values, " + f"expected {first_len} (matching set [0])" + ) + + # Detect SELECT — not supported in executemany. + first_word = operation.lstrip().split(None, 1)[0].upper() if operation.strip() else "" + if first_word == "SELECT": + raise NotSupportedError("executemany on SELECT is not supported") + + sql = _rewrite_numeric_to_qmark(operation) + + # Reset per-execute state. + self._description = None + self._columns = [] + self._rowcount = -1 + self._rows = [] + self._row_iter = None + self._statement_already_done = False + + # PREPARE once. + self._conn._send_pdu(self._build_prepare_pdu(sql, num_qmarks=first_len)) + self._read_describe_response() + + # BIND+EXECUTE per parameter set. + total_rowcount = 0 + for params in seq: + self._rowcount = -1 + self._conn._send_pdu(self._build_bind_execute_pdu(tuple(params))) + self._drain_to_eot() + if self._rowcount > 0: + total_rowcount += self._rowcount + + # RELEASE once. + self._conn._send_pdu(self._build_release_pdu()) + self._drain_to_eot() + self._rowcount = total_rowcount def fetchone(self) -> tuple | None: self._check_open() @@ -282,6 +354,31 @@ class Cursor: writer.write_short(MessageType.SQ_EOT) return buf.getvalue() + def _build_bind_only_pdu(self, params: tuple) -> bytes: + """SQ_BIND with parameter values + SQ_EOT (no SQ_EXECUTE). + + Used for parameterized SELECT — the cursor open (CURNAME+NFETCH) + is what triggers query execution; SQ_BIND just binds the values + in scope for the prepared statement. + """ + writer, buf = make_pdu_writer() + writer.write_short(MessageType.SQ_ID) + writer.write_int(MessageType.SQ_BIND) + writer.write_short(len(params)) + for value in params: + if value is None: + writer.write_short(0) + writer.write_short(-1) + writer.write_short(0) + else: + ifx_type, prec, raw = encode_param(value) + writer.write_short(ifx_type) + writer.write_short(0) + writer.write_short(prec) + writer.write_padded(raw) + writer.write_short(MessageType.SQ_EOT) + return buf.getvalue() + def _build_bind_execute_pdu(self, params: tuple) -> bytes: """SQ_BIND with parameter values + SQ_EXECUTE + SQ_EOT. diff --git a/tests/test_nulls.py b/tests/test_nulls.py new file mode 100644 index 0000000..e0c3314 --- /dev/null +++ b/tests/test_nulls.py @@ -0,0 +1,102 @@ +"""Phase 4.x integration tests — NULL row decoding for all supported types. + +Per-type NULL sentinels (verified empirically — see DECISION_LOG.md): + INT → 0x80000000 (INT_MIN) + BIGINT → 0x8000000000000000 (LONG_MIN) + REAL → all 0xFF (NaN bit pattern) + FLOAT → all 0xFF + VARCHAR → [byte 1][byte 0] (length=1, single nul) +""" + +from __future__ import annotations + +import pytest + +import informix_db +from tests.conftest import ConnParams + +pytestmark = pytest.mark.integration + + +def _connect(conn_params: ConnParams) -> informix_db.Connection: + return informix_db.connect( + host=conn_params.host, + port=conn_params.port, + user=conn_params.user, + password=conn_params.password, + database=conn_params.database, + server=conn_params.server, + connect_timeout=10.0, + read_timeout=10.0, + ) + + +def test_null_in_int_decoded_as_none(conn_params: ConnParams) -> None: + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_null_int (id INTEGER, val INTEGER)") + cur.execute("INSERT INTO t_null_int VALUES (?, ?)", (1, None)) + cur.execute("SELECT val FROM t_null_int") + assert cur.fetchone() == (None,) + + +def test_null_in_bigint_decoded_as_none(conn_params: ConnParams) -> None: + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_null_bi (id INTEGER, val BIGINT)") + cur.execute("INSERT INTO t_null_bi VALUES (?, ?)", (1, None)) + cur.execute("SELECT val FROM t_null_bi") + assert cur.fetchone() == (None,) + + +def test_null_in_float_decoded_as_none(conn_params: ConnParams) -> None: + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_null_f (id INTEGER, val FLOAT)") + cur.execute("INSERT INTO t_null_f VALUES (?, ?)", (1, None)) + cur.execute("SELECT val FROM t_null_f") + assert cur.fetchone() == (None,) + + +def test_null_in_real_decoded_as_none(conn_params: ConnParams) -> None: + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_null_r (id INTEGER, val REAL)") + cur.execute("INSERT INTO t_null_r VALUES (?, ?)", (1, None)) + cur.execute("SELECT val FROM t_null_r") + assert cur.fetchone() == (None,) + + +def test_null_in_varchar_decoded_as_none(conn_params: ConnParams) -> None: + """The trickiest one — VARCHAR NULL is `[byte 1][byte 0]`, distinct from empty `''`.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_null_vc (id INTEGER, val VARCHAR(50))") + cur.execute("INSERT INTO t_null_vc VALUES (?, ?)", (1, None)) + cur.execute("SELECT val FROM t_null_vc") + assert cur.fetchone() == (None,) + + +def test_empty_varchar_distinct_from_null(conn_params: ConnParams) -> None: + """Empty string `''` is encoded `[byte 0]` (length=0); must NOT be confused with NULL.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_empty_vc (id INTEGER, val VARCHAR(50))") + cur.execute("INSERT INTO t_empty_vc VALUES (?, ?)", (1, "")) + cur.execute("INSERT INTO t_empty_vc VALUES (?, ?)", (2, None)) + cur.execute("SELECT id, val FROM t_empty_vc ORDER BY id") + rows = cur.fetchall() + assert rows == [(1, ""), (2, None)] + + +def test_mixed_nulls_and_values_in_one_row(conn_params: ConnParams) -> None: + """Mixed NULL + non-NULL columns in one row — proves per-column null detection.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute( + "CREATE TEMP TABLE t_mix (a INTEGER, b VARCHAR(20), c FLOAT, d BIGINT)" + ) + cur.execute("INSERT INTO t_mix VALUES (?, ?, ?, ?)", (1, None, 3.14, None)) + cur.execute("INSERT INTO t_mix VALUES (?, ?, ?, ?)", (None, "two", None, 200)) + cur.execute("SELECT a, b, c, d FROM t_mix ORDER BY a NULLS LAST") + assert cur.fetchall() == [(1, None, 3.14, None), (None, "two", None, 200)] diff --git a/tests/test_params.py b/tests/test_params.py index 4e43144..b959253 100644 --- a/tests/test_params.py +++ b/tests/test_params.py @@ -101,12 +101,101 @@ def test_unsupported_param_type_raises(conn_params: ConnParams) -> None: cur.execute("INSERT INTO t_p_f VALUES (?)", (b"raw bytes",)) -def test_parameterized_select_not_yet_supported(conn_params: ConnParams) -> None: - """Parameterized SELECT lands in Phase 4.x — currently raises.""" +def test_parameterized_select_int(conn_params: ConnParams) -> None: + """Parameterized SELECT with an int parameter.""" with _connect(conn_params) as conn: cur = conn.cursor() - with pytest.raises(informix_db.NotSupportedError, match=r"Phase 4\.x"): - cur.execute("SELECT 1 FROM systables WHERE tabid = ?", (1,)) + cur.execute("SELECT tabname FROM systables WHERE tabid = ?", (1,)) + assert cur.fetchone() == ("systables",) + + +def test_parameterized_select_multiple_params(conn_params: ConnParams) -> None: + """Parameterized SELECT with two int parameters bounding a range.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute( + "SELECT tabname FROM systables WHERE tabid >= ? AND tabid <= ? ORDER BY tabid", + (1, 3), + ) + assert cur.fetchall() == [("systables",), ("syscolumns",), ("sysindices",)] + + +def test_parameterized_select_string_param(conn_params: ConnParams) -> None: + """Parameterized SELECT with a string parameter.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("SELECT tabid FROM systables WHERE tabname = ?", ("systables",)) + assert cur.fetchone() == (1,) + + +def test_parameterized_select_numeric_style(conn_params: ConnParams) -> None: + """``:1`` style works for SELECT too.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("SELECT tabname FROM systables WHERE tabid = :1", (2,)) + assert cur.fetchone() == ("syscolumns",) + + +def test_executemany_basic_insert(conn_params: ConnParams) -> None: + """``executemany`` for batched INSERT — PREPARE once, BIND/EXECUTE per row.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_em_a (id INTEGER, name VARCHAR(50))") + rows = [(1, "alpha"), (2, "beta"), (3, "gamma"), (4, "delta")] + cur.executemany("INSERT INTO t_em_a VALUES (?, ?)", rows) + assert cur.rowcount == 4 + + cur.execute("SELECT id, name FROM t_em_a ORDER BY id") + assert cur.fetchall() == rows + + +def test_executemany_update(conn_params: ConnParams) -> None: + """``executemany`` works for UPDATE too — per-row WHERE matches.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_em_u (id INTEGER, name VARCHAR(50))") + cur.executemany( + "INSERT INTO t_em_u VALUES (?, ?)", + [(1, "old"), (2, "old"), (3, "old")], + ) + cur.executemany( + "UPDATE t_em_u SET name = ? WHERE id = ?", + [("A", 1), ("B", 2), ("C", 3)], + ) + cur.execute("SELECT id, name FROM t_em_u ORDER BY id") + assert cur.fetchall() == [(1, "A"), (2, "B"), (3, "C")] + + +def test_executemany_empty_list(conn_params: ConnParams) -> None: + """Empty parameter list is a no-op with rowcount=0.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_em_e (id INTEGER)") + cur.executemany("INSERT INTO t_em_e VALUES (?)", []) + assert cur.rowcount == 0 + + +def test_executemany_inconsistent_param_lens_raises(conn_params: ConnParams) -> None: + """Mismatched parameter-set lengths must raise ProgrammingError.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + cur.execute("CREATE TEMP TABLE t_em_bad (id INTEGER, name VARCHAR(50))") + with pytest.raises(informix_db.ProgrammingError, match="parameter set"): + cur.executemany( + "INSERT INTO t_em_bad VALUES (?, ?)", + [(1, "ok"), (2,)], # second has only 1 value + ) + + +def test_executemany_select_unsupported(conn_params: ConnParams) -> None: + """``executemany`` on SELECT doesn't make sense — must raise.""" + with _connect(conn_params) as conn: + cur = conn.cursor() + with pytest.raises(informix_db.NotSupportedError, match="SELECT"): + cur.executemany( + "SELECT tabname FROM systables WHERE tabid = ?", + [(1,), (2,)], + ) def test_dict_params_unsupported(conn_params: ConnParams) -> None: