Python: Add self-validating CFG tests

python/ql/test/library-tests/ControlFlow/evaluation-order/AllLiveReachable.ql

@@ -0,0 +1,18 @@
+/**
+ * Checks that every live (non-dead) annotation in the test function's
+ * own scope is reachable from the function entry in the CFG.
+ * Annotations in nested scopes (generators, async, lambdas, comprehensions)
+ * have separate CFGs and are excluded from this check.
+ */
+
+import TimerUtils
+import OldCfgImpl
+
+private module Utils = EvalOrderCfgUtils<OldCfg>;
+
+private import Utils
+private import Utils::CfgTests
+
+from TimerCfgNode a, TestFunction f
+where allLiveReachable(a, f)
+select a, "Unreachable live annotation; entry of $@ does not reach this node", f, f.getName()

python/ql/test/library-tests/ControlFlow/evaluation-order/AnnotationHasCfgNode.expected

@@ -0,0 +1 @@
+

python/ql/test/library-tests/ControlFlow/evaluation-order/AnnotationHasCfgNode.ql

@@ -0,0 +1,15 @@
+/**
+ * Checks that every timer annotation has a corresponding CFG node.
+ */
+
+import TimerUtils
+import OldCfgImpl
+
+private module Utils = EvalOrderCfgUtils<OldCfg>;
+
+private import Utils::CfgTests
+
+from TimerAnnotation ann
+where annotationWithoutCfgNode(ann)
+select ann, "Annotation in $@ has no CFG node", ann.getTestFunction(),
+  ann.getTestFunction().getName()

python/ql/test/library-tests/ControlFlow/evaluation-order/BasicBlockAnnotationGap.ql

@@ -0,0 +1,22 @@
+/**
+ * Checks that within a basic block, if a node is annotated then its
+ * successor is also annotated (or excluded). A gap in annotations
+ * within a basic block indicates a missing annotation, since there
+ * are no branches to justify the gap.
+ *
+ * Nodes with exceptional successors are excluded, as the exception
+ * edge leaves the basic block and the normal successor may be dead.
+ */
+
+import TimerUtils
+import OldCfgImpl
+
+private module Utils = EvalOrderCfgUtils<OldCfg>;
+
+private import Utils
+private import Utils::CfgTests
+
+from TimerCfgNode a, CfgNode succ
+where basicBlockAnnotationGap(a, succ)
+select a, "Annotated node followed by unannotated $@ in the same basic block", succ,
+  succ.getNode().toString()

python/ql/test/library-tests/ControlFlow/evaluation-order/BasicBlockOrdering.expected

@@ -0,0 +1,13 @@
+| test_boolean.py:9:10:9:43 | ControlFlowNode for BoolExpr | Basic block ordering: $@ appears before $@ | test_boolean.py:9:59:9:59 | IntegerLiteral | timestamp 2 | test_boolean.py:9:19:9:19 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:15:10:15:43 | ControlFlowNode for BoolExpr | Basic block ordering: $@ appears before $@ | test_boolean.py:15:50:15:50 | IntegerLiteral | timestamp 1 | test_boolean.py:15:20:15:20 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:21:10:21:42 | ControlFlowNode for BoolExpr | Basic block ordering: $@ appears before $@ | test_boolean.py:21:49:21:49 | IntegerLiteral | timestamp 1 | test_boolean.py:21:19:21:19 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:27:10:27:43 | ControlFlowNode for BoolExpr | Basic block ordering: $@ appears before $@ | test_boolean.py:27:59:27:59 | IntegerLiteral | timestamp 2 | test_boolean.py:27:20:27:20 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:40:10:40:61 | ControlFlowNode for BoolExpr | Basic block ordering: $@ appears before $@ | test_boolean.py:40:86:40:86 | IntegerLiteral | timestamp 3 | test_boolean.py:40:16:40:16 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:46:10:46:61 | ControlFlowNode for BoolExpr | Basic block ordering: $@ appears before $@ | test_boolean.py:46:86:46:86 | IntegerLiteral | timestamp 3 | test_boolean.py:46:16:46:16 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:52:10:52:95 | ControlFlowNode for BoolExpr | Basic block ordering: $@ appears before $@ | test_boolean.py:52:120:52:120 | IntegerLiteral | timestamp 4 | test_boolean.py:52:20:52:20 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:52:10:52:95 | ControlFlowNode for BoolExpr | Basic block ordering: $@ appears before $@ | test_boolean.py:52:120:52:120 | IntegerLiteral | timestamp 4 | test_boolean.py:52:63:52:63 | IntegerLiteral | timestamp 2 |
+| test_boolean.py:52:11:52:47 | ControlFlowNode for BoolExpr | Basic block ordering: $@ appears before $@ | test_boolean.py:52:63:52:63 | IntegerLiteral | timestamp 2 | test_boolean.py:52:20:52:20 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:64:10:64:52 | ControlFlowNode for BoolExpr | Basic block ordering: $@ appears before $@ | test_boolean.py:64:59:64:59 | IntegerLiteral | timestamp 6 | test_boolean.py:64:17:64:17 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:64:10:64:52 | ControlFlowNode for BoolExpr | Basic block ordering: $@ appears before $@ | test_boolean.py:64:59:64:59 | IntegerLiteral | timestamp 6 | test_boolean.py:64:27:64:27 | IntegerLiteral | timestamp 2 |
+| test_boolean.py:76:10:76:51 | ControlFlowNode for BoolExpr | Basic block ordering: $@ appears before $@ | test_boolean.py:76:58:76:58 | IntegerLiteral | timestamp 6 | test_boolean.py:76:17:76:17 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:76:10:76:51 | ControlFlowNode for BoolExpr | Basic block ordering: $@ appears before $@ | test_boolean.py:76:58:76:58 | IntegerLiteral | timestamp 6 | test_boolean.py:76:27:76:27 | IntegerLiteral | timestamp 2 |

python/ql/test/library-tests/ControlFlow/evaluation-order/BasicBlockOrdering.ql

@@ -0,0 +1,17 @@
+/**
+ * Checks that within a single basic block, annotations appear in
+ * increasing minimum-timestamp order.
+ */
+
+import TimerUtils
+import OldCfgImpl
+
+private module Utils = EvalOrderCfgUtils<OldCfg>;
+
+private import Utils
+private import Utils::CfgTests
+
+from TimerCfgNode a, TimerCfgNode b, int minA, int minB
+where basicBlockOrdering(a, b, minA, minB)
+select a, "Basic block ordering: $@ appears before $@", a.getTimestampExpr(minA),
+  "timestamp " + minA, b.getTimestampExpr(minB), "timestamp " + minB

python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutiveTimestamps.expected

@@ -0,0 +1,9 @@
+| test_boolean.py:9:26:9:27 | IntegerLiteral | $@ in $@ has no consecutive successor (expected 2) | test_boolean.py:9:33:9:33 | IntegerLiteral | Timestamp 1 | test_boolean.py:7:1:7:27 | Function test_and_both_sides | test_and_both_sides |
+| test_boolean.py:15:10:15:14 | False | $@ in $@ has no consecutive successor (expected 1) | test_boolean.py:15:20:15:20 | IntegerLiteral | Timestamp 0 | test_boolean.py:13:1:13:30 | Function test_and_short_circuit | test_and_short_circuit |
+| test_boolean.py:21:10:21:13 | True | $@ in $@ has no consecutive successor (expected 1) | test_boolean.py:21:19:21:19 | IntegerLiteral | Timestamp 0 | test_boolean.py:19:1:19:29 | Function test_or_short_circuit | test_or_short_circuit |
+| test_boolean.py:27:26:27:27 | IntegerLiteral | $@ in $@ has no consecutive successor (expected 2) | test_boolean.py:27:33:27:33 | IntegerLiteral | Timestamp 1 | test_boolean.py:25:1:25:26 | Function test_or_both_sides | test_or_both_sides |
+| test_boolean.py:40:45:40:45 | IntegerLiteral | $@ in $@ has no consecutive successor (expected 3) | test_boolean.py:40:51:40:51 | IntegerLiteral | Timestamp 2 | test_boolean.py:38:1:38:24 | Function test_chained_and | test_chained_and |
+| test_boolean.py:46:44:46:45 | IntegerLiteral | $@ in $@ has no consecutive successor (expected 3) | test_boolean.py:46:51:46:51 | IntegerLiteral | Timestamp 2 | test_boolean.py:44:1:44:23 | Function test_chained_or | test_chained_or |
+| test_boolean.py:52:11:52:47 | BoolExpr | $@ in $@ has no consecutive successor (expected 3) | test_boolean.py:52:63:52:63 | IntegerLiteral | Timestamp 2 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or |
+| test_boolean.py:52:27:52:31 | False | $@ in $@ has no consecutive successor (expected 2) | test_boolean.py:52:37:52:37 | IntegerLiteral | Timestamp 1 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or |
+| test_boolean.py:52:78:52:79 | IntegerLiteral | $@ in $@ has no consecutive successor (expected 4) | test_boolean.py:52:85:52:85 | IntegerLiteral | Timestamp 3 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or |

python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutiveTimestamps.ql

@@ -0,0 +1,25 @@
+/**
+ * Checks that consecutive annotated nodes have consecutive timestamps:
+ * for each annotation with timestamp `a`, some CFG node for that annotation
+ * must have a next annotation containing `a + 1`.
+ *
+ * Handles CFG splitting (e.g., finally blocks duplicated for normal/exceptional
+ * flow) by checking that at least one split has the required successor.
+ *
+ * Only applies to functions where all annotations are in the function's
+ * own scope (excludes tests with generators, async, comprehensions, or
+ * lambdas that have annotations in nested scopes).
+ */
+
+import TimerUtils
+import OldCfgImpl
+
+private module Utils = EvalOrderCfgUtils<OldCfg>;
+
+private import Utils
+private import Utils::CfgTests
+
+from TimerAnnotation ann, int a
+where consecutiveTimestamps(ann, a)
+select ann, "$@ in $@ has no consecutive successor (expected " + (a + 1) + ")",
+  ann.getTimestampExpr(a), "Timestamp " + a, ann.getTestFunction(), ann.getTestFunction().getName()

python/ql/test/library-tests/ControlFlow/evaluation-order/ContiguousTimestamps.ql

@@ -0,0 +1,17 @@
+/**
+ * Checks that timestamps form a contiguous sequence {0, 1, ..., max}
+ * within each test function. Every integer in the range must appear
+ * in at least one annotation (live or dead).
+ */
+
+import TimerUtils
+
+from TestFunction f, int missing, int maxTs, TimerAnnotation maxAnn
+where
+  maxTs = max(TimerAnnotation a | a.getTestFunction() = f | a.getATimestamp()) and
+  maxAnn.getTestFunction() = f and
+  maxAnn.getATimestamp() = maxTs and
+  missing = [0 .. maxTs] and
+  not exists(TimerAnnotation a | a.getTestFunction() = f and a.getATimestamp() = missing)
+select f, "Missing timestamp " + missing + " (max is $@)", maxAnn.getTimestampExpr(maxTs),
+  maxTs.toString()

python/ql/test/library-tests/ControlFlow/evaluation-order/MissingAnnotations.ql

@@ -0,0 +1,15 @@
+/**
+ * Finds expressions in test functions that lack a timer annotation
+ * and are not part of the timer mechanism or otherwise excluded.
+ * An empty result means every annotatable expression is covered.
+ */
+
+import python
+import TimerUtils
+
+from TestFunction f, Expr e
+where
+  e.getScope().getEnclosingScope*() = f and
+  not isTimerMechanism(e, f) and
+  not isUnannotatable(e)
+select e, "Missing annotation in $@", f, f.getName()

python/ql/test/library-tests/ControlFlow/evaluation-order/NeverReachable.expected

@@ -0,0 +1,2 @@
+| test_match.py:159:13:159:13 | IntegerLiteral | Node annotated with t.never is reachable in $@ | test_match.py:151:1:151:42 | Function test_match_exhaustive_return_first | test_match_exhaustive_return_first |
+| test_match.py:172:13:172:13 | IntegerLiteral | Node annotated with t.never is reachable in $@ | test_match.py:164:1:164:45 | Function test_match_exhaustive_return_wildcard | test_match_exhaustive_return_wildcard |

python/ql/test/library-tests/ControlFlow/evaluation-order/NeverReachable.ql

@@ -0,0 +1,17 @@
+/**
+ * Checks that expressions annotated with `t.never` either have no CFG
+ * node, or if they do, that the node is not reachable from its scope's
+ * entry (including within the same basic block).
+ */
+
+import TimerUtils
+import OldCfgImpl
+
+private module Utils = EvalOrderCfgUtils<OldCfg>;
+
+private import Utils::CfgTests
+
+from TimerAnnotation ann
+where neverReachable(ann)
+select ann, "Node annotated with t.never is reachable in $@", ann.getTestFunction(),
+  ann.getTestFunction().getName()

python/ql/test/library-tests/ControlFlow/evaluation-order/NoBackwardFlow.expected

@@ -0,0 +1,10 @@
+| test_boolean.py:9:10:9:43 | ControlFlowNode for BoolExpr | Backward flow: $@ flows to $@ (max timestamp $@) | test_boolean.py:9:59:9:59 | IntegerLiteral | 2 | test_boolean.py:9:10:9:13 | ControlFlowNode for True | True | test_boolean.py:9:19:9:19 | IntegerLiteral | 0 |
+| test_boolean.py:15:10:15:43 | ControlFlowNode for BoolExpr | Backward flow: $@ flows to $@ (max timestamp $@) | test_boolean.py:15:50:15:50 | IntegerLiteral | 1 | test_boolean.py:15:10:15:14 | ControlFlowNode for False | False | test_boolean.py:15:20:15:20 | IntegerLiteral | 0 |
+| test_boolean.py:21:10:21:42 | ControlFlowNode for BoolExpr | Backward flow: $@ flows to $@ (max timestamp $@) | test_boolean.py:21:49:21:49 | IntegerLiteral | 1 | test_boolean.py:21:10:21:13 | ControlFlowNode for True | True | test_boolean.py:21:19:21:19 | IntegerLiteral | 0 |
+| test_boolean.py:27:10:27:43 | ControlFlowNode for BoolExpr | Backward flow: $@ flows to $@ (max timestamp $@) | test_boolean.py:27:59:27:59 | IntegerLiteral | 2 | test_boolean.py:27:10:27:14 | ControlFlowNode for False | False | test_boolean.py:27:20:27:20 | IntegerLiteral | 0 |
+| test_boolean.py:40:10:40:61 | ControlFlowNode for BoolExpr | Backward flow: $@ flows to $@ (max timestamp $@) | test_boolean.py:40:86:40:86 | IntegerLiteral | 3 | test_boolean.py:40:10:40:10 | ControlFlowNode for IntegerLiteral | IntegerLiteral | test_boolean.py:40:16:40:16 | IntegerLiteral | 0 |
+| test_boolean.py:46:10:46:61 | ControlFlowNode for BoolExpr | Backward flow: $@ flows to $@ (max timestamp $@) | test_boolean.py:46:86:46:86 | IntegerLiteral | 3 | test_boolean.py:46:10:46:10 | ControlFlowNode for IntegerLiteral | IntegerLiteral | test_boolean.py:46:16:46:16 | IntegerLiteral | 0 |
+| test_boolean.py:52:10:52:95 | ControlFlowNode for BoolExpr | Backward flow: $@ flows to $@ (max timestamp $@) | test_boolean.py:52:120:52:120 | IntegerLiteral | 4 | test_boolean.py:52:11:52:47 | ControlFlowNode for BoolExpr | BoolExpr | test_boolean.py:52:63:52:63 | IntegerLiteral | 2 |
+| test_boolean.py:52:11:52:47 | ControlFlowNode for BoolExpr | Backward flow: $@ flows to $@ (max timestamp $@) | test_boolean.py:52:63:52:63 | IntegerLiteral | 2 | test_boolean.py:52:11:52:14 | ControlFlowNode for True | True | test_boolean.py:52:20:52:20 | IntegerLiteral | 0 |
+| test_boolean.py:64:10:64:52 | ControlFlowNode for BoolExpr | Backward flow: $@ flows to $@ (max timestamp $@) | test_boolean.py:64:59:64:59 | IntegerLiteral | 6 | test_boolean.py:64:11:64:11 | ControlFlowNode for f | f | test_boolean.py:64:17:64:17 | IntegerLiteral | 0 |
+| test_boolean.py:76:10:76:51 | ControlFlowNode for BoolExpr | Backward flow: $@ flows to $@ (max timestamp $@) | test_boolean.py:76:58:76:58 | IntegerLiteral | 6 | test_boolean.py:76:11:76:11 | ControlFlowNode for f | f | test_boolean.py:76:17:76:17 | IntegerLiteral | 0 |

python/ql/test/library-tests/ControlFlow/evaluation-order/NoBackwardFlow.ql

@@ -0,0 +1,18 @@
+/**
+ * Checks that time never flows backward between consecutive timer annotations
+ * in the CFG. For each pair of consecutive annotated nodes (A -> B), there must
+ * exist timestamps a in A and b in B with a < b.
+ */
+
+import TimerUtils
+import OldCfgImpl
+
+private module Utils = EvalOrderCfgUtils<OldCfg>;
+
+private import Utils
+private import Utils::CfgTests
+
+from TimerCfgNode a, TimerCfgNode b, int minA, int maxB
+where noBackwardFlow(a, b, minA, maxB)
+select a, "Backward flow: $@ flows to $@ (max timestamp $@)", a.getTimestampExpr(minA),
+  minA.toString(), b, b.getNode().toString(), b.getTimestampExpr(maxB), maxB.toString()

python/ql/test/library-tests/ControlFlow/evaluation-order/NoBasicBlock.expected

@@ -0,0 +1 @@
+

python/ql/test/library-tests/ControlFlow/evaluation-order/NoBasicBlock.ql

@@ -0,0 +1,15 @@
+/**
+ * Checks that every annotated CFG node belongs to a basic block.
+ */
+
+import TimerUtils
+import OldCfgImpl
+
+private module Utils = EvalOrderCfgUtils<OldCfg>;
+
+private import Utils
+private import Utils::CfgTests
+
+from CfgNode n, TestFunction f
+where noBasicBlock(n, f)
+select n, "CFG node in $@ does not belong to any basic block", f, f.getName()

python/ql/test/library-tests/ControlFlow/evaluation-order/NoSharedReachable.ql

@@ -0,0 +1,17 @@
+/**
+ * Checks that two annotations sharing a timestamp value are on
+ * mutually exclusive CFG paths (neither can reach the other).
+ */
+
+import TimerUtils
+import OldCfgImpl
+
+private module Utils = EvalOrderCfgUtils<OldCfg>;
+
+private import Utils
+private import Utils::CfgTests
+
+from TimerCfgNode a, TimerCfgNode b, int ts
+where noSharedReachable(a, b, ts)
+select a, "Shared timestamp $@ but this node reaches $@", a.getTimestampExpr(ts), ts.toString(), b,
+  b.getNode().toString()

python/ql/test/library-tests/ControlFlow/evaluation-order/OldCfgImpl.qll

@@ -0,0 +1,16 @@
+/**
+ * Implementation of the evaluation-order CFG signature using the existing
+ * Python control flow graph.
+ */
+
+private import python as Py
+import TimerUtils
+
+/** Existing Python CFG implementation of the evaluation-order signature. */
+module OldCfg implements EvalOrderCfgSig {
+  class CfgNode = Py::ControlFlowNode;
+
+  class BasicBlock = Py::BasicBlock;
+
+  CfgNode scopeGetEntryNode(Py::Scope s) { result = s.getEntryNode() }
+}

python/ql/test/library-tests/ControlFlow/evaluation-order/StrictForward.expected

@@ -0,0 +1,10 @@
+| test_boolean.py:9:10:9:43 | ControlFlowNode for BoolExpr | Strict forward violation: $@ flows to $@ | test_boolean.py:9:59:9:59 | IntegerLiteral | timestamp 2 | test_boolean.py:9:19:9:19 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:15:10:15:43 | ControlFlowNode for BoolExpr | Strict forward violation: $@ flows to $@ | test_boolean.py:15:50:15:50 | IntegerLiteral | timestamp 1 | test_boolean.py:15:20:15:20 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:21:10:21:42 | ControlFlowNode for BoolExpr | Strict forward violation: $@ flows to $@ | test_boolean.py:21:49:21:49 | IntegerLiteral | timestamp 1 | test_boolean.py:21:19:21:19 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:27:10:27:43 | ControlFlowNode for BoolExpr | Strict forward violation: $@ flows to $@ | test_boolean.py:27:59:27:59 | IntegerLiteral | timestamp 2 | test_boolean.py:27:20:27:20 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:40:10:40:61 | ControlFlowNode for BoolExpr | Strict forward violation: $@ flows to $@ | test_boolean.py:40:86:40:86 | IntegerLiteral | timestamp 3 | test_boolean.py:40:16:40:16 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:46:10:46:61 | ControlFlowNode for BoolExpr | Strict forward violation: $@ flows to $@ | test_boolean.py:46:86:46:86 | IntegerLiteral | timestamp 3 | test_boolean.py:46:16:46:16 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:52:10:52:95 | ControlFlowNode for BoolExpr | Strict forward violation: $@ flows to $@ | test_boolean.py:52:120:52:120 | IntegerLiteral | timestamp 4 | test_boolean.py:52:63:52:63 | IntegerLiteral | timestamp 2 |
+| test_boolean.py:52:11:52:47 | ControlFlowNode for BoolExpr | Strict forward violation: $@ flows to $@ | test_boolean.py:52:63:52:63 | IntegerLiteral | timestamp 2 | test_boolean.py:52:20:52:20 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:64:10:64:52 | ControlFlowNode for BoolExpr | Strict forward violation: $@ flows to $@ | test_boolean.py:64:59:64:59 | IntegerLiteral | timestamp 6 | test_boolean.py:64:17:64:17 | IntegerLiteral | timestamp 0 |
+| test_boolean.py:76:10:76:51 | ControlFlowNode for BoolExpr | Strict forward violation: $@ flows to $@ | test_boolean.py:76:58:76:58 | IntegerLiteral | timestamp 6 | test_boolean.py:76:17:76:17 | IntegerLiteral | timestamp 0 |

python/ql/test/library-tests/ControlFlow/evaluation-order/StrictForward.ql

@@ -0,0 +1,18 @@
+/**
+ * Stronger version of NoBackwardFlow: for consecutive annotated nodes
+ * A -> B that both have a single timestamp (non-loop code) and B does
+ * NOT dominate A (forward edge), requires max(A) < min(B).
+ */
+
+import TimerUtils
+import OldCfgImpl
+
+private module Utils = EvalOrderCfgUtils<OldCfg>;
+
+private import Utils
+private import Utils::CfgTests
+
+from TimerCfgNode a, TimerCfgNode b, int maxA, int minB
+where strictForward(a, b, maxA, minB)
+select a, "Strict forward violation: $@ flows to $@", a.getTimestampExpr(maxA), "timestamp " + maxA,
+  b.getTimestampExpr(minB), "timestamp " + minB