Python: Prevent divergence in type-hint analysis. (ODASA-8075)

python/ql/src/semmle/python/objects/Callables.qll

@@ -163,6 +163,8 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti
 
     override predicate useOriginAsLegacyObject() { none() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 
@@ -288,6 +290,8 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc
 
     override predicate useOriginAsLegacyObject() { none() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 /** Class representing methods of built-in classes (otherwise known as method-descriptors) such as `list.append`.
@@ -382,6 +386,8 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod
 
     override predicate useOriginAsLegacyObject() { none() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 /** Class representing bound-methods.
@@ -473,4 +479,6 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
         this.getFunction().contextSensitiveCallee()
     }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }

python/ql/src/semmle/python/objects/Classes.qll

@@ -103,6 +103,8 @@ abstract class ClassObjectInternal extends ObjectInternal {
         Types::getBase(this, _).hasAttribute(name)
     }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 /** Class representing Python source classes */
@@ -445,6 +447,8 @@ class SubscriptedTypeInternal extends ObjectInternal, TSubscriptedType {
     /* Classes aren't usually iterable, but can e.g. Enums */
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
+    override predicate isNotSubscriptedType() { none() }
+
 }
 
 

python/ql/src/semmle/python/objects/Constants.qll

@@ -76,6 +76,8 @@ abstract class ConstantObjectInternal extends ObjectInternal {
     /** Gets an AST literal with the same value as this object */
     abstract ImmutableLiteral getLiteral();
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 pragma[nomagic]

python/ql/src/semmle/python/objects/Descriptors.qll

@@ -101,6 +101,8 @@ class PropertyInternal extends ObjectInternal, TProperty {
     /* Properties aren't iterable */
     override ObjectInternal getIterNext() { none() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 private class PropertySetterOrDeleter extends ObjectInternal, TPropertySetterOrDeleter {
@@ -174,6 +176,8 @@ private class PropertySetterOrDeleter extends ObjectInternal, TPropertySetterOrD
 
     override predicate useOriginAsLegacyObject() { none() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 
@@ -267,6 +271,8 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
     /* Classmethods aren't iterable */
     override ObjectInternal getIterNext() { none() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
@@ -345,4 +351,6 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
     /* Staticmethods aren't iterable */
     override ObjectInternal getIterNext() { none() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }

python/ql/src/semmle/python/objects/Instances.qll

@@ -55,6 +55,8 @@ abstract class InstanceObject extends ObjectInternal {
 
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 private predicate self_variable_reaching_init_exit(EssaVariable self) {
@@ -387,6 +389,8 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal {
 
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 private int lengthFromClass(ClassObjectInternal cls) {
@@ -499,5 +503,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal {
 
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 

python/ql/src/semmle/python/objects/Modules.qll

@@ -71,6 +71,8 @@ abstract class ModuleObjectInternal extends ObjectInternal {
         py_exports(this.getSourceModule(), name)
     }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 /** A class representing built-in modules */
@@ -445,5 +447,7 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA
     /* Modules aren't iterable */
     override ObjectInternal getIterNext() { none() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 

python/ql/src/semmle/python/objects/ObjectInternal.qll

@@ -187,6 +187,8 @@ class ObjectInternal extends TObject {
         this.(ObjectInternal).attribute(name, _, _)
     }
 
+    abstract predicate isNotSubscriptedType();
+
 }
 
 
@@ -276,6 +278,8 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject {
 
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 
@@ -359,6 +363,8 @@ class UnknownInternal extends ObjectInternal, TUnknown {
 
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 class UndefinedInternal extends ObjectInternal, TUndefined {
@@ -445,6 +451,8 @@ class UndefinedInternal extends ObjectInternal, TUndefined {
 
     override ObjectInternal getIterNext() { none() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 module ObjectInternal {
@@ -630,6 +638,8 @@ class DecoratedFunction extends ObjectInternal, TDecoratedFunction {
 
     override predicate useOriginAsLegacyObject() { none() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 /** Helper for boolean predicates returning both `true` and `false` */

python/ql/src/semmle/python/objects/Sequences.qll

@@ -131,6 +131,8 @@ class BuiltinTupleObjectInternal extends TBuiltinTuple, TupleObjectInternal {
 
     override predicate useOriginAsLegacyObject() { none() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 /** A tuple declared by a tuple expression in the Python source code */
@@ -164,6 +166,8 @@ class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal {
 
     override predicate useOriginAsLegacyObject() { none() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 /** A tuple created by a `*` parameter */
@@ -195,6 +199,8 @@ class VarargsTupleObjectInternal extends TVarargsTuple,  TupleObjectInternal {
 
     override predicate useOriginAsLegacyObject() { any() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }
 
 
@@ -277,4 +283,6 @@ class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectIntern
 
     override predicate useOriginAsLegacyObject() { any() }
 
+    override predicate isNotSubscriptedType() { any() }
+
 }

python/ql/src/semmle/python/objects/TObject.qll

@@ -240,6 +240,8 @@ cached newtype TObject =
     /* Represents a subscript operation applied to a type. For type-hint analysis */
     TSubscriptedType(ObjectInternal generic, ObjectInternal index) {
         isType(generic) and
+        generic.isNotSubscriptedType() and
+        index.isNotSubscriptedType() and
         Expressions::subscriptPartsPointsTo(_, _, generic, index)
     }
 

python/ql/test/3/library-tests/PointsTo/typehints/Values.expected

@@ -10,3 +10,12 @@
 | test.py:6:1:6:20 | test.py:6 | ControlFlowNode for FunctionExpr | import | test.py:6:1:6:20 | Function bar |
 | test.py:6:11:6:13 | test.py:6 | ControlFlowNode for set | import | file://:0:0:0:0 | builtin-class set |
 | test.py:6:17:6:19 | test.py:6 | ControlFlowNode for Set | import | ../../lib/typing.py:23:1:23:23 | class Set |
+| test.py:9:6:9:13 | test.py:9 | ControlFlowNode for Optional | import | ../../lib/typing.py:18:12:18:32 | _Optional() |
+| test.py:9:6:9:28 | test.py:9 | ControlFlowNode for Subscript | import | file://:0:0:0:0 | _Optional()[Unknown value] |
+| test.py:9:15:9:22 | test.py:9 | ControlFlowNode for Optional | import | ../../lib/typing.py:18:12:18:32 | _Optional() |
+| test.py:9:15:9:27 | test.py:9 | ControlFlowNode for Subscript | import | file://:0:0:0:0 | _Optional()[builtin-class int] |
+| test.py:9:24:9:26 | test.py:9 | ControlFlowNode for int | import | file://:0:0:0:0 | builtin-class int |
+| test.py:10:6:10:13 | test.py:10 | ControlFlowNode for Optional | import | ../../lib/typing.py:18:12:18:32 | _Optional() |
+| test.py:10:6:10:18 | test.py:10 | ControlFlowNode for Subscript | import | file://:0:0:0:0 | _Optional()[builtin-class int] |
+| test.py:10:15:10:17 | test.py:10 | ControlFlowNode for int | import | file://:0:0:0:0 | builtin-class int |
+| test.py:10:20:10:22 | test.py:10 | ControlFlowNode for int | import | file://:0:0:0:0 | builtin-class int |

python/ql/test/3/library-tests/PointsTo/typehints/test.py

@@ -5,3 +5,14 @@ def foo(x:Optional[int]) -> int:
 
 def bar(s:set)->Set:
     pass
+
+t1 = Optional[Optional[int]]
+t2 = Optional[int][int]
+
+# ODASA-8075
+# Commented out until the fix has been pushed to LGTM.com
+#class baz():
+#    pass
+#
+#while True:
+#    baz = baz[baz]