Python: Enhance points-to to support type-hint analysis.

python/ql/src/semmle/python/objects/Classes.qll

@@ -365,3 +365,81 @@ class DynamicallyCreatedClass extends ClassObjectInternal, TDynamicClass {
 
 }
 
+class SubscriptedTypeInternal extends ObjectInternal, TSubscriptedType {
+
+    ObjectInternal getGeneric() {
+        this = TSubscriptedType(result, _)
+    }
+
+    ObjectInternal getSpecializer() {
+        this = TSubscriptedType(_, result)
+    }
+
+    override string getName() { result = this.getGeneric().getName() }
+
+    override string toString() { result = this.getGeneric().toString() + "[" + this.getSpecializer().toString() + "]" }
+
+    override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
+        exists(ObjectInternal generic, ObjectInternal index |
+            this = TSubscriptedType(generic, index) and
+            Expressions::subscriptPartsPointsTo(node, context, generic, index)
+        )
+    }
+
+    /** Gets the class declaration for this object, if it is a class with a declaration. */
+    override ClassDecl getClassDeclaration() {
+        result = this.getGeneric().getClassDeclaration()
+    }
+
+    /** True if this "object" is a class. That is, its class inherits from `type` */
+    override boolean isClass() { result = true }
+
+    override ObjectInternal getClass() {
+        result = this.getGeneric().getClass()
+    }
+
+    override predicate notTestableForEquality() { none() }
+
+    override Builtin getBuiltin() { none() }
+
+    override ControlFlowNode getOrigin() { none() }
+
+    override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() }
+
+    override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() }
+
+    override predicate calleeAndOffset(Function scope, int paramOffset){ none() }
+
+    override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate attributesUnknown() { none() }
+
+    override boolean isDescriptor() { result = false }
+
+    override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
+
+    override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
+
+    override int length() { none() }
+
+    override boolean booleanValue() { result = true }
+
+    override int intValue() { none()}
+
+    override string strValue() { none() }
+
+    override predicate subscriptUnknown() { none() }
+
+    override predicate contextSensitiveCallee() { none() }
+
+    override predicate useOriginAsLegacyObject() { none() }
+
+    /* Classes aren't usually iterable, but can e.g. Enums */
+    override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
+
+}
+
+
+

python/ql/src/semmle/python/objects/Instances.qll

@@ -91,7 +91,14 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject {
     override predicate notTestableForEquality() { none() }
 
     override ObjectInternal getClass() {
-        this = TSpecificInstance(_, result, _)
+        exists(ClassObjectInternal cls, ClassDecl decl |
+            this = TSpecificInstance(_, cls, _) and
+            decl = cls.getClassDeclaration() |
+            if decl.callReturnsInstance() then
+                result = cls
+            else
+               result = TUnknownClass()
+        )
     }
 
     /** Gets the `Builtin` for this object, if any.

python/ql/src/semmle/python/objects/ObjectAPI.qll

@@ -322,7 +322,7 @@ class ClassValue extends Value {
 
     /** Gets an improper super type of this class. */
     ClassValue getASuperType() {
-        result = Types::getMro(this).getAnItem()
+        result = this.getABaseType*()
     }
 
     /** Looks up the attribute `name` on this class.
@@ -367,6 +367,11 @@ class ClassValue extends Value {
         result = Types::getBase(this, n)
     }
 
+    /** Gets a base class of this class */
+    ClassValue getABaseType() {
+        result = Types::getBase(this, _)
+    }
+
     /** Holds if this class is a new style class.
         A new style class is one that implicitly or explicitly inherits from `object`. */
     predicate isNewStyle() {

python/ql/src/semmle/python/objects/ObjectInternal.qll

@@ -557,6 +557,8 @@ class DecoratedFunction extends ObjectInternal, TDecoratedFunction {
 
     override string toString() {
         result = "Decorated " + this.decoratedObject().toString()
+        or
+        not exists(this.decoratedObject()) and result = "Decorated function"
     }
 
     override boolean booleanValue() { result = true }

python/ql/src/semmle/python/objects/Sequences.qll

@@ -49,7 +49,14 @@ abstract class TupleObjectInternal extends SequenceObjectInternal {
         or
         n = 3 and this.length() > 3 and result = (this.length()-3).toString() + " more..."
         or
-        result = this.getItem(n).toString() + ", " + this.contents(n+1)
+        result = this.item(n) + ", " + this.contents(n+1)
+    }
+
+    private string item(int n) {
+        result = this.getItem(n).toString()
+        or
+        n in [0..this.length()-1] and
+        not exists(this.getItem(n)) and result = "?"
     }
 
     /** Gets the class declaration for this object, if it is a declared class. */

python/ql/src/semmle/python/objects/TObject.qll

@@ -132,7 +132,7 @@ cached newtype TObject =
     /* An instance of `cls`, instantiated at `instantiation` given the `context`. */
     TSpecificInstance(ControlFlowNode instantiation, ClassObjectInternal cls, PointsToContext context) {
         PointsToInternal::pointsTo(instantiation.(CallNode).getFunction(), context, cls, _) and
-        cls.isSpecial() = false and cls.getClassDeclaration().callReturnsInstance()
+        cls.isSpecial() = false
         or
         literal_instantiation(instantiation, cls, context)
     }
@@ -236,6 +236,18 @@ cached newtype TObject =
     TDecoratedFunction(CallNode call) {
         call.isFunctionDecoratorCall()
     }
+    or
+    /* Represents a subscript operation applied to a type. For type-hint analysis */
+    TSubscriptedType(ObjectInternal generic, ObjectInternal index) {
+        isType(generic) and
+        Expressions::subscriptPartsPointsTo(_, _, generic, index)
+    }
+
+predicate isType(ObjectInternal t) {
+    t.isClass() = true
+    or
+    t.getOrigin().getEnclosingModule().getName().matches("%typing")
+}
 
 private predicate is_power_2(int n) {
     n = 1 or

python/ql/src/semmle/python/pointsto/PointsTo.qll

@@ -1229,6 +1229,13 @@ module Expressions {
         origin = subscr
     }
 
+    predicate subscriptPartsPointsTo(SubscriptNode subscr, PointsToContext context, ObjectInternal objvalue, ObjectInternal indexvalue) {
+        exists(ControlFlowNode index |
+            subscriptObjectAndIndex(subscr, context, _, objvalue, index) and
+            PointsToInternal::pointsTo(index, context, indexvalue, _)
+        )
+    }
+
     pragma [noinline]
     private predicate subscriptObjectAndIndex(SubscriptNode subscr, PointsToContext context, ControlFlowNode obj, ObjectInternal objvalue, ControlFlowNode index) {
         subscr.isLoad() and

python/ql/test/3/library-tests/PointsTo/typehints/Values.expected

@@ -0,0 +1,12 @@
+| test.py:1:6:1:11 | test.py:1 | ControlFlowNode for ImportExpr | import | ../../lib/typing.py:0:0:0:0 | Module typing |
+| test.py:1:6:1:11 | test.py:1 | ControlFlowNode for ImportExpr | import | ../../lib/typing.py:0:0:0:0 | Module typing |
+| test.py:1:20:1:27 | test.py:1 | ControlFlowNode for ImportMember | import | ../../lib/typing.py:18:12:18:32 | _Optional() |
+| test.py:1:30:1:32 | test.py:1 | ControlFlowNode for ImportMember | import | ../../lib/typing.py:23:1:23:23 | class Set |
+| test.py:3:1:3:32 | test.py:3 | ControlFlowNode for FunctionExpr | import | test.py:3:1:3:32 | Function foo |
+| test.py:3:11:3:18 | test.py:3 | ControlFlowNode for Optional | import | ../../lib/typing.py:18:12:18:32 | _Optional() |
+| test.py:3:11:3:23 | test.py:3 | ControlFlowNode for Subscript | import | file://:0:0:0:0 | _Optional()[builtin-class int] |
+| test.py:3:20:3:22 | test.py:3 | ControlFlowNode for int | import | file://:0:0:0:0 | builtin-class int |
+| test.py:3:29:3:31 | test.py:3 | ControlFlowNode for int | import | file://:0:0:0:0 | builtin-class int |
+| test.py:6:1:6:20 | test.py:6 | ControlFlowNode for FunctionExpr | import | test.py:6:1:6:20 | Function bar |
+| test.py:6:11:6:13 | test.py:6 | ControlFlowNode for set | import | file://:0:0:0:0 | builtin-class set |
+| test.py:6:17:6:19 | test.py:6 | ControlFlowNode for Set | import | ../../lib/typing.py:23:1:23:23 | class Set |

python/ql/test/3/library-tests/PointsTo/typehints/Values.ql

@@ -0,0 +1,8 @@
+
+import python
+
+from ControlFlowNode f, Context ctx, Value v, ControlFlowNode origin
+where
+  f.pointsTo(ctx, v, origin) and
+  f.getLocation().getFile().getBaseName() = "test.py"
+select f.getLocation(), f.toString(), ctx, v

python/ql/test/3/library-tests/PointsTo/typehints/options

@@ -0,0 +1,2 @@
+semmle-extractor-options: -p ../../lib/ --max-import-depth=3
+optimize: true

python/ql/test/3/library-tests/PointsTo/typehints/test.py

@@ -0,0 +1,7 @@
+from typing import Optional, Set
+
+def foo(x:Optional[int]) -> int:
+    pass
+
+def bar(s:set)->Set:
+    pass

python/ql/test/3/library-tests/lib/typing.py

@@ -0,0 +1,29 @@
+#Fake typing module for testing.
+
+class ComplexMetaclass(type):
+
+    def __new__(self):
+        pass
+
+class ComplexBaseClass(metaclass=ComplexMetaclass):
+
+    def __new__(self):
+        pass
+
+class _Optional(ComplexBaseClass, extras=...):
+
+    def __new__(self):
+        pass
+
+Optional = _Optional("Optional")
+
+class Collections(ComplexBaseClass, extras=...):
+    pass
+
+class Set(Collections):
+    pass
+
+class List(Collections):
+    pass
+
+Optional

python/ql/test/library-tests/PointsTo/general/GlobalPointsTo.expected

@@ -143,6 +143,8 @@
 | Module pointsto_test | 161 | ControlFlowNode for ClassExpr | class Derived3 |
 | Module pointsto_test | 161 | ControlFlowNode for Derived3 | class Derived3 |
 | Module pointsto_test | 164 | ControlFlowNode for Base | class Base |
+| Module pointsto_test | 164 | ControlFlowNode for Base() | Base() |
+| Module pointsto_test | 164 | ControlFlowNode for thing | Base() |
 | Module pointsto_test | 167 | ControlFlowNode for FunctionExpr | Function multiple_assignment |
 | Module pointsto_test | 167 | ControlFlowNode for multiple_assignment | Function multiple_assignment |
 | Module pointsto_test | 173 | ControlFlowNode for Base2 | class Base2 |

python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected

@@ -233,6 +233,8 @@
 | 161 | ControlFlowNode for ClassExpr | class Derived3 |
 | 161 | ControlFlowNode for Derived3 | class Derived3 |
 | 164 | ControlFlowNode for Base | class Base |
+| 164 | ControlFlowNode for Base() | Base() |
+| 164 | ControlFlowNode for thing | Base() |
 | 167 | ControlFlowNode for FunctionExpr | Function multiple_assignment |
 | 167 | ControlFlowNode for multiple_assignment | Function multiple_assignment |
 | 168 | ControlFlowNode for Tuple | Tuple |

python/ql/test/library-tests/PointsTo/new/NameSpace.expected

@@ -79,6 +79,7 @@
 | h_classes.py:0 | Module code.h_classes | f | Function f |
 | h_classes.py:0 | Module code.h_classes | k | Function k |
 | h_classes.py:0 | Module code.h_classes | sys | Module sys |
+| h_classes.py:0 | Module code.h_classes | thing | Base() |
 | h_classes.py:3 | Class C | __init__ | Function __init__ |
 | h_classes.py:3 | Class C | x | 'C_x' |
 | h_classes.py:23 | Class Base | __init__ | Function __init__ |

python/ql/test/library-tests/PointsTo/new/PointsToWithContext.expected

@@ -403,6 +403,8 @@ WARNING: Predicate points_to has been deprecated and may be removed in future (P
 | h_classes.py:39 | ControlFlowNode for ClassExpr | class Derived3 | builtin-class type | 39 | import |
 | h_classes.py:39 | ControlFlowNode for Derived3 | class Derived3 | builtin-class type | 39 | import |
 | h_classes.py:42 | ControlFlowNode for Base | class Base | builtin-class type | 23 | import |
+| h_classes.py:42 | ControlFlowNode for Base() | Base() | *UNKNOWN TYPE* | 42 | import |
+| h_classes.py:42 | ControlFlowNode for thing | Base() | *UNKNOWN TYPE* | 42 | import |
 | h_classes.py:45 | ControlFlowNode for FunctionExpr | Function f | builtin-class function | 45 | import |
 | h_classes.py:45 | ControlFlowNode for f | Function f | builtin-class function | 45 | import |
 | h_classes.py:48 | ControlFlowNode for ClassExpr | class D | builtin-class type | 48 | import |

python/ql/test/library-tests/PointsTo/new/PointsToWithType.expected

@@ -500,6 +500,8 @@ WARNING: Predicate points_to has been deprecated and may be removed in future (P
 | h_classes.py:39 | ControlFlowNode for ClassExpr | class Derived3 | builtin-class type | 39 |
 | h_classes.py:39 | ControlFlowNode for Derived3 | class Derived3 | builtin-class type | 39 |
 | h_classes.py:42 | ControlFlowNode for Base | class Base | builtin-class type | 23 |
+| h_classes.py:42 | ControlFlowNode for Base() | Base() | *UNKNOWN TYPE* | 42 |
+| h_classes.py:42 | ControlFlowNode for thing | Base() | *UNKNOWN TYPE* | 42 |
 | h_classes.py:45 | ControlFlowNode for FunctionExpr | Function f | builtin-class function | 45 |
 | h_classes.py:45 | ControlFlowNode for f | Function f | builtin-class function | 45 |
 | h_classes.py:48 | ControlFlowNode for ClassExpr | class D | builtin-class type | 48 |