Merge pull request #14848 from hvitved/python/shared-type-tracking

Python: Adopt shared type tracking library

Author: yoff

config/identical-files.json

@@ -454,10 +454,6 @@
     "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll",
     "swift/ql/lib/codeql/swift/security/internal/SensitiveDataHeuristics.qll"
   ],
-  "SummaryTypeTracker": [
-    "python/ql/lib/semmle/python/dataflow/new/internal/SummaryTypeTracker.qll",
-    "ruby/ql/lib/codeql/ruby/typetracking/internal/SummaryTypeTracker.qll"
-  ],
   "IncompleteUrlSubstringSanitization": [
     "javascript/ql/src/Security/CWE-020/IncompleteUrlSubstringSanitization.qll",
     "ruby/ql/src/queries/security/cwe-020/IncompleteUrlSubstringSanitization.qll"

python/ql/lib/change-notes/2023-11-21-new-type-tracking-lib.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Python now makes use of the shared type tracking library, exposed as `semmle.python.dataflow.new.TypeTracking`. The existing type tracking library, `semmle.python.dataflow.new.TypeTracker`, has consequently been deprecated.

python/ql/lib/experimental/cryptography/utils/CallCfgNodeWithTarget.qll

@@ -4,9 +4,9 @@
  */
 
 import python
-private import semmle.python.dataflow.new.internal.TypeTrackerSpecific
+private import semmle.python.dataflow.new.internal.TypeTrackingImpl
 private import semmle.python.ApiGraphs
 
 class CallCfgNodeWithTarget extends DataFlow::Node instanceof DataFlow::CallCfgNode {
-  DataFlow::Node getTarget() { returnStep(result, this) }
+  DataFlow::Node getTarget() { TypeTrackingInput::returnStep(result, this) }
 }

python/ql/lib/semmle/python/dataflow/new/TypeTracker.qll

@@ -1,4 +1,6 @@
 /**
+ * DEPRECATED: Use `semmle.python.dataflow.new.TypeTracking` instead.
+ *
  * This file acts as a wrapper for `internal.TypeTracker`, exposing some of the functionality with
  * names that are more appropriate for Python.
  */
@@ -8,12 +10,14 @@ private import internal.TypeTracker as Internal
 private import internal.TypeTrackerSpecific as InternalSpecific
 
 /** A string that may appear as the name of an attribute or access path. */
-class AttributeName = InternalSpecific::TypeTrackerContent;
+deprecated class AttributeName = InternalSpecific::TypeTrackerContent;
 
 /** An attribute name, or the empty string (representing no attribute). */
-class OptionalAttributeName = InternalSpecific::OptionalTypeTrackerContent;
+deprecated class OptionalAttributeName = InternalSpecific::OptionalTypeTrackerContent;
 
 /**
+ * DEPRECATED: Use `semmle.python.dataflow.new.TypeTracking` instead.
+ *
  * The summary of the steps needed to track a value to a given dataflow node.
  *
  * This can be used to track objects that implement a certain API in order to
@@ -40,7 +44,7 @@ class OptionalAttributeName = InternalSpecific::OptionalTypeTrackerContent;
  * `t = t2.step(myType(t2), result)`. If you additionally want to track individual
  * intra-procedural steps, use `t = t2.smallstep(myCallback(t2), result)`.
  */
-class TypeTracker extends Internal::TypeTracker {
+deprecated class TypeTracker extends Internal::TypeTracker {
   /**
    * Holds if this is the starting point of type tracking, and the value starts in the attribute named `attrName`.
    * The type tracking only ends after the attribute has been loaded.
@@ -55,12 +59,12 @@ class TypeTracker extends Internal::TypeTracker {
   string getAttr() { result = this.getContent() }
 }
 
-module TypeTracker = Internal::TypeTracker;
+deprecated module TypeTracker = Internal::TypeTracker;
 
-class StepSummary = Internal::StepSummary;
+deprecated class StepSummary = Internal::StepSummary;
 
-module StepSummary = Internal::StepSummary;
+deprecated module StepSummary = Internal::StepSummary;
 
-class TypeBackTracker = Internal::TypeBackTracker;
+deprecated class TypeBackTracker = Internal::TypeBackTracker;
 
-module TypeBackTracker = Internal::TypeBackTracker;
+deprecated module TypeBackTracker = Internal::TypeBackTracker;

python/ql/lib/semmle/python/dataflow/new/TypeTracking.qll

@@ -0,0 +1,56 @@
+/**
+ * Provides classes and predicates for simple data-flow reachability suitable
+ * for tracking types.
+ */
+
+private import internal.TypeTrackingImpl as Impl
+import Impl::Shared::TypeTracking<Impl::TypeTrackingInput>
+
+/** A string that may appear as the name of an attribute or access path. */
+class AttributeName = Impl::TypeTrackingInput::Content;
+
+/**
+ * A summary of the steps needed to track a value to a given dataflow node.
+ *
+ * This can be used to track objects that implement a certain API in order to
+ * recognize calls to that API. Note that type-tracking does not by itself provide a
+ * source/sink relation, that is, it may determine that a node has a given type,
+ * but it won't determine where that type came from.
+ *
+ * It is recommended that all uses of this type are written in the following form,
+ * for tracking some type `myType`:
+ * ```ql
+ * Node myType(TypeTracker tt) {
+ *   tt.start() and
+ *   result = < source of myType >
+ *   or
+ *   exists(TypeTracker tt2 |
+ *     tt = tt2.step(myType(tt2), result)
+ *   )
+ * }
+ *
+ * Node myType() { myType(TypeTracker::end()).flowsTo(result) }
+ * ```
+ *
+ * If you want to track individual intra-procedural steps, use `tt2.smallstep`
+ * instead of `tt2.step`.
+ */
+class TypeTracker extends Impl::TypeTracker {
+  /**
+   * Holds if this is the starting point of type tracking, and the value starts in the attribute named `attrName`.
+   * The type tracking only ends after the attribute has been loaded.
+   */
+  predicate startInAttr(string attrName) { this.startInContent(attrName) }
+
+  /**
+   * INTERNAL. DO NOT USE.
+   *
+   * Gets the attribute associated with this type tracker.
+   */
+  string getAttr() {
+    result = this.getContent().asSome()
+    or
+    this.getContent().isNone() and
+    result = ""
+  }
+}

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowDispatch.qll

@@ -37,7 +37,7 @@ private import DataFlowPublic
 private import DataFlowPrivate
 private import FlowSummaryImpl as FlowSummaryImpl
 private import semmle.python.internal.CachedStages
-private import semmle.python.dataflow.new.internal.TypeTracker::CallGraphConstruction as CallGraphConstruction
+private import semmle.python.dataflow.new.internal.TypeTrackingImpl::CallGraphConstruction as CallGraphConstruction
 
 newtype TParameterPosition =
   /** Used for `self` in methods, and `cls` in classmethods. */

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -4,7 +4,7 @@
 
 private import python
 private import DataFlowPrivate
-import semmle.python.dataflow.new.TypeTracker
+import semmle.python.dataflow.new.TypeTracking
 import Attributes
 import LocalSources
 private import semmle.python.essa.SsaCompute

python/ql/lib/semmle/python/dataflow/new/internal/ImportResolution.qll

@@ -7,7 +7,7 @@
 private import python
 private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.dataflow.new.internal.ImportStar
-private import semmle.python.dataflow.new.TypeTracker
+private import semmle.python.dataflow.new.TypeTracking
 private import semmle.python.dataflow.new.internal.DataFlowPrivate
 
 /**

python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll

@@ -72,6 +72,8 @@ class LocalSourceNode extends Node {
     // We include all scope entry definitions, as these act as the local source within the scope they
     // enter.
     this.asCfgNode() = any(ScopeEntryDefinition def).getDefiningNode()
+    or
+    this instanceof ParameterNode
   }
 
   /** Holds if this `LocalSourceNode` can flow to `nodeTo` in one or more local flow steps. */
@@ -151,7 +153,7 @@ class LocalSourceNode extends Node {
    * See `TypeBackTracker` for more details about how to use this.
    */
   pragma[inline]
-  LocalSourceNode backtrack(TypeBackTracker t2, TypeBackTracker t) { t2 = t.step(result, this) }
+  LocalSourceNode backtrack(TypeBackTracker t2, TypeBackTracker t) { t = t2.step(result, this) }
 }
 
 /**
@@ -238,7 +240,7 @@ private module Cached {
    * Helper predicate for `hasLocalSource`. Removes any steps go to module variable reads, as these
    * are already local source nodes in their own right.
    */
-  cached
+  pragma[nomagic]
   private predicate localSourceFlowStep(Node nodeFrom, Node nodeTo) {
     simpleLocalFlowStep(nodeFrom, nodeTo) and
     not nodeTo = any(ModuleVariableNode v).getARead()