Skip to content

Commit 95c2427

Browse files
RasmusWLRasmusWL
authored
Merge pull request #15044 from RasmusWL/automated-subclass-models
Python: Automated subclass models
2 parents 8810f16 + 72687e0 commit 95c2427

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+176310
-294
lines changed

‎.gitattributes

+3
Original file line numberDiff line numberDiff line change
@@ -71,3 +71,6 @@ go/extractor/opencsv/CSVReader.java -text
7171
# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
7272
javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
7373
javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge
74+
75+
# Auto-generated modeling for Python
76+
python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true

‎python/ql/lib/change-notes/2023-12-08-automated-subclass-models.md

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
category: minorAnalysis
3+
---
4+
* Captured subclass relationships ahead-of-time for most popular PyPI packages so we are able to resolve subclass relationships even without having the packages installed. For example we have captured that `flask_restful.Resource` is a subclass of `flask.views.MethodView`, so our Flask modeling will still consider a function named `post` on a `class Foo(flask_restful.Resource):` as a HTTP request handler.

‎python/ql/lib/semmle/python/Module.qll

+1-1
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ private predicate legalDottedName(string name) {
177177
}
178178

179179
bindingset[name]
180-
private predicate legalShortName(string name) { name.regexpMatch("(\\p{L}|_)(\\p{L}|\\d|_)*") }
180+
predicate legalShortName(string name) { name.regexpMatch("(\\p{L}|_)(\\p{L}|\\d|_)*") }
181181

182182
private string moduleNameFromBase(Container file) {
183183
// We used to also require `isPotentialPackage(f)` to hold in this case,

‎python/ql/lib/semmle/python/frameworks/Aioch.qll

+3
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ private import python
99
private import semmle.python.Concepts
1010
private import semmle.python.ApiGraphs
1111
private import semmle.python.frameworks.ClickhouseDriver
12+
private import semmle.python.frameworks.data.ModelsAsData
1213

1314
/**
1415
* INTERNAL: Do not use.
@@ -24,6 +25,8 @@ module Aioch {
2425
/** Gets a reference to the `aioch.Client` class or any subclass. */
2526
API::Node subclassRef() {
2627
result = API::moduleImport("aioch").getMember("Client").getASubclass*()
28+
or
29+
result = ModelOutput::getATypeNode("aioch.Client~Subclass").getASubclass*()
2730
}
2831

2932
/** Gets a reference to an instance of `clickhouse_driver.Client` or any subclass. */

‎python/ql/lib/semmle/python/frameworks/Aiohttp.qll

+9-2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ private import semmle.python.frameworks.internal.SelfRefMixin
1414
private import semmle.python.frameworks.Multidict
1515
private import semmle.python.frameworks.Yarl
1616
private import semmle.python.frameworks.internal.InstanceTaintStepsHelper
17+
private import semmle.python.frameworks.data.ModelsAsData
1718

1819
/**
1920
* INTERNAL: Do not use.
@@ -31,6 +32,8 @@ module AiohttpWebModel {
3132
/** Gets a reference to the `aiohttp.web.View` class or any subclass. */
3233
API::Node subclassRef() {
3334
result = API::moduleImport("aiohttp").getMember("web").getMember("View").getASubclass*()
35+
or
36+
result = ModelOutput::getATypeNode("aiohttp.web.View~Subclass").getASubclass*()
3437
}
3538
}
3639

@@ -706,19 +709,23 @@ module AiohttpWebModel {
706709
}
707710

708711
/**
712+
* INTERNAL: Do not use.
713+
*
709714
* Provides models for the web server part (`aiohttp.client`) of the `aiohttp` PyPI package.
710715
* See https://docs.aiohttp.org/en/stable/client.html
711716
*/
712-
private module AiohttpClientModel {
717+
module AiohttpClientModel {
713718
/**
714719
* Provides models for the `aiohttp.ClientSession` class
715720
*
716721
* See https://docs.aiohttp.org/en/stable/client_reference.html#aiohttp.ClientSession.
717722
*/
718723
module ClientSession {
719724
/** Gets a reference to the `aiohttp.ClientSession` class. */
720-
private API::Node classRef() {
725+
API::Node classRef() {
721726
result = API::moduleImport("aiohttp").getMember("ClientSession")
727+
or
728+
result = ModelOutput::getATypeNode("aiohttp.ClientSession~Subclass").getASubclass*()
722729
}
723730

724731
/** Gets a reference to an instance of `aiohttp.ClientSession`. */

‎python/ql/lib/semmle/python/frameworks/ClickhouseDriver.qll

+4
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ private import python
99
private import semmle.python.Concepts
1010
private import semmle.python.ApiGraphs
1111
private import semmle.python.frameworks.PEP249
12+
private import semmle.python.frameworks.data.ModelsAsData
1213

1314
/**
1415
* INTERNAL: Do not use.
@@ -37,6 +38,9 @@ module ClickhouseDriver {
3738
or
3839
// commonly used alias
3940
classRef = API::moduleImport("clickhouse_driver").getMember("Client")
41+
or
42+
// Models-as-Data subclass
43+
classRef = ModelOutput::getATypeNode("clickhouse_driver.client.Client~Subclass")
4044
|
4145
result = classRef.getASubclass*()
4246
)

‎python/ql/lib/semmle/python/frameworks/Django.qll

+104-13
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ private import semmle.python.frameworks.internal.PoorMansFunctionResolution
1616
private import semmle.python.frameworks.internal.SelfRefMixin
1717
private import semmle.python.frameworks.internal.InstanceTaintStepsHelper
1818
private import semmle.python.security.dataflow.UrlRedirectCustomizations
19+
private import semmle.python.frameworks.data.ModelsAsData
1920

2021
/**
2122
* INTERNAL: Do not use.
@@ -85,6 +86,10 @@ module Django {
8586
}
8687
}
8788

89+
private class MaDSubclass extends ModeledSubclass {
90+
MaDSubclass() { this = ModelOutput::getATypeNode("Django.Views.View~Subclass") }
91+
}
92+
8893
/** Gets a reference to the `django.views.generic.View` class or any subclass. */
8994
API::Node subclassRef() { result = any(ModeledSubclass subclass).getASubclass*() }
9095
}
@@ -185,6 +190,10 @@ module Django {
185190
}
186191
}
187192

193+
private class MaDSubclass extends ModeledSubclass {
194+
MaDSubclass() { this = ModelOutput::getATypeNode("django.forms.BaseForm~Subclass") }
195+
}
196+
188197
/** Gets a reference to the `django.forms.forms.BaseForm` class or any subclass. */
189198
API::Node subclassRef() { result = any(ModeledSubclass subclass).getASubclass*() }
190199
}
@@ -290,6 +299,10 @@ module Django {
290299
}
291300
}
292301

302+
private class MaDSubclass extends ModeledSubclass {
303+
MaDSubclass() { this = ModelOutput::getATypeNode("Django.Forms.Field~Subclass") }
304+
}
305+
293306
/** Gets a reference to the `django.forms.fields.Field` class or any subclass. */
294307
API::Node subclassRef() { result = any(ModeledSubclass subclass).getASubclass*() }
295308
}
@@ -596,6 +609,8 @@ module PrivateDjango {
596609
.getMember("models")
597610
.getMember("PolymorphicModel")
598611
.getASubclass*()
612+
or
613+
result = ModelOutput::getATypeNode("Django.db.models.Model~Subclass").getASubclass*()
599614
}
600615

601616
/**
@@ -766,6 +781,9 @@ module PrivateDjango {
766781
.getMember(className)
767782
.getASubclass*()
768783
)
784+
or
785+
result =
786+
ModelOutput::getATypeNode("django.db.models.FileField~Subclass").getASubclass*()
769787
}
770788
}
771789

@@ -823,6 +841,10 @@ module PrivateDjango {
823841
or
824842
// Commonly used alias
825843
result = models().getMember("RawSQL")
844+
or
845+
result =
846+
ModelOutput::getATypeNode("django.db.models.expressions.RawSQL~Subclass")
847+
.getASubclass*()
826848
}
827849

828850
/**
@@ -1157,6 +1179,9 @@ module PrivateDjango {
11571179
or
11581180
// handle django.http.HttpRequest alias
11591181
result = http().getMember("HttpRequest")
1182+
or
1183+
result =
1184+
ModelOutput::getATypeNode("django.http.request.HttpRequest~Subclass").getASubclass*()
11601185
}
11611186

11621187
/**
@@ -1322,7 +1347,13 @@ module PrivateDjango {
13221347
}
13231348

13241349
/** Gets a reference to the `django.http.response.HttpResponse` class or any subclass. */
1325-
API::Node classRef() { result = baseClassRef().getASubclass*() }
1350+
API::Node classRef() {
1351+
result = baseClassRef().getASubclass*()
1352+
or
1353+
result =
1354+
ModelOutput::getATypeNode("django.http.response.HttpResponse~Subclass")
1355+
.getASubclass*()
1356+
}
13261357

13271358
/**
13281359
* A source of instances of `django.http.response.HttpResponse`, extend this class to model new instances.
@@ -1383,7 +1414,12 @@ module PrivateDjango {
13831414
}
13841415

13851416
/** Gets a reference to a subclass of the `django.http.response.HttpResponseRedirect` class. */
1386-
API::Node classRef() { result = baseClassRef().getASubclass*() }
1417+
API::Node classRef() {
1418+
result = baseClassRef().getASubclass*() or
1419+
result =
1420+
ModelOutput::getATypeNode("django.http.response.HttpResponseRedirect~Subclass")
1421+
.getASubclass*()
1422+
}
13871423

13881424
/**
13891425
* A source of instances of `django.http.response.HttpResponseRedirect`, extend this class to model new instances.
@@ -1446,7 +1482,12 @@ module PrivateDjango {
14461482
}
14471483

14481484
/** Gets a reference to the `django.http.response.HttpResponsePermanentRedirect` class. */
1449-
API::Node classRef() { result = baseClassRef().getASubclass*() }
1485+
API::Node classRef() {
1486+
result = baseClassRef().getASubclass*() or
1487+
result =
1488+
ModelOutput::getATypeNode("django.http.response.HttpResponsePermanentRedirect~Subclass")
1489+
.getASubclass*()
1490+
}
14501491

14511492
/**
14521493
* A source of instances of `django.http.response.HttpResponsePermanentRedirect`, extend this class to model new instances.
@@ -1510,7 +1551,12 @@ module PrivateDjango {
15101551
}
15111552

15121553
/** Gets a reference to the `django.http.response.HttpResponseNotModified` class. */
1513-
API::Node classRef() { result = baseClassRef().getASubclass*() }
1554+
API::Node classRef() {
1555+
result = baseClassRef().getASubclass*() or
1556+
result =
1557+
ModelOutput::getATypeNode("django.http.response.HttpResponseNotModified~Subclass")
1558+
.getASubclass*()
1559+
}
15141560

15151561
/**
15161562
* A source of instances of `django.http.response.HttpResponseNotModified`, extend this class to model new instances.
@@ -1562,7 +1608,12 @@ module PrivateDjango {
15621608
}
15631609

15641610
/** Gets a reference to the `django.http.response.HttpResponseBadRequest` class. */
1565-
API::Node classRef() { result = baseClassRef().getASubclass*() }
1611+
API::Node classRef() {
1612+
result = baseClassRef().getASubclass*() or
1613+
result =
1614+
ModelOutput::getATypeNode("django.http.response.HttpResponseBadRequest~Subclass")
1615+
.getASubclass*()
1616+
}
15661617

15671618
/**
15681619
* A source of instances of `django.http.response.HttpResponseBadRequest`, extend this class to model new instances.
@@ -1616,7 +1667,12 @@ module PrivateDjango {
16161667
}
16171668

16181669
/** Gets a reference to the `django.http.response.HttpResponseNotFound` class. */
1619-
API::Node classRef() { result = baseClassRef().getASubclass*() }
1670+
API::Node classRef() {
1671+
result = baseClassRef().getASubclass*() or
1672+
result =
1673+
ModelOutput::getATypeNode("django.http.response.HttpResponseNotFound~Subclass")
1674+
.getASubclass*()
1675+
}
16201676

16211677
/**
16221678
* A source of instances of `django.http.response.HttpResponseNotFound`, extend this class to model new instances.
@@ -1670,7 +1726,12 @@ module PrivateDjango {
16701726
}
16711727

16721728
/** Gets a reference to the `django.http.response.HttpResponseForbidden` class. */
1673-
API::Node classRef() { result = baseClassRef().getASubclass*() }
1729+
API::Node classRef() {
1730+
result = baseClassRef().getASubclass*() or
1731+
result =
1732+
ModelOutput::getATypeNode("django.http.response.HttpResponseForbidden~Subclass")
1733+
.getASubclass*()
1734+
}
16741735

16751736
/**
16761737
* A source of instances of `django.http.response.HttpResponseForbidden`, extend this class to model new instances.
@@ -1724,7 +1785,12 @@ module PrivateDjango {
17241785
}
17251786

17261787
/** Gets a reference to the `django.http.response.HttpResponseNotAllowed` class. */
1727-
API::Node classRef() { result = baseClassRef().getASubclass*() }
1788+
API::Node classRef() {
1789+
result = baseClassRef().getASubclass*() or
1790+
result =
1791+
ModelOutput::getATypeNode("django.http.response.HttpResponseNotAllowed~Subclass")
1792+
.getASubclass*()
1793+
}
17281794

17291795
/**
17301796
* A source of instances of `django.http.response.HttpResponseNotAllowed`, extend this class to model new instances.
@@ -1779,7 +1845,12 @@ module PrivateDjango {
17791845
}
17801846

17811847
/** Gets a reference to the `django.http.response.HttpResponseGone` class. */
1782-
API::Node classRef() { result = baseClassRef().getASubclass*() }
1848+
API::Node classRef() {
1849+
result = baseClassRef().getASubclass*() or
1850+
result =
1851+
ModelOutput::getATypeNode("django.http.response.HttpResponseGone~Subclass")
1852+
.getASubclass*()
1853+
}
17831854

17841855
/**
17851856
* A source of instances of `django.http.response.HttpResponseGone`, extend this class to model new instances.
@@ -1833,7 +1904,12 @@ module PrivateDjango {
18331904
}
18341905

18351906
/** Gets a reference to the `django.http.response.HttpResponseServerError` class. */
1836-
API::Node classRef() { result = baseClassRef().getASubclass*() }
1907+
API::Node classRef() {
1908+
result = baseClassRef().getASubclass*() or
1909+
result =
1910+
ModelOutput::getATypeNode("django.http.response.HttpResponseServerError~Subclass")
1911+
.getASubclass*()
1912+
}
18371913

18381914
/**
18391915
* A source of instances of `django.http.response.HttpResponseServerError`, extend this class to model new instances.
@@ -1887,7 +1963,12 @@ module PrivateDjango {
18871963
}
18881964

18891965
/** Gets a reference to the `django.http.response.JsonResponse` class. */
1890-
API::Node classRef() { result = baseClassRef().getASubclass*() }
1966+
API::Node classRef() {
1967+
result = baseClassRef().getASubclass*() or
1968+
result =
1969+
ModelOutput::getATypeNode("django.http.response.JsonResponse~Subclass")
1970+
.getASubclass*()
1971+
}
18911972

18921973
/**
18931974
* A source of instances of `django.http.response.JsonResponse`, extend this class to model new instances.
@@ -1944,7 +2025,12 @@ module PrivateDjango {
19442025
}
19452026

19462027
/** Gets a reference to the `django.http.response.StreamingHttpResponse` class. */
1947-
API::Node classRef() { result = baseClassRef().getASubclass*() }
2028+
API::Node classRef() {
2029+
result = baseClassRef().getASubclass*() or
2030+
result =
2031+
ModelOutput::getATypeNode("django.http.response.StreamingHttpResponse~Subclass")
2032+
.getASubclass*()
2033+
}
19482034

19492035
/**
19502036
* A source of instances of `django.http.response.StreamingHttpResponse`, extend this class to model new instances.
@@ -1998,7 +2084,12 @@ module PrivateDjango {
19982084
}
19992085

20002086
/** Gets a reference to the `django.http.response.FileResponse` class. */
2001-
API::Node classRef() { result = baseClassRef().getASubclass*() }
2087+
API::Node classRef() {
2088+
result = baseClassRef().getASubclass*() or
2089+
result =
2090+
ModelOutput::getATypeNode("django.http.response.FileResponse~Subclass")
2091+
.getASubclass*()
2092+
}
20022093

20032094
/**
20042095
* A source of instances of `django.http.response.FileResponse`, extend this class to model new instances.

‎python/ql/lib/semmle/python/frameworks/Fabric.qll

+7-1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ private import semmle.python.dataflow.new.DataFlow
1212
private import semmle.python.dataflow.new.RemoteFlowSources
1313
private import semmle.python.Concepts
1414
private import semmle.python.ApiGraphs
15+
private import semmle.python.frameworks.data.ModelsAsData
1516

1617
/**
1718
* Provides classes modeling security-relevant aspects of the `fabric` PyPI package, for
@@ -65,12 +66,14 @@ private module FabricV1 {
6566
}
6667

6768
/**
69+
* INTERNAL: Do not use.
70+
*
6871
* Provides classes modeling security-relevant aspects of the `fabric` PyPI package, for
6972
* version 2.x.
7073
*
7174
* See http://docs.fabfile.org/en/2.5/getting-st arted.html.
7275
*/
73-
private module FabricV2 {
76+
module FabricV2 {
7477
/** Gets a reference to the `fabric` module. */
7578
API::Node fabric() { result = API::moduleImport("fabric") }
7679

@@ -95,6 +98,9 @@ private module FabricV2 {
9598
result = fabric().getMember("Connection")
9699
or
97100
result = connection().getMember("Connection")
101+
or
102+
result =
103+
ModelOutput::getATypeNode("fabric.connection.Connection~Subclass").getASubclass*()
98104
}
99105

100106
/**

0 commit comments

Comments
 (0)