Skip to content

Commit b9cdbbb

Browse files
eramongodberamongodb
authored
DRIVERS-2489 Improve test coverage for retryable handshake errors (#1336)
* Add more handshake network error tests for retryable writes * Add more handshake network error tests for retryable reads * Generate handshake network error tests using Python
1 parent 9226c34 commit b9cdbbb

File tree

8 files changed

+6817
-228
lines changed

8 files changed

+6817
-228
lines changed

‎.github/workflows/json-regenerate-check.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,5 @@ jobs:
3030
python3 ./source/client-side-encryption/etc/generate-corpus.py ./source/client-side-encryption/corpus
3131
python3 ./source/client-side-encryption/etc/generate-test.py ./source/client-side-encryption/etc/test-templates/*.template ./source/client-side-encryption/tests/legacy
3232
python3 ./source/client-side-operations-timeout/etc/generate-basic-tests.py ./source/client-side-operations-timeout/etc/templates ./source/client-side-operations-timeout/tests
33+
python3 ./source/etc/generate-handshakeError-tests.py
3334
cd source && make && git diff --exit-code

‎source/etc/generate-handshakeError-tests.py

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
from collections import namedtuple
2+
from jinja2 import Template
3+
import os
4+
import sys
5+
6+
Operation = namedtuple(
7+
'Operation', ['operation_name', 'command_name', 'object', 'arguments'])
8+
9+
CLIENT_OPERATIONS = [
10+
Operation('listDatabases', 'listDatabases', 'client', ['filter: {}']),
11+
Operation('listDatabaseNames', 'listDatabases', 'client', []),
12+
Operation('createChangeStream', 'aggregate', 'client', ['pipeline: []'])
13+
]
14+
15+
RUN_COMMAND_ARGUMENTS = '''command: { ping: 1 }
16+
commandName: ping'''
17+
18+
DB_OPERATIONS = [
19+
Operation('aggregate', 'aggregate', 'database', [
20+
'pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ]']),
21+
Operation('listCollections', 'listCollections',
22+
'database', ['filter: {}']),
23+
Operation('listCollectionNames', 'listCollections',
24+
'database', ['filter: {}']), # Optional.
25+
Operation('runCommand', 'ping', 'database', [RUN_COMMAND_ARGUMENTS]),
26+
Operation('createChangeStream', 'aggregate', 'database', ['pipeline: []'])
27+
]
28+
29+
INSERT_MANY_ARGUMENTS = '''documents:
30+
- { _id: 2, x: 22 }'''
31+
32+
BULK_WRITE_ARGUMENTS = '''requests:
33+
- insertOne:
34+
document: { _id: 2, x: 22 }'''
35+
36+
COLLECTION_READ_OPERATIONS = [
37+
Operation('aggregate', 'aggregate', 'collection', ['pipeline: []']),
38+
# Operation('count', 'count', 'collection', ['filter: {}']), # Deprecated.
39+
Operation('countDocuments', 'aggregate', 'collection', ['filter: {}']),
40+
Operation('estimatedDocumentCount', 'count', 'collection', []),
41+
Operation('distinct', 'distinct', 'collection',
42+
['fieldName: x', 'filter: {}']),
43+
Operation('find', 'find', 'collection', ['filter: {}']),
44+
Operation('findOne', 'find', 'collection', ['filter: {}']), # Optional.
45+
Operation('listIndexes', 'listIndexes', 'collection', []),
46+
Operation('listIndexNames', 'listIndexes', 'collection', []), # Optional.
47+
Operation('createChangeStream', 'aggregate',
48+
'collection', ['pipeline: []']),
49+
]
50+
51+
COLLECTION_WRITE_OPERATIONS = [
52+
Operation('insertOne', 'insert', 'collection',
53+
['document: { _id: 2, x: 22 }']),
54+
Operation('insertMany', 'insert', 'collection', [INSERT_MANY_ARGUMENTS]),
55+
Operation('deleteOne', 'delete', 'collection', ['filter: {}']),
56+
Operation('deleteMany', 'delete', 'collection', ['filter: {}']),
57+
Operation('replaceOne', 'update', 'collection', [
58+
'filter: {}', 'replacement: { x: 22 }']),
59+
Operation('updateOne', 'update', 'collection', [
60+
'filter: {}', 'update: { $set: { x: 22 } }']),
61+
Operation('updateMany', 'update', 'collection', [
62+
'filter: {}', 'update: { $set: { x: 22 } }']),
63+
Operation('findOneAndDelete', 'findAndModify',
64+
'collection', ['filter: {}']),
65+
Operation('findOneAndReplace', 'findAndModify', 'collection',
66+
['filter: {}', 'replacement: { x: 22 }']),
67+
Operation('findOneAndUpdate', 'findAndModify', 'collection',
68+
['filter: {}', 'update: { $set: { x: 22 } }']),
69+
Operation('bulkWrite', 'insert', 'collection', [BULK_WRITE_ARGUMENTS]),
70+
Operation('createIndex', 'createIndexes', 'collection',
71+
['keys: { x: 11 }', 'name: "x_11"']),
72+
Operation('dropIndex', 'dropIndexes', 'collection', ['name: "x_11"']),
73+
Operation('dropIndexes', 'dropIndexes', 'collection', []),
74+
]
75+
76+
COLLECTION_OPERATIONS = COLLECTION_READ_OPERATIONS + COLLECTION_WRITE_OPERATIONS
77+
78+
# Session and GridFS operations are generally tested in other files, so they're not included in the list of all
79+
# operations. Individual generation functions can choose to include them if needed.
80+
OPERATIONS = CLIENT_OPERATIONS + DB_OPERATIONS + COLLECTION_OPERATIONS
81+
82+
RETRYABLE_READ_OPERATIONS = [op for op in OPERATIONS if op.operation_name in
83+
['find',
84+
'findOne',
85+
'aggregate',
86+
'distinct',
87+
'count',
88+
'estimatedDocumentCount',
89+
'countDocuments',
90+
'createChangeStream',
91+
'listDatabases',
92+
'listDatabaseNames',
93+
'listCollections',
94+
'listCollectionNames',
95+
'listIndexes',
96+
'listIndexNames',
97+
]
98+
]
99+
100+
RETRYABLE_WRITE_OPERATIONS = [op for op in OPERATIONS if op.operation_name in
101+
['insertOne',
102+
'updateOne',
103+
'deleteOne',
104+
'replaceOne',
105+
'findOneAndDelete',
106+
'findOneAndUpdate',
107+
'findOneAndReplace',
108+
'insertMany',
109+
'bulkWrite',
110+
]
111+
]
112+
113+
114+
# ./source/etc
115+
DIR = os.path.dirname(os.path.realpath(__file__))
116+
117+
118+
def get_template(file, templates_dir):
119+
path = f'{templates_dir}/{file}.yml.template'
120+
return Template(open(path, 'r').read())
121+
122+
123+
def write_yaml(file, template, tests_dir, injections):
124+
rendered = template.render(**injections)
125+
path = f'{tests_dir}/{file}.yml'
126+
open(path, 'w').write(rendered)
127+
128+
129+
def generate(name, templates_dir, tests_dir, operations):
130+
template = get_template(name, templates_dir)
131+
injections = {
132+
'operations': operations,
133+
}
134+
write_yaml(name, template, tests_dir, injections)
135+
136+
137+
def generate_retryable_reads_handshake_error_tests():
138+
# ./source/retryable-reads/tests/etc/templates
139+
templates_dir = f'{os.path.dirname(DIR)}/retryable-reads/tests/etc/templates'
140+
# ./source/retryable-reads/tests/unified
141+
tests_dir = f'{os.path.dirname(DIR)}/retryable-reads/tests/unified'
142+
generate('handshakeError', templates_dir,
143+
tests_dir, RETRYABLE_READ_OPERATIONS)
144+
145+
146+
def generate_retryable_writes_handshake_error_tests():
147+
# ./source/retryable-writes/tests/etc/templates
148+
templates_dir = f'{os.path.dirname(DIR)}/retryable-writes/tests/etc/templates'
149+
# ./source/retryable-writes/tests/unified
150+
tests_dir = f'{os.path.dirname(DIR)}/retryable-writes/tests/unified'
151+
generate('handshakeError', templates_dir,
152+
tests_dir, RETRYABLE_WRITE_OPERATIONS)
153+
154+
155+
generate_retryable_reads_handshake_error_tests()
156+
generate_retryable_writes_handshake_error_tests()

‎source/retryable-reads/tests/etc/templates/handshakeError.yml.template

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
# Tests in this file are generated from handshakeError.yml.template.
2+
3+
description: "retryable reads handshake failures"
4+
5+
schemaVersion: "1.3"
6+
7+
runOnRequirements:
8+
- minServerVersion: "4.2"
9+
topologies: [replicaset, sharded, load-balanced]
10+
auth: true
11+
12+
createEntities:
13+
- client:
14+
id: &client client
15+
useMultipleMongoses: false
16+
observeEvents:
17+
- connectionCheckOutStartedEvent
18+
- commandStartedEvent
19+
- commandSucceededEvent
20+
- commandFailedEvent
21+
- database:
22+
id: &database database
23+
client: *client
24+
databaseName: &databaseName retryable-reads-handshake-tests
25+
- collection:
26+
id: &collection collection
27+
database: *database
28+
collectionName: &collectionName coll
29+
30+
initialData:
31+
- collectionName: *collectionName
32+
databaseName: *databaseName
33+
documents:
34+
- { _id: 1, x: 11 }
35+
- { _id: 2, x: 22 }
36+
- { _id: 3, x: 33 }
37+
38+
tests:
39+
# Because setting a failPoint creates a connection in the connection pool, run
40+
# a ping operation that fails immediately after the failPoint operation in
41+
# order to discard the connection before running the actual operation to be
42+
# tested. The saslContinue command is used to avoid SDAM errors.
43+
#
44+
# Description of events:
45+
# - Failpoint operation.
46+
# - Creates a connection in the connection pool that must be closed.
47+
# - Ping operation.
48+
# - Triggers failpoint (first time).
49+
# - Closes the connection made by the fail point operation.
50+
# - Test operation.
51+
# - New connection is created.
52+
# - Triggers failpoint (second time).
53+
# - Tests whether operation successfully retries the handshake and succeeds.
54+
{% for operation in operations %}
55+
- description: "{{operation.operation_name}} succeeds after retryable handshake network error"
56+
operations:
57+
- name: failPoint
58+
object: testRunner
59+
arguments:
60+
client: *client
61+
failPoint:
62+
configureFailPoint: failCommand
63+
mode: { times: 2 }
64+
data:
65+
failCommands: [ping, saslContinue]
66+
closeConnection: true
67+
- name: runCommand
68+
object: *database
69+
arguments: { commandName: ping, command: { ping: 1 } }
70+
expectError: { isError: true }
71+
- name: {{operation.operation_name}}
72+
object: *{{operation.object}}
73+
{%- if operation.arguments|length > 0 %}
74+
arguments:
75+
{%- for arg in operation.arguments %}
76+
{{arg}}
77+
{%- endfor -%}
78+
{%- endif %}
79+
{%- if operation.operation_name == "createChangeStream" %}
80+
saveResultAsEntity: changeStream
81+
{%- endif %}
82+
expectEvents:
83+
- client: *client
84+
eventType: cmap
85+
events:
86+
- { connectionCheckOutStartedEvent: {} }
87+
- { connectionCheckOutStartedEvent: {} }
88+
- { connectionCheckOutStartedEvent: {} }
89+
- { connectionCheckOutStartedEvent: {} }
90+
- client: *client
91+
events:
92+
- commandStartedEvent:
93+
command: { ping: 1 }
94+
databaseName: *databaseName
95+
- commandFailedEvent:
96+
commandName: ping
97+
- commandStartedEvent:
98+
commandName: {{operation.command_name}}
99+
- commandSucceededEvent:
100+
commandName: {{operation.command_name}}
101+
102+
- description: "{{operation.operation_name}} succeeds after retryable handshake server error (ShutdownInProgress)"
103+
operations:
104+
- name: failPoint
105+
object: testRunner
106+
arguments:
107+
client: *client
108+
failPoint:
109+
configureFailPoint: failCommand
110+
mode: { times: 2 }
111+
data:
112+
failCommands: [ping, saslContinue]
113+
closeConnection: true
114+
- name: runCommand
115+
object: *database
116+
arguments: { commandName: ping, command: { ping: 1 } }
117+
expectError: { isError: true }
118+
- name: {{operation.operation_name}}
119+
object: *{{operation.object}}
120+
{%- if operation.arguments|length > 0 %}
121+
arguments:
122+
{%- for arg in operation.arguments %}
123+
{{arg}}
124+
{%- endfor -%}
125+
{%- endif %}
126+
{%- if operation.operation_name == "createChangeStream" %}
127+
saveResultAsEntity: changeStream
128+
{%- endif %}
129+
expectEvents:
130+
- client: *client
131+
eventType: cmap
132+
events:
133+
- { connectionCheckOutStartedEvent: {} }
134+
- { connectionCheckOutStartedEvent: {} }
135+
- { connectionCheckOutStartedEvent: {} }
136+
- { connectionCheckOutStartedEvent: {} }
137+
- client: *client
138+
events:
139+
- commandStartedEvent:
140+
command: { ping: 1 }
141+
databaseName: *databaseName
142+
- commandFailedEvent:
143+
commandName: ping
144+
- commandStartedEvent:
145+
commandName: {{operation.command_name}}
146+
- commandSucceededEvent:
147+
commandName: {{operation.command_name}}
148+
{% endfor -%}

0 commit comments

Comments
 (0)