Skip to content

Commit 432410b

Browse files
keelerm84keelerm84
committed
test: Atomically update file data source file
1 parent d3a0488 commit 432410b

File tree

4 files changed

+131
-72
lines changed

4 files changed

+131
-72
lines changed

‎.github/workflows/ci.yml

Lines changed: 82 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ jobs:
1717
strategy:
1818
fail-fast: false
1919
matrix:
20-
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
20+
# python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
21+
python-version: ["3.8"]
2122

2223
services:
2324
redis:
@@ -49,6 +50,28 @@ jobs:
4950
- name: Run tests
5051
run: make test
5152

53+
- run: make test
54+
- run: make test
55+
- run: make test
56+
- run: make test
57+
- run: make test
58+
- run: make test
59+
- run: make test
60+
- run: make test
61+
- run: make test
62+
- run: make test
63+
- run: make test
64+
- run: make test
65+
- run: make test
66+
- run: make test
67+
- run: make test
68+
- run: make test
69+
- run: make test
70+
- run: make test
71+
- run: make test
72+
- run: make test
73+
- run: make test
74+
5275
- name: Verify typehints
5376
run: make lint
5477

@@ -67,61 +90,61 @@ jobs:
6790
test_service_port: 9000
6891
token: ${{ secrets.GITHUB_TOKEN }}
6992

70-
windows:
71-
runs-on: windows-latest
72-
73-
defaults:
74-
run:
75-
shell: powershell
76-
77-
strategy:
78-
fail-fast: false
79-
matrix:
80-
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
81-
82-
steps:
83-
- uses: actions/checkout@v4
84-
- name: Set up Python ${{ matrix.python-version }}
85-
uses: actions/setup-python@v5
86-
with:
87-
python-version: ${{ matrix.python-version }}
88-
89-
- name: Setup DynamoDB
90-
run: |
91-
$ProgressPreference = "SilentlyContinue"
92-
iwr -outf dynamo.zip https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip
93-
mkdir dynamo
94-
Expand-Archive -Path dynamo.zip -DestinationPath dynamo
95-
cd dynamo
96-
cmd /c "START /b java -Djava.library.path=./DynamoDBLocal_lib -jar ./DynamoDBLocal.jar"
97-
98-
- name: Setup Consul
99-
run: |
100-
$ProgressPreference = "SilentlyContinue"
101-
iwr -outf consul.zip https://releases.hashicorp.com/consul/1.4.2/consul_1.4.2_windows_amd64.zip
102-
mkdir consul
103-
Expand-Archive -Path consul.zip -DestinationPath consul
104-
cd consul
105-
sc.exe create "Consul" binPath="$(Get-Location)/consul.exe agent -dev"
106-
sc.exe start "Consul"
107-
108-
- name: Setup Redis
109-
run: |
110-
$ProgressPreference = "SilentlyContinue"
111-
iwr -outf redis.zip https://github.com/MicrosoftArchive/redis/releases/download/win-3.0.504/Redis-x64-3.0.504.zip
112-
mkdir redis
113-
Expand-Archive -Path redis.zip -DestinationPath redis
114-
cd redis
115-
./redis-server --service-install
116-
./redis-server --service-start
117-
Start-Sleep -s 5
118-
./redis-cli ping
119-
120-
- name: Install poetry
121-
uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439
122-
123-
- name: Install requirements
124-
run: poetry install --all-extras
125-
126-
- name: Run tests
127-
run: make test
93+
# windows:
94+
# runs-on: windows-latest
95+
#
96+
# defaults:
97+
# run:
98+
# shell: powershell
99+
#
100+
# strategy:
101+
# fail-fast: false
102+
# matrix:
103+
# python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
104+
#
105+
# steps:
106+
# - uses: actions/checkout@v4
107+
# - name: Set up Python ${{ matrix.python-version }}
108+
# uses: actions/setup-python@v5
109+
# with:
110+
# python-version: ${{ matrix.python-version }}
111+
#
112+
# - name: Setup DynamoDB
113+
# run: |
114+
# $ProgressPreference = "SilentlyContinue"
115+
# iwr -outf dynamo.zip https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip
116+
# mkdir dynamo
117+
# Expand-Archive -Path dynamo.zip -DestinationPath dynamo
118+
# cd dynamo
119+
# cmd /c "START /b java -Djava.library.path=./DynamoDBLocal_lib -jar ./DynamoDBLocal.jar"
120+
#
121+
# - name: Setup Consul
122+
# run: |
123+
# $ProgressPreference = "SilentlyContinue"
124+
# iwr -outf consul.zip https://releases.hashicorp.com/consul/1.4.2/consul_1.4.2_windows_amd64.zip
125+
# mkdir consul
126+
# Expand-Archive -Path consul.zip -DestinationPath consul
127+
# cd consul
128+
# sc.exe create "Consul" binPath="$(Get-Location)/consul.exe agent -dev"
129+
# sc.exe start "Consul"
130+
#
131+
# - name: Setup Redis
132+
# run: |
133+
# $ProgressPreference = "SilentlyContinue"
134+
# iwr -outf redis.zip https://github.com/MicrosoftArchive/redis/releases/download/win-3.0.504/Redis-x64-3.0.504.zip
135+
# mkdir redis
136+
# Expand-Archive -Path redis.zip -DestinationPath redis
137+
# cd redis
138+
# ./redis-server --service-install
139+
# ./redis-server --service-start
140+
# Start-Sleep -s 5
141+
# ./redis-cli ping
142+
#
143+
# - name: Install poetry
144+
# uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439
145+
#
146+
# - name: Install requirements
147+
# run: poetry install --all-extras
148+
#
149+
# - name: Run tests
150+
# run: make test

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ install:
2828
.PHONY: test
2929
test: #! Run unit tests
3030
test: install
31-
@poetry run pytest $(PYTEST_FLAGS)
31+
@poetry run pytest $(PYTEST_FLAGS) ldclient/testing/test_file_data_source.py
3232

3333
.PHONY: lint
3434
lint: #! Run type analysis and linting checks

‎ldclient/impl/integrations/files/file_data_source.py

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,10 @@ def _load_all(self):
8888
for path in self._paths:
8989
try:
9090
self._load_file(path, all_data)
91-
except Exception as e:
91+
except FileDataSourceEmpty:
92+
log.warning('No flag data found in any file, continuing with current state')
93+
return
94+
except (Exception, FileNotFoundError) as e:
9295
log.error('Unable to load flag data from "%s": %s' % (path, repr(e)))
9396
traceback.print_exc()
9497
if self._data_source_update_sink is not None:
@@ -110,6 +113,10 @@ def _load_file(self, path, all_data):
110113
with open(path, 'r') as f:
111114
content = f.read()
112115
parsed = self._parse_content(content)
116+
117+
if parsed is None:
118+
raise FileDataSourceEmpty()
119+
113120
for key, flag in parsed.get('flags', {}).items():
114121
_sanitize_json_item(flag)
115122
self._add_item(all_data, FEATURES, flag)
@@ -155,6 +162,13 @@ def __init__(self, resolved_paths, reloader):
155162

156163
class LDWatchdogHandler(watchdog.events.FileSystemEventHandler):
157164
def on_any_event(self, event):
165+
if isinstance(event, watchdog.events.FileDeletedEvent):
166+
return
167+
168+
if isinstance(event, watchdog.events.FileMovedEvent) and event.dest_path in watched_files:
169+
reloader()
170+
return
171+
158172
if event.src_path in watched_files:
159173
reloader()
160174

@@ -203,4 +217,9 @@ def _check_file_times(self):
203217
ret[path] = os.path.getmtime(path)
204218
except Exception:
205219
ret[path] = None
220+
206221
return ret
222+
223+
224+
class FileDataSourceEmpty(Exception):
225+
pass

‎ldclient/testing/test_file_data_source.py

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -120,11 +120,13 @@ def make_temp_file(content):
120120
os.close(f)
121121
return path
122122

123-
124-
def replace_file(path, content):
123+
def update_file(path, content):
125124
with open(path, 'w') as f:
126125
f.write(content)
127126

127+
def replace_file(path, content):
128+
new_file = make_temp_file(content)
129+
os.replace(new_file, path)
128130

129131
def test_does_not_load_data_prior_to_start():
130132
path = make_temp_file('{"flagValues":{"key":"value"}}')
@@ -234,8 +236,7 @@ def test_does_not_allow_duplicate_keys():
234236
os.remove(path1)
235237
os.remove(path2)
236238

237-
238-
def test_does_not_reload_modified_file_if_auto_update_is_off():
239+
def test_does_not_reload_modified_file_if_auto_update_is_off_when_replacing_file():
239240
path = make_temp_file(flag_only_json)
240241
try:
241242
source = make_data_source(Config("SDK_KEY"), paths=path)
@@ -248,16 +249,29 @@ def test_does_not_reload_modified_file_if_auto_update_is_off():
248249
finally:
249250
os.remove(path)
250251

252+
def test_does_not_reload_modified_file_if_auto_update_is_off_when_updating_file():
253+
path = make_temp_file(flag_only_json)
254+
try:
255+
source = make_data_source(Config("SDK_KEY"), paths = path)
256+
source.start()
257+
assert len(store.all(SEGMENTS, lambda x: x)) == 0
258+
time.sleep(0.5)
259+
update_file(path, segment_only_json)
260+
time.sleep(0.5)
261+
assert len(store.all(SEGMENTS, lambda x: x)) == 0
262+
finally:
263+
os.remove(path)
251264

252-
def do_auto_update_test(options):
265+
def do_auto_update_test(options, update_fn):
253266
path = make_temp_file(flag_only_json)
267+
254268
options['paths'] = path
255269
try:
256270
source = make_data_source(Config("SDK_KEY"), **options)
257271
source.start()
258272
assert len(store.all(SEGMENTS, lambda x: x)) == 0
259273
time.sleep(0.5)
260-
replace_file(path, segment_only_json)
274+
update_fn(path, segment_only_json)
261275
deadline = time.time() + 20
262276
while time.time() < deadline:
263277
time.sleep(0.1)
@@ -267,14 +281,17 @@ def do_auto_update_test(options):
267281
finally:
268282
os.remove(path)
269283

284+
def test_reloads_modified_file_if_auto_update_is_on_when_replacing_file():
285+
do_auto_update_test({ 'auto_update': True }, replace_file)
270286

271-
def test_reloads_modified_file_if_auto_update_is_on():
272-
do_auto_update_test({'auto_update': True})
273-
287+
def test_reloads_modified_file_in_polling_mode_when_replacing_file():
288+
do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 }, replace_file)
274289

275-
def test_reloads_modified_file_in_polling_mode():
276-
do_auto_update_test({'auto_update': True, 'force_polling': True, 'poll_interval': 0.1})
290+
def test_reloads_modified_file_if_auto_update_is_on_when_updating_file():
291+
do_auto_update_test({ 'auto_update': True }, update_file)
277292

293+
def test_reloads_modified_file_in_polling_mode_when_updating_file():
294+
do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 }, update_file)
278295

279296
def test_evaluates_full_flag_with_client_as_expected():
280297
path = make_temp_file(all_properties_json)

0 commit comments

Comments
 (0)