Skip to content

Commit b84f998

Browse files
keelerm84keelerm84
committed
test: Atomically update file data source file
1 parent 3cc6e35 commit b84f998

File tree

4 files changed

+137
-69
lines changed

4 files changed

+137
-69
lines changed

‎.github/workflows/ci.yml

Lines changed: 82 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ jobs:
1717
strategy:
1818
fail-fast: false
1919
matrix:
20-
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
20+
# python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
21+
python-version: ["3.8"]
2122

2223
services:
2324
redis:
@@ -49,6 +50,28 @@ jobs:
4950
- name: Run tests
5051
run: make test
5152

53+
- run: make test
54+
- run: make test
55+
- run: make test
56+
- run: make test
57+
- run: make test
58+
- run: make test
59+
- run: make test
60+
- run: make test
61+
- run: make test
62+
- run: make test
63+
- run: make test
64+
- run: make test
65+
- run: make test
66+
- run: make test
67+
- run: make test
68+
- run: make test
69+
- run: make test
70+
- run: make test
71+
- run: make test
72+
- run: make test
73+
- run: make test
74+
5275
- name: Verify typehints
5376
run: make lint
5477

@@ -67,61 +90,61 @@ jobs:
6790
test_service_port: 9000
6891
token: ${{ secrets.GITHUB_TOKEN }}
6992

70-
windows:
71-
runs-on: windows-latest
72-
73-
defaults:
74-
run:
75-
shell: powershell
76-
77-
strategy:
78-
fail-fast: false
79-
matrix:
80-
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
81-
82-
steps:
83-
- uses: actions/checkout@v4
84-
- name: Set up Python ${{ matrix.python-version }}
85-
uses: actions/setup-python@v5
86-
with:
87-
python-version: ${{ matrix.python-version }}
88-
89-
- name: Setup DynamoDB
90-
run: |
91-
$ProgressPreference = "SilentlyContinue"
92-
iwr -outf dynamo.zip https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip
93-
mkdir dynamo
94-
Expand-Archive -Path dynamo.zip -DestinationPath dynamo
95-
cd dynamo
96-
cmd /c "START /b java -Djava.library.path=./DynamoDBLocal_lib -jar ./DynamoDBLocal.jar"
97-
98-
- name: Setup Consul
99-
run: |
100-
$ProgressPreference = "SilentlyContinue"
101-
iwr -outf consul.zip https://releases.hashicorp.com/consul/1.4.2/consul_1.4.2_windows_amd64.zip
102-
mkdir consul
103-
Expand-Archive -Path consul.zip -DestinationPath consul
104-
cd consul
105-
sc.exe create "Consul" binPath="$(Get-Location)/consul.exe agent -dev"
106-
sc.exe start "Consul"
107-
108-
- name: Setup Redis
109-
run: |
110-
$ProgressPreference = "SilentlyContinue"
111-
iwr -outf redis.zip https://github.com/MicrosoftArchive/redis/releases/download/win-3.0.504/Redis-x64-3.0.504.zip
112-
mkdir redis
113-
Expand-Archive -Path redis.zip -DestinationPath redis
114-
cd redis
115-
./redis-server --service-install
116-
./redis-server --service-start
117-
Start-Sleep -s 5
118-
./redis-cli ping
119-
120-
- name: Install poetry
121-
uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439
122-
123-
- name: Install requirements
124-
run: poetry install --all-extras
125-
126-
- name: Run tests
127-
run: make test
93+
# windows:
94+
# runs-on: windows-latest
95+
#
96+
# defaults:
97+
# run:
98+
# shell: powershell
99+
#
100+
# strategy:
101+
# fail-fast: false
102+
# matrix:
103+
# python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
104+
#
105+
# steps:
106+
# - uses: actions/checkout@v4
107+
# - name: Set up Python ${{ matrix.python-version }}
108+
# uses: actions/setup-python@v5
109+
# with:
110+
# python-version: ${{ matrix.python-version }}
111+
#
112+
# - name: Setup DynamoDB
113+
# run: |
114+
# $ProgressPreference = "SilentlyContinue"
115+
# iwr -outf dynamo.zip https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip
116+
# mkdir dynamo
117+
# Expand-Archive -Path dynamo.zip -DestinationPath dynamo
118+
# cd dynamo
119+
# cmd /c "START /b java -Djava.library.path=./DynamoDBLocal_lib -jar ./DynamoDBLocal.jar"
120+
#
121+
# - name: Setup Consul
122+
# run: |
123+
# $ProgressPreference = "SilentlyContinue"
124+
# iwr -outf consul.zip https://releases.hashicorp.com/consul/1.4.2/consul_1.4.2_windows_amd64.zip
125+
# mkdir consul
126+
# Expand-Archive -Path consul.zip -DestinationPath consul
127+
# cd consul
128+
# sc.exe create "Consul" binPath="$(Get-Location)/consul.exe agent -dev"
129+
# sc.exe start "Consul"
130+
#
131+
# - name: Setup Redis
132+
# run: |
133+
# $ProgressPreference = "SilentlyContinue"
134+
# iwr -outf redis.zip https://github.com/MicrosoftArchive/redis/releases/download/win-3.0.504/Redis-x64-3.0.504.zip
135+
# mkdir redis
136+
# Expand-Archive -Path redis.zip -DestinationPath redis
137+
# cd redis
138+
# ./redis-server --service-install
139+
# ./redis-server --service-start
140+
# Start-Sleep -s 5
141+
# ./redis-cli ping
142+
#
143+
# - name: Install poetry
144+
# uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439
145+
#
146+
# - name: Install requirements
147+
# run: poetry install --all-extras
148+
#
149+
# - name: Run tests
150+
# run: make test

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ install:
2828
.PHONY: test
2929
test: #! Run unit tests
3030
test: install
31-
@poetry run pytest $(PYTEST_FLAGS)
31+
@poetry run pytest $(PYTEST_FLAGS) ldclient/testing/test_file_data_source.py
3232

3333
.PHONY: lint
3434
lint: #! Run type analysis and linting checks

‎ldclient/impl/integrations/files/file_data_source.py

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,10 @@ def _load_all(self):
8484
for path in self._paths:
8585
try:
8686
self._load_file(path, all_data)
87-
except Exception as e:
87+
except FileDataSourceEmpty:
88+
log.warning('No flag data found in any file, continuing with current state')
89+
return
90+
except (Exception, FileNotFoundError) as e:
8891
log.error('Unable to load flag data from "%s": %s' % (path, repr(e)))
8992
traceback.print_exc()
9093
if self._data_source_update_sink is not None:
@@ -112,6 +115,10 @@ def _load_file(self, path, all_data):
112115
with open(path, 'r') as f:
113116
content = f.read()
114117
parsed = self._parse_content(content)
118+
119+
if parsed is None:
120+
raise FileDataSourceEmpty()
121+
115122
for key, flag in parsed.get('flags', {}).items():
116123
_sanitize_json_item(flag)
117124
self._add_item(all_data, FEATURES, flag)
@@ -165,6 +172,13 @@ def __init__(self, resolved_paths, reloader):
165172

166173
class LDWatchdogHandler(watchdog.events.FileSystemEventHandler):
167174
def on_any_event(self, event):
175+
if isinstance(event, watchdog.events.FileDeletedEvent):
176+
return
177+
178+
if isinstance(event, watchdog.events.FileMovedEvent) and event.dest_path in watched_files:
179+
reloader()
180+
return
181+
168182
if event.src_path in watched_files:
169183
reloader()
170184

@@ -213,4 +227,9 @@ def _check_file_times(self):
213227
ret[path] = os.path.getmtime(path)
214228
except:
215229
ret[path] = None
230+
216231
return ret
232+
233+
234+
class FileDataSourceEmpty(Exception):
235+
pass

‎ldclient/testing/test_file_data_source.py

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
import os
33
from typing import List
44

5+
from ldclient.impl.util import log
6+
57
import pytest
68
import tempfile
79
import threading
@@ -116,10 +118,14 @@ def make_temp_file(content):
116118
os.close(f)
117119
return path
118120

119-
def replace_file(path, content):
121+
def update_file(path, content):
120122
with open(path, 'w') as f:
121123
f.write(content)
122124

125+
def replace_file(path, content):
126+
new_file = make_temp_file(content)
127+
os.replace(new_file, path)
128+
123129
def test_does_not_load_data_prior_to_start():
124130
path = make_temp_file('{"flagValues":{"key":"value"}}')
125131
try:
@@ -221,7 +227,7 @@ def test_does_not_allow_duplicate_keys():
221227
os.remove(path1)
222228
os.remove(path2)
223229

224-
def test_does_not_reload_modified_file_if_auto_update_is_off():
230+
def test_does_not_reload_modified_file_if_auto_update_is_off_when_replacing_file():
225231
path = make_temp_file(flag_only_json)
226232
try:
227233
source = make_data_source(Config("SDK_KEY"), paths = path)
@@ -234,15 +240,29 @@ def test_does_not_reload_modified_file_if_auto_update_is_off():
234240
finally:
235241
os.remove(path)
236242

237-
def do_auto_update_test(options):
243+
def test_does_not_reload_modified_file_if_auto_update_is_off_when_updating_file():
238244
path = make_temp_file(flag_only_json)
245+
try:
246+
source = make_data_source(Config("SDK_KEY"), paths = path)
247+
source.start()
248+
assert len(store.all(SEGMENTS, lambda x: x)) == 0
249+
time.sleep(0.5)
250+
update_file(path, segment_only_json)
251+
time.sleep(0.5)
252+
assert len(store.all(SEGMENTS, lambda x: x)) == 0
253+
finally:
254+
os.remove(path)
255+
256+
def do_auto_update_test(options, update_fn):
257+
path = make_temp_file(flag_only_json)
258+
239259
options['paths'] = path
240260
try:
241261
source = make_data_source(Config("SDK_KEY"), **options)
242262
source.start()
243263
assert len(store.all(SEGMENTS, lambda x: x)) == 0
244264
time.sleep(0.5)
245-
replace_file(path, segment_only_json)
265+
update_fn(path, segment_only_json)
246266
deadline = time.time() + 20
247267
while time.time() < deadline:
248268
time.sleep(0.1)
@@ -252,11 +272,17 @@ def do_auto_update_test(options):
252272
finally:
253273
os.remove(path)
254274

255-
def test_reloads_modified_file_if_auto_update_is_on():
256-
do_auto_update_test({ 'auto_update': True })
275+
def test_reloads_modified_file_if_auto_update_is_on_when_replacing_file():
276+
do_auto_update_test({ 'auto_update': True }, replace_file)
277+
278+
def test_reloads_modified_file_in_polling_mode_when_replacing_file():
279+
do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 }, replace_file)
280+
281+
def test_reloads_modified_file_if_auto_update_is_on_when_updating_file():
282+
do_auto_update_test({ 'auto_update': True }, update_file)
257283

258-
def test_reloads_modified_file_in_polling_mode():
259-
do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 })
284+
def test_reloads_modified_file_in_polling_mode_when_updating_file():
285+
do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 }, update_file)
260286

261287
def test_evaluates_full_flag_with_client_as_expected():
262288
path = make_temp_file(all_properties_json)

0 commit comments

Comments
 (0)