Skip to content

Commit 8d7cc91

Browse files
tumiditumidi
committed
feat: watch mode for run command
Closes #101
1 parent 97bb84b commit 8d7cc91

File tree

9 files changed

+403
-55
lines changed

9 files changed

+403
-55
lines changed

‎questionpy_sdk/commands/run.py

+37-3
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,27 @@
11
# This file is part of the QuestionPy SDK. (https://questionpy.org)
22
# The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md.
33
# (c) Technische Universität Berlin, innoCampus <[email protected]>
4+
5+
import asyncio
46
from pathlib import Path
7+
from typing import TYPE_CHECKING
58

69
import click
710

811
from questionpy_sdk.commands._helper import get_package_location
12+
from questionpy_sdk.watcher import Watcher
913
from questionpy_sdk.webserver.app import DEFAULT_STATE_STORAGE_PATH, WebServer
14+
from questionpy_server.worker.runtime.package_location import DirPackageLocation
15+
16+
if TYPE_CHECKING:
17+
from collections.abc import Coroutine
18+
19+
20+
async def run_watcher(
21+
pkg_path: Path, pkg_location: DirPackageLocation, state_storage_path: Path, host: str, port: int
22+
) -> None:
23+
async with Watcher(pkg_path, pkg_location, state_storage_path, host, port) as watcher:
24+
await watcher.run_forever()
1025

1126

1227
@click.command()
@@ -16,8 +31,17 @@
1631
type=click.Path(path_type=Path, exists=False, file_okay=False, dir_okay=True, resolve_path=True),
1732
default=DEFAULT_STATE_STORAGE_PATH,
1833
envvar="QPY_STATE_STORAGE_PATH",
34+
show_default=True,
35+
help="State storage path to use.",
36+
)
37+
@click.option(
38+
"--host", "-h", "host", default="localhost", show_default=True, type=click.STRING, help="Host to listen on."
39+
)
40+
@click.option(
41+
"--port", "-p", "port", default=8080, show_default=True, type=click.IntRange(1024, 65535), help="Port to bind to."
1942
)
20-
def run(package: str, state_storage_path: Path) -> None:
43+
@click.option("--watch", "-w", "watch", is_flag=True, help="Watch source directory and rebuild on changes.")
44+
def run(package: str, state_storage_path: Path, host: str, port: int, *, watch: bool) -> None:
2145
"""Run a package.
2246
2347
\b
@@ -27,5 +51,15 @@ def run(package: str, state_storage_path: Path) -> None:
2751
- a source directory (built on-the-fly).
2852
""" # noqa: D301
2953
pkg_path = Path(package).resolve()
30-
web_server = WebServer(get_package_location(package, pkg_path), state_storage_path)
31-
web_server.start_server()
54+
pkg_location = get_package_location(package, pkg_path)
55+
coro: Coroutine
56+
57+
if watch:
58+
if not isinstance(pkg_location, DirPackageLocation) or pkg_path == pkg_location.path:
59+
msg = "The --watch option only works with source directories."
60+
raise click.BadParameter(msg)
61+
coro = run_watcher(pkg_path, pkg_location, state_storage_path, host, port)
62+
else:
63+
coro = WebServer(pkg_location, state_storage_path, host, port).run_forever()
64+
65+
asyncio.run(coro)

‎questionpy_sdk/package/builder.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
from questionpy_sdk.package.errors import PackageBuildError
2424
from questionpy_sdk.package.source import PackageSource
2525

26-
log = logging.getLogger(__name__)
26+
log = logging.getLogger("questionpy-sdk:builder")
2727

2828

2929
class PackageBuilderBase(AbstractContextManager):

‎questionpy_sdk/watcher.py

+176
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
# This file is part of the QuestionPy SDK. (https://questionpy.org)
2+
# The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md.
3+
# (c) Technische Universität Berlin, innoCampus <[email protected]>
4+
5+
import asyncio
6+
import logging
7+
from collections.abc import Awaitable, Callable
8+
from contextlib import AbstractAsyncContextManager
9+
from pathlib import Path
10+
from types import TracebackType
11+
from typing import TYPE_CHECKING, Self
12+
13+
from watchdog.events import (
14+
FileClosedEvent,
15+
FileOpenedEvent,
16+
FileSystemEvent,
17+
FileSystemEventHandler,
18+
FileSystemMovedEvent,
19+
)
20+
from watchdog.observers import Observer
21+
from watchdog.utils.event_debouncer import EventDebouncer
22+
23+
from questionpy_common.constants import DIST_DIR
24+
from questionpy_sdk.package.builder import DirPackageBuilder
25+
from questionpy_sdk.package.errors import PackageBuildError, PackageSourceValidationError
26+
from questionpy_sdk.package.source import PackageSource
27+
from questionpy_sdk.webserver.app import WebServer
28+
from questionpy_server.worker.runtime.package_location import DirPackageLocation
29+
30+
if TYPE_CHECKING:
31+
from watchdog.observers.api import ObservedWatch
32+
33+
log = logging.getLogger("questionpy-sdk:watcher")
34+
35+
_DEBOUNCE_INTERVAL = 0.5 # seconds
36+
37+
38+
class _EventHandler(FileSystemEventHandler):
39+
"""Debounces events for watchdog file monitoring, ignoring events in the `dist` directory."""
40+
41+
def __init__(
42+
self, loop: asyncio.AbstractEventLoop, notify_callback: Callable[[], Awaitable[None]], watch_path: Path
43+
) -> None:
44+
self._loop = loop
45+
self._notify_callback = notify_callback
46+
self._watch_path = watch_path
47+
48+
self._event_debouncer = EventDebouncer(_DEBOUNCE_INTERVAL, self._on_file_changes)
49+
50+
def start(self) -> None:
51+
self._event_debouncer.start()
52+
53+
def stop(self) -> None:
54+
if self._event_debouncer.is_alive():
55+
self._event_debouncer.stop()
56+
self._event_debouncer.join()
57+
58+
def dispatch(self, event: FileSystemEvent) -> None:
59+
# filter events and debounce
60+
if not self._ignore_event(event):
61+
self._event_debouncer.handle_event(event)
62+
63+
def _on_file_changes(self, events: list[FileSystemEvent]) -> None:
64+
# skip synchronization hassle by delegating this to the event loop in the main thread
65+
asyncio.run_coroutine_threadsafe(self._notify_callback(), self._loop)
66+
67+
def _ignore_event(self, event: FileSystemEvent) -> bool:
68+
"""Ignores events that should not trigger a rebuild.
69+
70+
Args:
71+
event: The event to check.
72+
73+
Returns:
74+
`True` if event should be ignored, otherwise `False`.
75+
"""
76+
if isinstance(event, FileOpenedEvent | FileClosedEvent):
77+
return True
78+
79+
# ignore events events in `dist` dir
80+
relevant_path = event.dest_path if isinstance(event, FileSystemMovedEvent) else event.src_path
81+
try:
82+
return Path(relevant_path).relative_to(self._watch_path).parts[0] == DIST_DIR
83+
except IndexError:
84+
return False
85+
86+
87+
class Watcher(AbstractAsyncContextManager):
88+
"""Watch a package source path and rebuild package/restart server on file changes."""
89+
90+
def __init__(
91+
self, source_path: Path, pkg_location: DirPackageLocation, state_storage_path: Path, host: str, port: int
92+
) -> None:
93+
self._source_path = source_path
94+
self._pkg_location = pkg_location
95+
self._host = host
96+
self._port = port
97+
98+
self._event_handler = _EventHandler(asyncio.get_running_loop(), self._notify, self._source_path)
99+
self._observer = Observer()
100+
self._webserver = WebServer(self._pkg_location, state_storage_path, self._host, self._port)
101+
self._on_change_event = asyncio.Event()
102+
self._watch: ObservedWatch | None = None
103+
104+
async def __aenter__(self) -> Self:
105+
self._event_handler.start()
106+
self._observer.start()
107+
log.info("Watching '%s' for changes...", self._source_path)
108+
109+
return self
110+
111+
async def __aexit__(
112+
self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None
113+
) -> None:
114+
if self._observer.is_alive():
115+
self._observer.stop()
116+
self._event_handler.stop()
117+
await self._webserver.stop_server()
118+
119+
def _schedule(self) -> None:
120+
if self._watch is None:
121+
log.debug("Starting file watching...")
122+
self._watch = self._observer.schedule(self._event_handler, self._source_path, recursive=True)
123+
124+
def _unschedule(self) -> None:
125+
if self._watch:
126+
log.debug("Stopping file watching...")
127+
self._observer.unschedule(self._watch)
128+
self._watch = None
129+
130+
async def _notify(self) -> None:
131+
self._on_change_event.set()
132+
133+
async def run_forever(self) -> None:
134+
try:
135+
await self._webserver.start_server()
136+
except Exception:
137+
log.exception("Failed to start webserver. The exception was:")
138+
# When user messed up the their package on initial run, we just bail out.
139+
return
140+
141+
self._schedule()
142+
143+
while True:
144+
await self._on_change_event.wait()
145+
146+
# Try to rebuild package and restart web server which might fail.
147+
self._unschedule()
148+
await self._rebuild_and_restart()
149+
self._schedule()
150+
151+
self._on_change_event.clear()
152+
153+
async def _rebuild_and_restart(self) -> None:
154+
log.info("File changes detected. Rebuilding package...")
155+
156+
# Stop webserver.
157+
try:
158+
await self._webserver.stop_server()
159+
except Exception:
160+
log.exception("Failed to stop web server. The exception was:")
161+
raise # Should not happen, thus we're propagating.
162+
163+
# Build package.
164+
try:
165+
package_source = PackageSource(self._source_path)
166+
with DirPackageBuilder(package_source) as builder:
167+
builder.write_package()
168+
except (PackageBuildError, PackageSourceValidationError):
169+
log.exception("Failed to build package. The exception was:")
170+
return
171+
172+
# Start server.
173+
try:
174+
await self._webserver.start_server()
175+
except Exception:
176+
log.exception("Failed to start web server. The exception was:")

‎questionpy_sdk/webserver/app.py

+54-19
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
# This file is part of the QuestionPy SDK. (https://questionpy.org)
22
# The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md.
33
# (c) Technische Universität Berlin, innoCampus <[email protected]>
4+
import asyncio
5+
import logging
46
import traceback
57
from enum import StrEnum
68
from functools import cached_property
@@ -22,6 +24,8 @@
2224
if TYPE_CHECKING:
2325
from questionpy_server.worker import Worker
2426

27+
log = logging.getLogger("questionpy-sdk:web-server")
28+
2529

2630
async def _extract_manifest(app: web.Application) -> None:
2731
webserver = app[SDK_WEBSERVER_APP_KEY]
@@ -59,29 +63,37 @@ def __init__(
5963
self,
6064
package_location: PackageLocation,
6165
state_storage_path: Path,
66+
host: str = "localhost",
67+
port: int = 8080,
6268
) -> None:
63-
# We import here, so we don't have to work around circular imports.
64-
from questionpy_sdk.webserver.routes.attempt import routes as attempt_routes # noqa: PLC0415
65-
from questionpy_sdk.webserver.routes.options import routes as options_routes # noqa: PLC0415
66-
from questionpy_sdk.webserver.routes.worker import routes as worker_routes # noqa: PLC0415
67-
6869
self.package_location = package_location
6970
self._state_storage_root = state_storage_path
71+
self._host = host
72+
self._port = port
7073

71-
self.web_app = web.Application()
72-
self.web_app[SDK_WEBSERVER_APP_KEY] = self
74+
self._web_app: web.Application | None = None
75+
self._runner: web.AppRunner | None = None
76+
self.worker_pool: WorkerPool = WorkerPool(1, 500 * MiB, worker_type=ThreadWorker)
7377

74-
self.web_app.add_routes(attempt_routes)
75-
self.web_app.add_routes(options_routes)
76-
self.web_app.add_routes(worker_routes)
77-
self.web_app.router.add_static("/static", Path(__file__).parent / "static")
78+
async def start_server(self) -> None:
79+
if self._web_app:
80+
msg = "Web app is already running"
81+
raise RuntimeError(msg)
7882

79-
self.web_app.on_startup.append(_extract_manifest)
80-
self.web_app.middlewares.append(_invalid_question_state_middleware)
83+
self._web_app = self._create_webapp()
84+
self._runner = web.AppRunner(self._web_app)
85+
await self._runner.setup()
86+
await web.TCPSite(self._runner, self._host, self._port).start()
8187

82-
jinja2_extensions = ["jinja2.ext.do"]
83-
aiohttp_jinja2.setup(self.web_app, loader=PackageLoader(__package__), extensions=jinja2_extensions)
84-
self.worker_pool: WorkerPool = WorkerPool(1, 500 * MiB, worker_type=ThreadWorker)
88+
async def stop_server(self) -> None:
89+
if self._runner:
90+
await self._runner.cleanup()
91+
self._web_app = None
92+
self._runner = None
93+
94+
async def run_forever(self) -> None:
95+
await self.start_server()
96+
await asyncio.Event().wait() # run forever
8597

8698
def read_state_file(self, filename: StateFilename) -> str | None:
8799
try:
@@ -100,12 +112,35 @@ def delete_state_files(self, filename_1: StateFilename, *filenames: StateFilenam
100112
# Remove package state dir if it's now empty.
101113
self._package_state_dir.rmdir()
102114

103-
def start_server(self) -> None:
104-
web.run_app(self.web_app)
115+
def _create_webapp(self) -> web.Application:
116+
# We import here, so we don't have to work around circular imports.
117+
from questionpy_sdk.webserver.routes.attempt import routes as attempt_routes # noqa: PLC0415
118+
from questionpy_sdk.webserver.routes.options import routes as options_routes # noqa: PLC0415
119+
from questionpy_sdk.webserver.routes.worker import routes as worker_routes # noqa: PLC0415
120+
121+
app = web.Application()
122+
app[SDK_WEBSERVER_APP_KEY] = self
123+
124+
app.add_routes(attempt_routes)
125+
app.add_routes(options_routes)
126+
app.add_routes(worker_routes)
127+
app.router.add_static("/static", Path(__file__).parent / "static")
128+
129+
app.on_startup.append(_extract_manifest)
130+
app.middlewares.append(_invalid_question_state_middleware)
131+
132+
jinja2_extensions = ["jinja2.ext.do"]
133+
aiohttp_jinja2.setup(app, loader=PackageLoader(__package__), extensions=jinja2_extensions)
134+
135+
return app
105136

106137
@cached_property
107138
def _package_state_dir(self) -> Path:
108-
manifest = self.web_app[MANIFEST_APP_KEY]
139+
if self._web_app is None:
140+
msg = "Web app not initialized"
141+
raise RuntimeError(msg)
142+
143+
manifest = self._web_app[MANIFEST_APP_KEY]
109144
return self._state_storage_root / f"{manifest.namespace}-{manifest.short_name}-{manifest.version}"
110145

111146

‎tests/conftest.py

+7
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
# This file is part of the QuestionPy SDK. (https://questionpy.org)
22
# The QuestionPy SDK is free software released under terms of the MIT license. See LICENSE.md.
33
# (c) Technische Universität Berlin, innoCampus <[email protected]>
4+
5+
from collections.abc import Callable
46
from pathlib import Path
57
from shutil import copytree
68

@@ -19,3 +21,8 @@ def source_path(request: pytest.FixtureRequest, tmp_path: Path) -> Path:
1921
copytree(src_path, dest_path, ignore=lambda src, names: (DIST_DIR,))
2022

2123
return dest_path
24+
25+
26+
@pytest.fixture
27+
def port(unused_tcp_port_factory: Callable) -> int:
28+
return unused_tcp_port_factory()

0 commit comments

Comments
 (0)