Change the machine_cache folder to always be /var/meadowrun/machine_cache. Then it is truly per-machine rather than per-run_job_local configuration, and it will be the same across container and non-container jobs. Also document this feature

Author: hrichardlee

build_scripts/build_image_shared.py

@@ -5,6 +5,8 @@
 
 import asyncssh
 
+from meadowrun.run_job_local import MACHINE_CACHE_FOLDER
+
 if TYPE_CHECKING:
     from meadowrun.run_job_core import CloudProviderType
 from meadowrun.ssh import (
@@ -134,4 +136,7 @@ async def upload_and_configure_meadowrun(
         "sudo systemctl enable --now meadowrun-check-spot-eviction.timer",
     )
 
+    # TODO move this into the base image at some point
+    await run_and_print(connection, f"mkdir -p {MACHINE_CACHE_FOLDER}")
+
     return image_name

docs/how_to/.pages

@@ -6,5 +6,6 @@ nav:
 - private_git_repo_aws.md
 - private_git_repo_azure.md
 - ssh_to_instance.md
+- machine_cache.md
 - kubernetes.md
-- private_container_kubernetes.md
+- private_container_kubernetes.md

docs/how_to/machine_cache.md

@@ -0,0 +1,43 @@
+# Cache data on the underlying machine
+
+Your Meadowrun code will usually run in a container, which means that it will not be
+able to write files to the underlying file system. Meadowrun provides
+`meadowrun.MACHINE_CACHE_FOLDER` which points to `/var/meadowrun/machine_cache` when
+running on EC2 or Azure VMs. You can write data to this folder and it will be visible to
+any jobs that happen to run on the same node.
+
+The general pattern for using this folder should be something like:
+
+```python
+import os.path
+import filelock
+import meadowrun
+
+
+def a_slow_computation():
+    return "some sample data"
+
+
+def get_cached_data():
+    cached_data_filename = os.path.join(meadowrun.MACHINE_CACHE_FOLDER, "myfile")
+
+    with filelock.FileLock(f"{cached_data_filename}.lock"):
+        if not os.path.exists(cached_data_filename):
+            data = a_slow_computation()
+            with open(cached_data_filename, "w") as cached_data_file:
+                cached_data_file.write(data)
+
+            return data
+        else:
+            with open(cached_data_filename, "r") as cached_data_file:
+                return cached_data_file.read()
+```
+
+`filelock` is a [library](https://py-filelock.readthedocs.io/en/latest/) which makes
+sure only one process at a time is writing to the specified file. You're welcome to use
+whatever locking mechanism you like, but you should never assume that your job is the
+only process running on the machine.
+
+You should also never assume that any data you wrote will be available for a subsequent
+job. Meadowrun does not provide a way to guarantee that two jobs will run on the same
+machine.

src/meadowrun/__init__.py

@@ -69,7 +69,7 @@ def new__del__(self: Any, _warn: Any = warnings.warn) -> None:
     run_map_as_completed,
 )
 from meadowrun.run_job_core import Resources, SshHost, TaskException
-from meadowrun.run_job_local import Host, LocalHost
+from meadowrun.run_job_local import Host, LocalHost, MACHINE_CACHE_FOLDER
 from meadowrun.version import __version__
 
 __all__ = [
@@ -96,6 +96,7 @@ def new__del__(self: Any, _warn: Any = warnings.warn) -> None:
     "Secret",
     "Host",
     "LocalHost",
+    "MACHINE_CACHE_FOLDER",
     "Resources",
     "SshHost",
     "Kubernetes",

src/meadowrun/run_job_local.py

@@ -87,6 +87,7 @@
 _MEADOWRUN_CONTEXT_VARIABLES = "MEADOWRUN_CONTEXT_VARIABLES"
 _MEADOWRUN_RESULT_FILE = "MEADOWRUN_RESULT_FILE"
 _MEADOWRUN_RESULT_PICKLE_PROTOCOL = "MEADOWRUN_RESULT_PICKLE_PROTOCOL"
+MACHINE_CACHE_FOLDER = "/var/meadowrun/machine_cache"
 
 
 def _string_pairs_to_dict(pairs: Iterable[StringPair]) -> Dict[str, str]:
@@ -458,7 +459,6 @@ async def _launch_container_job(
     log_file_name: str,
     job: Job,
     io_folder: str,
-    machine_cache_folder: str,
     sidecar_container_images: List[str],
 ) -> Tuple[str, Coroutine[Any, Any, ProcessState]]:
     """
@@ -474,7 +474,7 @@ async def _launch_container_job(
     continuation.
     """
 
-    binds: List[Tuple[str, str]] = [(machine_cache_folder, "/meadowrun/machine_cache")]
+    binds: List[Tuple[str, str]] = [(MACHINE_CACHE_FOLDER, MACHINE_CACHE_FOLDER)]
 
     if cwd_path is not None:
         unique_code_paths = list(dict.fromkeys(itertools.chain([cwd_path], code_paths)))
@@ -736,7 +736,7 @@ def _get_default_working_folder() -> str:
 
 def _set_up_working_folder(
     working_folder: Optional[str],
-) -> Tuple[str, str, str, str, str, str]:
+) -> Tuple[str, str, str, str, str]:
     """
     Sets the working_folder to a default if it's not set, creates the necessary
     subfolders, gets a machine-wide lock on the working folder, then returns io_folder,
@@ -760,23 +760,19 @@ def _set_up_working_folder(
     local_copies_folder = os.path.join(working_folder, "local_copies")
     # misc folder for e.g. storing environment export files sent from local machine
     misc_folder = os.path.join(working_folder, "misc")
-    # machine_cache folder for providing a machine-wide folder on the machine
-    machine_cache_folder = os.path.join(working_folder, "machine_cache")
 
     os.makedirs(io_folder, exist_ok=True)
     os.makedirs(job_logs_folder, exist_ok=True)
     os.makedirs(git_repos_folder, exist_ok=True)
     os.makedirs(local_copies_folder, exist_ok=True)
     os.makedirs(misc_folder, exist_ok=True)
-    os.makedirs(machine_cache_folder, exist_ok=True)
 
     return (
         io_folder,
         job_logs_folder,
         git_repos_folder,
         local_copies_folder,
         misc_folder,
-        machine_cache_folder,
     )
 
 
@@ -952,7 +948,6 @@ async def run_local(
         git_repos_folder,
         local_copies_folder,
         misc_folder,
-        machine_cache_folder,
     ) = _set_up_working_folder(working_folder)
 
     # unpickle credentials if necessary
@@ -1142,7 +1137,6 @@ async def run_local(
                 log_file_name,
                 job,
                 io_folder,
-                machine_cache_folder,
                 sidecar_containers,
             )
             # due to the way protobuf works, this is equivalent to None

tests/basics.py

@@ -293,9 +293,19 @@ def remote_function() -> str:
     async def test_pip_file_in_git_repo_with_data_file(self) -> None:
         """This test is doing double-duty, also checking for the machine_cache folder"""
 
-        def remote_function() -> Tuple[bool, str]:
+        # TODO upgrade the Meadowrun referenced in requirements.txt and move this inside
+        # remote_function
+        machine_cache_folder = meadowrun.MACHINE_CACHE_FOLDER
+
+        def remote_function() -> str:
+            # make sure the machine cache folder is writable
+            with open(
+                os.path.join(machine_cache_folder, "foo"), "w", encoding="utf-8"
+            ) as f:
+                f.write("test")
+
             with open("example_package/test.txt", encoding="utf-8") as f:
-                return os.path.isdir("/meadowrun/machine_cache"), f.read()
+                return f.read()
 
         results = await run_function(
             remote_function,
@@ -306,7 +316,7 @@ def remote_function() -> Tuple[bool, str]:
                 interpreter=PipRequirementsFile("requirements.txt", "3.9"),
             ),
         )
-        assert results == (True, "Hello world!")
+        assert results == "Hello world!"
 
     @pytest.mark.skipif("sys.version_info < (3, 8)")
     @pytest.mark.asyncio