Build: POC of build.commands

Minimal implementation of `build.commands` as a POC to show how it could work.

It's using a `.readthedocs.yaml` similar to this one:

```yaml
version: 2

build:
  os: ubuntu-20.04
  commands:
    - mkdir output/
    - echo "Hello world" > output/index.html

  tools:
    python: "3"
```

This, of course, is not a good implementation and it's done pretty quick as a
way to show what parts of the code are required to be touched. I think this will
help with the discussion about how the UX around `build.commands` could work.

It defines an "implicit contract" where the `output/` folder under the
repository checkout will be uploaded to S3 and no HTML integrations will be done.

Author: humitos

readthedocs/config/config.py

@@ -791,6 +791,11 @@ def validate_build_config_with_tools(self):
                     BuildJobs.__slots__,
                 )
 
+        commands = []
+        with self.catch_validation_error("build.commands"):
+            commands = self.pop_config("build.commands")
+            validate_list(commands)
+
         if not tools:
             self.error(
                 key='build.tools',
@@ -802,13 +807,25 @@ def validate_build_config_with_tools(self):
                 code=CONFIG_REQUIRED,
             )
 
+        if commands and jobs:
+            self.error(
+                key="build.commands",
+                message="The keys build.jobs and build.commands can't be used together.",
+                code=INVALID_KEYS_COMBINATION,
+            )
+
         build["jobs"] = {}
         for job, commands in jobs.items():
             with self.catch_validation_error(f"build.jobs.{job}"):
                 build["jobs"][job] = [
                     validate_string(command) for command in validate_list(commands)
                 ]
 
+        build["commands"] = []
+        for command in commands:
+            with self.catch_validation_error("build.commands"):
+                build["commands"].append(validate_string(command))
+
         build['tools'] = {}
         for tool, version in tools.items():
             with self.catch_validation_error(f'build.tools.{tool}'):
@@ -1293,6 +1310,7 @@ def build(self):
                 os=build['os'],
                 tools=tools,
                 jobs=BuildJobs(**build["jobs"]),
+                commands=build["commands"],
                 apt_packages=build["apt_packages"],
             )
         return Build(**build)

readthedocs/config/models.py

@@ -37,10 +37,11 @@ def __init__(self, **kwargs):
 
 class BuildWithTools(Base):
 
-    __slots__ = ("os", "tools", "jobs", "apt_packages")
+    __slots__ = ("os", "tools", "jobs", "apt_packages", "commands")
 
     def __init__(self, **kwargs):
-        kwargs.setdefault('apt_packages', [])
+        kwargs.setdefault("apt_packages", [])
+        kwargs.setdefault("commands", [])
         super().__init__(**kwargs)
 
 

readthedocs/doc_builder/director.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 from collections import defaultdict
 
 import structlog
@@ -335,6 +336,24 @@ def run_build_job(self, job):
         for command in commands:
             environment.run(*command.split(), escape_command=False, cwd=cwd)
 
+    def run_build_commands(self):
+        cwd = self.data.project.checkout_path(self.data.version.slug)
+        environment = self.vcs_environment
+        for command in self.data.config.build.commands:
+            environment.run(*command.split(), escape_command=False, cwd=cwd)
+
+        # Copy files to artifacts path so they are uploaded to S3
+        target = self.data.project.artifact_path(
+            version=self.data.version.slug,
+            type_="sphinx",
+        )
+        artifacts_path = os.path.join(cwd, "output")
+        shutil.copytree(
+            artifacts_path,
+            target,
+            # ignore=shutil.ignore_patterns(*self.ignore_patterns),
+        )
+
     # Helpers
     #
     # TODO: move somewhere or change names to make them private or something to

readthedocs/projects/tasks/builds.py

@@ -6,6 +6,7 @@
 """
 import signal
 import socket
+from collections import defaultdict
 
 import structlog
 from celery import Task
@@ -601,13 +602,21 @@ def execute(self):
         self.data.build_director.create_build_environment()
         with self.data.build_director.build_environment:
             try:
-                # Installing
-                self.update_build(state=BUILD_STATE_INSTALLING)
-                self.data.build_director.setup_environment()
-
-                # Building
-                self.update_build(state=BUILD_STATE_BUILDING)
-                self.data.build_director.build()
+                # NOTE: check if the build uses `build.commands` and only run those
+                if self.data.config.build.commands:
+                    self.update_build(state=BUILD_STATE_BUILDING)
+                    self.data.build_director.run_build_commands()
+
+                    self.data.outcomes = defaultdict(lambda: False)
+                    self.data.outcomes["html"] = True
+                else:
+                    # Installing
+                    self.update_build(state=BUILD_STATE_INSTALLING)
+                    self.data.build_director.setup_environment()
+
+                    # Building
+                    self.update_build(state=BUILD_STATE_BUILDING)
+                    self.data.build_director.build()
             finally:
                 self.data.build_data = self.collect_build_data()