diff --git a/devcontainer/devcontainer.go b/devcontainer/devcontainer.go
index 92f3a8fa..19259072 100644
--- a/devcontainer/devcontainer.go
+++ b/devcontainer/devcontainer.go
@@ -56,11 +56,22 @@ type Compiled struct {
 	Env  []string
 }
 
+// HasImage returns true if the devcontainer.json specifies an image.
+func (s Spec) HasImage() bool {
+	return s.Image != ""
+}
+
+// HasDockerfile returns true if the devcontainer.json specifies the path to a
+// Dockerfile.
+func (s Spec) HasDockerfile() bool {
+	return s.Dockerfile != "" || s.Build.Dockerfile != ""
+}
+
 // Compile returns the build parameters for the workspace.
 // devcontainerDir is the path to the directory where the devcontainer.json file
 // is located. scratchDir is the path to the directory where the Dockerfile will
 // be written to if one doesn't exist.
-func (s *Spec) Compile(fs billy.Filesystem, devcontainerDir, scratchDir string) (*Compiled, error) {
+func (s *Spec) Compile(fs billy.Filesystem, devcontainerDir, scratchDir, fallbackDockerfile string) (*Compiled, error) {
 	env := make([]string, 0)
 	for key, value := range s.RemoteEnv {
 		env = append(env, key+"="+value)
@@ -93,7 +104,11 @@ func (s *Spec) Compile(fs billy.Filesystem, devcontainerDir, scratchDir string)
 			s.Build.Context = s.Context
 		}
 
-		params.DockerfilePath = filepath.Join(devcontainerDir, s.Build.Dockerfile)
+		if s.Build.Dockerfile != "" {
+			params.DockerfilePath = filepath.Join(devcontainerDir, s.Build.Dockerfile)
+		} else {
+			params.DockerfilePath = fallbackDockerfile
+		}
 		params.BuildContext = filepath.Join(devcontainerDir, s.Build.Context)
 	}
 
diff --git a/devcontainer/devcontainer_test.go b/devcontainer/devcontainer_test.go
index c19dfd96..1beb55f5 100644
--- a/devcontainer/devcontainer_test.go
+++ b/devcontainer/devcontainer_test.go
@@ -44,7 +44,7 @@ func TestCompileDevContainer(t *testing.T) {
 		dc := &devcontainer.Spec{
 			Image: "codercom/code-server:latest",
 		}
-		params, err := dc.Compile(fs, "", envbuilder.MagicDir)
+		params, err := dc.Compile(fs, "", envbuilder.MagicDir, "")
 		require.NoError(t, err)
 		require.Equal(t, filepath.Join(envbuilder.MagicDir, "Dockerfile"), params.DockerfilePath)
 		require.Equal(t, envbuilder.MagicDir, params.BuildContext)
@@ -69,7 +69,7 @@ func TestCompileDevContainer(t *testing.T) {
 		_, err = io.WriteString(file, "FROM ubuntu")
 		require.NoError(t, err)
 		_ = file.Close()
-		params, err := dc.Compile(fs, dcDir, envbuilder.MagicDir)
+		params, err := dc.Compile(fs, dcDir, envbuilder.MagicDir, "")
 		require.NoError(t, err)
 		require.Equal(t, "ARG1=value1", params.BuildArgs[0])
 		require.Equal(t, filepath.Join(dcDir, "Dockerfile"), params.DockerfilePath)
diff --git a/envbuilder.go b/envbuilder.go
index c282dd26..f926fdcb 100644
--- a/envbuilder.go
+++ b/envbuilder.go
@@ -314,36 +314,34 @@ func Run(ctx context.Context, options Options) error {
 		}
 	}
 
-	var buildParams *devcontainer.Compiled
-
-	defaultBuildParams := func() error {
+	defaultBuildParams := func() (*devcontainer.Compiled, error) {
 		dockerfile := filepath.Join(MagicDir, "Dockerfile")
 		file, err := options.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0644)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		defer file.Close()
 		if options.FallbackImage == "" {
 			if fallbackErr != nil {
-				return xerrors.Errorf("%s: %w", fallbackErr.Error(), ErrNoFallbackImage)
+				return nil, xerrors.Errorf("%s: %w", fallbackErr.Error(), ErrNoFallbackImage)
 			}
 			// We can't use errors.Join here because our tests
 			// don't support parsing a multiline error.
-			return ErrNoFallbackImage
+			return nil, ErrNoFallbackImage
 		}
 		content := "FROM " + options.FallbackImage
 		_, err = file.Write([]byte(content))
 		if err != nil {
-			return err
+			return nil, err
 		}
-		buildParams = &devcontainer.Compiled{
+		return &devcontainer.Compiled{
 			DockerfilePath:    dockerfile,
 			DockerfileContent: content,
 			BuildContext:      MagicDir,
-		}
-		return nil
+		}, nil
 	}
 
+	var buildParams *devcontainer.Compiled
 	if options.DockerfilePath == "" {
 		// Only look for a devcontainer if a Dockerfile wasn't specified.
 		// devcontainer is a standard, so it's reasonable to be the default.
@@ -364,7 +362,16 @@ func Run(ctx context.Context, options Options) error {
 			}
 			devContainer, err := devcontainer.Parse(content)
 			if err == nil {
-				buildParams, err = devContainer.Compile(options.Filesystem, devcontainerDir, MagicDir)
+				var fallbackDockerfile string
+				if !devContainer.HasImage() && !devContainer.HasDockerfile() {
+					defaultParams, err := defaultBuildParams()
+					if err != nil {
+						return fmt.Errorf("no Dockerfile or image found: %w", err)
+					}
+					logf(codersdk.LogLevelInfo, "No Dockerfile or image specified; falling back to the default image...")
+					fallbackDockerfile = defaultParams.DockerfilePath
+				}
+				buildParams, err = devContainer.Compile(options.Filesystem, devcontainerDir, MagicDir, fallbackDockerfile)
 				if err != nil {
 					return fmt.Errorf("compile devcontainer.json: %w", err)
 				}
@@ -393,7 +400,8 @@ func Run(ctx context.Context, options Options) error {
 	if buildParams == nil {
 		// If there isn't a devcontainer.json file in the repository,
 		// we fallback to whatever the `DefaultImage` is.
-		err := defaultBuildParams()
+		var err error
+		buildParams, err = defaultBuildParams()
 		if err != nil {
 			return fmt.Errorf("no Dockerfile or devcontainer.json found: %w", err)
 		}
@@ -538,7 +546,7 @@ func Run(ctx context.Context, options Options) error {
 		}
 		logf(codersdk.LogLevelError, "Failed to build: %s", err)
 		logf(codersdk.LogLevelError, "Falling back to the default image...")
-		err = defaultBuildParams()
+		buildParams, err = defaultBuildParams()
 		if err != nil {
 			return err
 		}
diff --git a/integration/integration_test.go b/integration/integration_test.go
index 1a605ce0..3c3fc8b6 100644
--- a/integration/integration_test.go
+++ b/integration/integration_test.go
@@ -264,6 +264,22 @@ RUN exit 1`,
 		})
 		require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error())
 	})
+	t.Run("NoImageOrDockerfile", func(t *testing.T) {
+		t.Parallel()
+		url := createGitServer(t, gitServerOptions{
+			files: map[string]string{
+				".devcontainer/devcontainer.json": "{}",
+			},
+		})
+		ctr, err := runEnvbuilder(t, []string{
+			"GIT_URL=" + url,
+			"FALLBACK_IMAGE=alpine:latest",
+		})
+		require.NoError(t, err)
+
+		output := execContainer(t, ctr, "echo hello")
+		require.Equal(t, "hello", strings.TrimSpace(output))
+	})
 }
 
 func TestPrivateRegistry(t *testing.T) {