diff --git a/packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/common.ts b/packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/common.ts
index 984efbe979a1..f05aff592888 100644
--- a/packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/common.ts
+++ b/packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/common.ts
@@ -468,6 +468,8 @@ export function getCommonConfig(wco: WebpackConfigOptions): Configuration {
         },
         {
           test: /\.js$/,
+          // Factory files are processed by BO in the rules added in typescript.ts.
+          exclude: /(ngfactory|ngstyle)\.js$/,
           ...buildOptimizerUseRule,
         },
         {
diff --git a/packages/ngtools/webpack/src/compiler_host.ts b/packages/ngtools/webpack/src/compiler_host.ts
index ff943bc5256c..93bf04cff1d2 100644
--- a/packages/ngtools/webpack/src/compiler_host.ts
+++ b/packages/ngtools/webpack/src/compiler_host.ts
@@ -141,10 +141,16 @@ export class WebpackCompilerHost implements ts.CompilerHost {
       return;
     }
 
-    for (const ext of this._virtualStyleFileExtensions) {
-      const virtualFile = (fullPath + ext) as Path;
-      if (this._memoryHost.exists(virtualFile)) {
-        this._memoryHost.delete(virtualFile);
+    if (!exists) {
+      // At this point we're only looking at resource files (html/css/scss/etc).
+      // If the original was deleted, we should delete the virtual files too.
+      // If the original wasn't deleted we should leave them to be overwritten, because webpack
+      // might begin the loading process before our plugin has re-emitted them.
+      for (const ext of this._virtualStyleFileExtensions) {
+        const virtualFile = (fullPath + ext) as Path;
+        if (this._memoryHost.exists(virtualFile)) {
+          this._memoryHost.delete(virtualFile);
+        }
       }
     }
   }