Fix EMFILE error when running on Android

constants.ts

@@ -91,3 +91,5 @@ export const enum AnalyticsClients {
 	NonInteractive = "Non-interactive",
 	Unknown = "Unknown"
 }
+
+export const DEFAULT_CHUNK_SIZE = 100;

definitions/mobile.d.ts

@@ -671,9 +671,9 @@ declare module Mobile {
 		 */
 		getShasumsFromDevice(): Promise<IStringDictionary>;
 		/**
-		 * Computes the shasums of localToDevicePaths and changes the content of hash file on device
+		 * Uploads updated shasums to hash file on device
 		 */
-		uploadHashFileToDevice(data: IStringDictionary | Mobile.ILocalToDevicePathData[]): Promise<void>;
+		uploadHashFileToDevice(data: IStringDictionary): Promise<void>;
 		/**
 		 * Computes the shasums of localToDevicePaths and updates hash file on device
 		 */
@@ -688,6 +688,14 @@ declare module Mobile {
 		 * @return {Promise<boolean>} boolean True if file exists and false otherwise.
 		 */
 		doesShasumFileExistsOnDevice(): Promise<boolean>;
+
+		/**
+		 * Generates hashes of specified localToDevicePaths by chunks and persists them in the passed @shasums argument.
+		 * @param {Mobile.ILocalToDevicePathData[]} localToDevicePaths The localToDevicePaths objects for which the hashes should be generated.
+		 * @param {IStringDicitionary} shasums Object in which the shasums will be persisted.
+		 * @returns {Promise<string>[]} DevicePaths of all elements from the input localToDevicePaths.
+		 */
+		generateHashesFromLocalToDevicePaths(localToDevicePaths: Mobile.ILocalToDevicePathData[], shasums: IStringDictionary): Promise<string[]>;
 	}
 
 	/**

helpers.ts

@@ -8,6 +8,24 @@ import * as crypto from "crypto";
 
 const Table = require("cli-table");
 
+export async function executeActionByChunks<T>(initialData: T[] | IDictionary<T>, chunkSize: number, elementAction: (element: T, key?: string | number) => Promise<any>): Promise<void> {
+	let arrayToChunk: (T | string)[];
+	let action: (key: string | T) => Promise<any>;
+
+	if (_.isArray(initialData)) {
+		arrayToChunk = initialData;
+		action = (element: T) => elementAction(element, initialData.indexOf(element));
+	} else {
+		arrayToChunk = _.keys(initialData);
+		action = (key: string) => elementAction(initialData[key], key);
+	}
+
+	const chunks = _.chunk(arrayToChunk, chunkSize);
+	for (const chunk of chunks) {
+		await Promise.all(_.map(chunk, element => action(element)));
+	}
+}
+
 export function deferPromise<T>(): IDeferPromise<T> {
 	let resolve: (value?: T | PromiseLike<T>) => void;
 	let reject: (reason?: any) => void;

mobile/android/android-device-file-system.ts

@@ -1,6 +1,8 @@
 import * as path from "path";
 import * as temp from "temp";
 import { AndroidDeviceHashService } from "./android-device-hash-service";
+import { executeActionByChunks } from "../../helpers";
+import { DEFAULT_CHUNK_SIZE } from '../../constants';
 
 export class AndroidDeviceFileSystem implements Mobile.IDeviceFileSystem {
 	private _deviceHashServices = Object.create(null);
@@ -51,28 +53,24 @@ export class AndroidDeviceFileSystem implements Mobile.IDeviceFileSystem {
 	}
 
 	public async transferFiles(deviceAppData: Mobile.IDeviceAppData, localToDevicePaths: Mobile.ILocalToDevicePathData[]): Promise<void> {
-		// TODO: Do not start all promises simultaneously as this leads to error EMFILE on Windows for too many opened files.
-		// Use chunks (for example on 100).
-		await Promise.all(
-			_(localToDevicePaths)
-				.filter(localToDevicePathData => this.$fs.getFsStats(localToDevicePathData.getLocalPath()).isFile())
-				.map(async localToDevicePathData => {
-					const devicePath = localToDevicePathData.getDevicePath();
-					await this.adb.executeCommand(["push", localToDevicePathData.getLocalPath(), devicePath]);
-					await this.adb.executeShellCommand(["chmod", "0777", path.dirname(devicePath)]);
-				}
-				)
-				.value()
-		);
-
-		await Promise.all(
-			_(localToDevicePaths)
-				.filter(localToDevicePathData => this.$fs.getFsStats(localToDevicePathData.getLocalPath()).isDirectory())
-				.map(async localToDevicePathData =>
-					await this.adb.executeShellCommand(["chmod", "0777", localToDevicePathData.getDevicePath()])
-				)
-				.value()
-		);
+		const directoriesToChmod: string[] = [];
+		const action = async (localToDevicePathData: Mobile.ILocalToDevicePathData) => {
+			const fstat = this.$fs.getFsStats(localToDevicePathData.getLocalPath());
+			if (fstat.isFile()) {
+				const devicePath = localToDevicePathData.getDevicePath();
+				await this.adb.executeCommand(["push", localToDevicePathData.getLocalPath(), devicePath]);
+				await this.adb.executeShellCommand(["chmod", "0777", path.dirname(devicePath)]);
+			} else if (fstat.isDirectory()) {
+				const dirToChmod = localToDevicePathData.getDevicePath();
+				directoriesToChmod.push(dirToChmod);
+			}
+		};
+
+		await executeActionByChunks<Mobile.ILocalToDevicePathData>(localToDevicePaths, DEFAULT_CHUNK_SIZE, action);
+
+		const dirsChmodAction = (directoryToChmod: string) => this.adb.executeShellCommand(["chmod", "0777", directoryToChmod]);
+
+		await executeActionByChunks<string>(_.uniq(directoriesToChmod), DEFAULT_CHUNK_SIZE, dirsChmodAction);
 
 		// Update hashes
 		const deviceHashService = this.getDeviceHashService(deviceAppData.appIdentifier);
@@ -82,25 +80,12 @@ export class AndroidDeviceFileSystem implements Mobile.IDeviceFileSystem {
 	}
 
 	public async transferDirectory(deviceAppData: Mobile.IDeviceAppData, localToDevicePaths: Mobile.ILocalToDevicePathData[], projectFilesPath: string): Promise<Mobile.ILocalToDevicePathData[]> {
-		const devicePaths: string[] = [];
 		const currentShasums: IStringDictionary = {};
-
-		await Promise.all(
-			localToDevicePaths.map(async localToDevicePathData => {
-				const localPath = localToDevicePathData.getLocalPath();
-				const stats = this.$fs.getFsStats(localPath);
-				if (stats.isFile()) {
-					const fileShasum = await this.$fs.getFileShasum(localPath);
-					currentShasums[localPath] = fileShasum;
-				}
-
-				devicePaths.push(`"${localToDevicePathData.getDevicePath()}"`);
-			})
-		);
+		const deviceHashService = this.getDeviceHashService(deviceAppData.appIdentifier);
+		const devicePaths: string[] = await deviceHashService.generateHashesFromLocalToDevicePaths(localToDevicePaths, currentShasums);
 
 		const commandsDeviceFilePath = this.$mobileHelper.buildDevicePath(await deviceAppData.getDeviceProjectRootPath(), "nativescript.commands.sh");
 
-		const deviceHashService = this.getDeviceHashService(deviceAppData.appIdentifier);
 		let filesToChmodOnDevice: string[] = devicePaths;
 		let tranferredFiles: Mobile.ILocalToDevicePathData[] = [];
 		const oldShasums = await deviceHashService.getShasumsFromDevice();
@@ -113,16 +98,15 @@ export class AndroidDeviceFileSystem implements Mobile.IDeviceFileSystem {
 			const changedShasums: any = _.omitBy(currentShasums, (hash: string, pathToFile: string) => !!_.find(oldShasums, (oldHash: string, oldPath: string) => pathToFile === oldPath && hash === oldHash));
 			this.$logger.trace("Changed file hashes are:", changedShasums);
 			filesToChmodOnDevice = [];
-			await Promise.all(
-				_(changedShasums)
-					.map((hash: string, filePath: string) => _.find(localToDevicePaths, ldp => ldp.getLocalPath() === filePath))
-					.map(localToDevicePathData => {
-						tranferredFiles.push(localToDevicePathData);
-						filesToChmodOnDevice.push(`"${localToDevicePathData.getDevicePath()}"`);
-						return this.transferFile(localToDevicePathData.getLocalPath(), localToDevicePathData.getDevicePath());
-					})
-					.value()
-			);
+
+			const transferFileAction = async (hash: string, filePath: string) => {
+				const localToDevicePathData = _.find(localToDevicePaths, ldp => ldp.getLocalPath() === filePath);
+				tranferredFiles.push(localToDevicePathData);
+				filesToChmodOnDevice.push(`"${localToDevicePathData.getDevicePath()}"`);
+				return this.transferFile(localToDevicePathData.getLocalPath(), localToDevicePathData.getDevicePath());
+			};
+
+			await executeActionByChunks<string>(changedShasums, DEFAULT_CHUNK_SIZE, transferFileAction);
 		}
 
 		if (filesToChmodOnDevice.length) {

mobile/android/android-device-hash-service.ts

@@ -1,6 +1,8 @@
 import * as path from "path";
 import * as temp from "temp";
 import { cache } from "../../decorators";
+import { executeActionByChunks } from "../../helpers";
+import { DEFAULT_CHUNK_SIZE } from "../../constants";
 
 export class AndroidDeviceHashService implements Mobile.IAndroidDeviceHashService {
 	private static HASH_FILE_NAME = "hashes";
@@ -32,40 +34,15 @@ export class AndroidDeviceHashService implements Mobile.IAndroidDeviceHashServic
 		return null;
 	}
 
-	public async uploadHashFileToDevice(data: IStringDictionary | Mobile.ILocalToDevicePathData[]): Promise<void> {
-		let shasums: IStringDictionary = {};
-		if (_.isArray(data)) {
-			await Promise.all(
-				(<Mobile.ILocalToDevicePathData[]>data).map(async localToDevicePathData => {
-					const localPath = localToDevicePathData.getLocalPath();
-					const stats = this.$fs.getFsStats(localPath);
-					if (stats.isFile()) {
-						const fileShasum = await this.$fs.getFileShasum(localPath);
-						shasums[localPath] = fileShasum;
-					}
-				})
-			);
-		} else {
-			shasums = <IStringDictionary>data;
-		}
-
-		this.$fs.writeJson(this.hashFileLocalPath, shasums);
+	public async uploadHashFileToDevice(data: IStringDictionary): Promise<void> {
+		this.$fs.writeJson(this.hashFileLocalPath, data);
 		await this.adb.executeCommand(["push", this.hashFileLocalPath, this.hashFileDevicePath]);
 	}
 
 	public async updateHashes(localToDevicePaths: Mobile.ILocalToDevicePathData[]): Promise<boolean> {
 		const oldShasums = await this.getShasumsFromDevice();
 		if (oldShasums) {
-			await Promise.all(
-				_.map(localToDevicePaths, async ldp => {
-					const localPath = ldp.getLocalPath();
-					if (this.$fs.getFsStats(localPath).isFile()) {
-						// TODO: Use relative to project path for key
-						// This will speed up livesync on the same device for the same project on different PCs.
-						oldShasums[localPath] = await this.$fs.getFileShasum(localPath);
-					}
-				})
-			);
+			await this.generateHashesFromLocalToDevicePaths(localToDevicePaths, oldShasums);
 
 			await this.uploadHashFileToDevice(oldShasums);
 
@@ -75,6 +52,24 @@ export class AndroidDeviceHashService implements Mobile.IAndroidDeviceHashServic
 		return false;
 	}
 
+	public async generateHashesFromLocalToDevicePaths(localToDevicePaths: Mobile.ILocalToDevicePathData[], shasums: IStringDictionary): Promise<string[]> {
+		const devicePaths: string[] = [];
+		const action = async (localToDevicePathData: Mobile.ILocalToDevicePathData) => {
+			const localPath = localToDevicePathData.getLocalPath();
+			if (this.$fs.getFsStats(localPath).isFile()) {
+				// TODO: Use relative to project path for key
+				// This will speed up livesync on the same device for the same project on different PCs.
+				shasums[localPath] = await this.$fs.getFileShasum(localPath);
+			}
+
+			devicePaths.push(`"${localToDevicePathData.getDevicePath()}"`);
+		};
+
+		await executeActionByChunks<Mobile.ILocalToDevicePathData>(localToDevicePaths, DEFAULT_CHUNK_SIZE, action);
+
+		return devicePaths;
+	}
+
 	public async removeHashes(localToDevicePaths: Mobile.ILocalToDevicePathData[]): Promise<boolean> {
 		const oldShasums = await this.getShasumsFromDevice();
 		if (oldShasums) {

test/unit-tests/helpers.ts

@@ -15,6 +15,61 @@ describe("helpers", () => {
 		assert.deepEqual(actualResult, testData.expectedResult, `For input ${testData.input}, the expected result is: ${testData.expectedResult}, but actual result is: ${actualResult}.`);
 	};
 
+	describe("executeActionByChunks", () => {
+		const chunkSize = 2;
+
+		const assertElements = (initialDataValues: any[], handledElements: any[], element: any, passedChunkSize: number) => {
+			return new Promise(resolve => setImmediate(() => {
+				const remainingElements = _.difference(initialDataValues, handledElements);
+				const isFromLastChunk = (element + passedChunkSize) > initialDataValues.length;
+				// If the element is one of the last chunk, the remainingElements must be empty.
+				// If the element is from any other chunk, the remainingElements must contain all elements outside of this chunk.
+				if (isFromLastChunk) {
+					assert.isTrue(!remainingElements.length);
+				} else {
+					const indexOfElement = initialDataValues.indexOf(element);
+					const chunkNumber = Math.floor(indexOfElement / passedChunkSize) + 1;
+					const expectedRemainingElements = _.drop(initialDataValues, chunkNumber * passedChunkSize);
+
+					assert.deepEqual(remainingElements, expectedRemainingElements);
+				}
+
+				resolve();
+			}));
+		};
+
+		it("works correctly with array", () => {
+			const initialData = _.range(7);
+			const handledElements: number[] = [];
+
+			return helpers.executeActionByChunks(initialData, chunkSize, (element: number, index: number) => {
+				handledElements.push(element);
+				assert.deepEqual(element, initialData[index]);
+				assert.isTrue(initialData.indexOf(element) !== -1);
+				return assertElements(initialData, handledElements, element, chunkSize);
+			});
+		});
+
+		it("works correctly with IDictionary", () => {
+			const initialData: IDictionary<any> = {
+				a: 1,
+				b: 2,
+				c: 3,
+				d: 4,
+				e: 5,
+				f: 6
+			};
+
+			const initialDataValues = _.values(initialData);
+			const handledElements: number[] = [];
+			return helpers.executeActionByChunks(initialData, chunkSize, (element, key) => {
+				handledElements.push(element);
+				assert.isTrue(initialData[key] === element);
+				return assertElements(initialDataValues, handledElements, element, chunkSize);
+			});
+		});
+	});
+
 	describe("getPropertyName", () => {
 		const ES5Functions: ITestData[] = [
 			{