Skip to content
This repository was archived by the owner on Feb 2, 2021. It is now read-only.

Commit 0d98103

Browse files
rosen-vladimirovrosen-vladimirov
authored
Merge pull request #1036 from telerik/vladimirov/fix-emfile-error
Fix EMFILE error when running on Android
2 parents 1bb198e + 0d1a466 commit 0d98103

File tree

6 files changed

+139
-77
lines changed

6 files changed

+139
-77
lines changed

‎constants.ts

+2
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,5 @@ export const enum AnalyticsClients {
9191
NonInteractive = "Non-interactive",
9292
Unknown = "Unknown"
9393
}
94+
95+
export const DEFAULT_CHUNK_SIZE = 100;

‎definitions/mobile.d.ts

+10-2
Original file line numberDiff line numberDiff line change
@@ -671,9 +671,9 @@ declare module Mobile {
671671
*/
672672
getShasumsFromDevice(): Promise<IStringDictionary>;
673673
/**
674-
* Computes the shasums of localToDevicePaths and changes the content of hash file on device
674+
* Uploads updated shasums to hash file on device
675675
*/
676-
uploadHashFileToDevice(data: IStringDictionary | Mobile.ILocalToDevicePathData[]): Promise<void>;
676+
uploadHashFileToDevice(data: IStringDictionary): Promise<void>;
677677
/**
678678
* Computes the shasums of localToDevicePaths and updates hash file on device
679679
*/
@@ -688,6 +688,14 @@ declare module Mobile {
688688
* @return {Promise<boolean>} boolean True if file exists and false otherwise.
689689
*/
690690
doesShasumFileExistsOnDevice(): Promise<boolean>;
691+
692+
/**
693+
* Generates hashes of specified localToDevicePaths by chunks and persists them in the passed @shasums argument.
694+
* @param {Mobile.ILocalToDevicePathData[]} localToDevicePaths The localToDevicePaths objects for which the hashes should be generated.
695+
* @param {IStringDicitionary} shasums Object in which the shasums will be persisted.
696+
* @returns {Promise<string>[]} DevicePaths of all elements from the input localToDevicePaths.
697+
*/
698+
generateHashesFromLocalToDevicePaths(localToDevicePaths: Mobile.ILocalToDevicePathData[], shasums: IStringDictionary): Promise<string[]>;
691699
}
692700

693701
/**

‎helpers.ts

+18
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,24 @@ import * as crypto from "crypto";
88

99
const Table = require("cli-table");
1010

11+
export async function executeActionByChunks<T>(initialData: T[] | IDictionary<T>, chunkSize: number, elementAction: (element: T, key?: string | number) => Promise<any>): Promise<void> {
12+
let arrayToChunk: (T | string)[];
13+
let action: (key: string | T) => Promise<any>;
14+
15+
if (_.isArray(initialData)) {
16+
arrayToChunk = initialData;
17+
action = (element: T) => elementAction(element, initialData.indexOf(element));
18+
} else {
19+
arrayToChunk = _.keys(initialData);
20+
action = (key: string) => elementAction(initialData[key], key);
21+
}
22+
23+
const chunks = _.chunk(arrayToChunk, chunkSize);
24+
for (const chunk of chunks) {
25+
await Promise.all(_.map(chunk, element => action(element)));
26+
}
27+
}
28+
1129
export function deferPromise<T>(): IDeferPromise<T> {
1230
let resolve: (value?: T | PromiseLike<T>) => void;
1331
let reject: (reason?: any) => void;

‎mobile/android/android-device-file-system.ts

+31-47
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import * as path from "path";
22
import * as temp from "temp";
33
import { AndroidDeviceHashService } from "./android-device-hash-service";
4+
import { executeActionByChunks } from "../../helpers";
5+
import { DEFAULT_CHUNK_SIZE } from '../../constants';
46

57
export class AndroidDeviceFileSystem implements Mobile.IDeviceFileSystem {
68
private _deviceHashServices = Object.create(null);
@@ -51,28 +53,24 @@ export class AndroidDeviceFileSystem implements Mobile.IDeviceFileSystem {
5153
}
5254

5355
public async transferFiles(deviceAppData: Mobile.IDeviceAppData, localToDevicePaths: Mobile.ILocalToDevicePathData[]): Promise<void> {
54-
// TODO: Do not start all promises simultaneously as this leads to error EMFILE on Windows for too many opened files.
55-
// Use chunks (for example on 100).
56-
await Promise.all(
57-
_(localToDevicePaths)
58-
.filter(localToDevicePathData => this.$fs.getFsStats(localToDevicePathData.getLocalPath()).isFile())
59-
.map(async localToDevicePathData => {
60-
const devicePath = localToDevicePathData.getDevicePath();
61-
await this.adb.executeCommand(["push", localToDevicePathData.getLocalPath(), devicePath]);
62-
await this.adb.executeShellCommand(["chmod", "0777", path.dirname(devicePath)]);
63-
}
64-
)
65-
.value()
66-
);
67-
68-
await Promise.all(
69-
_(localToDevicePaths)
70-
.filter(localToDevicePathData => this.$fs.getFsStats(localToDevicePathData.getLocalPath()).isDirectory())
71-
.map(async localToDevicePathData =>
72-
await this.adb.executeShellCommand(["chmod", "0777", localToDevicePathData.getDevicePath()])
73-
)
74-
.value()
75-
);
56+
const directoriesToChmod: string[] = [];
57+
const action = async (localToDevicePathData: Mobile.ILocalToDevicePathData) => {
58+
const fstat = this.$fs.getFsStats(localToDevicePathData.getLocalPath());
59+
if (fstat.isFile()) {
60+
const devicePath = localToDevicePathData.getDevicePath();
61+
await this.adb.executeCommand(["push", localToDevicePathData.getLocalPath(), devicePath]);
62+
await this.adb.executeShellCommand(["chmod", "0777", path.dirname(devicePath)]);
63+
} else if (fstat.isDirectory()) {
64+
const dirToChmod = localToDevicePathData.getDevicePath();
65+
directoriesToChmod.push(dirToChmod);
66+
}
67+
};
68+
69+
await executeActionByChunks<Mobile.ILocalToDevicePathData>(localToDevicePaths, DEFAULT_CHUNK_SIZE, action);
70+
71+
const dirsChmodAction = (directoryToChmod: string) => this.adb.executeShellCommand(["chmod", "0777", directoryToChmod]);
72+
73+
await executeActionByChunks<string>(_.uniq(directoriesToChmod), DEFAULT_CHUNK_SIZE, dirsChmodAction);
7674

7775
// Update hashes
7876
const deviceHashService = this.getDeviceHashService(deviceAppData.appIdentifier);
@@ -82,25 +80,12 @@ export class AndroidDeviceFileSystem implements Mobile.IDeviceFileSystem {
8280
}
8381

8482
public async transferDirectory(deviceAppData: Mobile.IDeviceAppData, localToDevicePaths: Mobile.ILocalToDevicePathData[], projectFilesPath: string): Promise<Mobile.ILocalToDevicePathData[]> {
85-
const devicePaths: string[] = [];
8683
const currentShasums: IStringDictionary = {};
87-
88-
await Promise.all(
89-
localToDevicePaths.map(async localToDevicePathData => {
90-
const localPath = localToDevicePathData.getLocalPath();
91-
const stats = this.$fs.getFsStats(localPath);
92-
if (stats.isFile()) {
93-
const fileShasum = await this.$fs.getFileShasum(localPath);
94-
currentShasums[localPath] = fileShasum;
95-
}
96-
97-
devicePaths.push(`"${localToDevicePathData.getDevicePath()}"`);
98-
})
99-
);
84+
const deviceHashService = this.getDeviceHashService(deviceAppData.appIdentifier);
85+
const devicePaths: string[] = await deviceHashService.generateHashesFromLocalToDevicePaths(localToDevicePaths, currentShasums);
10086

10187
const commandsDeviceFilePath = this.$mobileHelper.buildDevicePath(await deviceAppData.getDeviceProjectRootPath(), "nativescript.commands.sh");
10288

103-
const deviceHashService = this.getDeviceHashService(deviceAppData.appIdentifier);
10489
let filesToChmodOnDevice: string[] = devicePaths;
10590
let tranferredFiles: Mobile.ILocalToDevicePathData[] = [];
10691
const oldShasums = await deviceHashService.getShasumsFromDevice();
@@ -113,16 +98,15 @@ export class AndroidDeviceFileSystem implements Mobile.IDeviceFileSystem {
11398
const changedShasums: any = _.omitBy(currentShasums, (hash: string, pathToFile: string) => !!_.find(oldShasums, (oldHash: string, oldPath: string) => pathToFile === oldPath && hash === oldHash));
11499
this.$logger.trace("Changed file hashes are:", changedShasums);
115100
filesToChmodOnDevice = [];
116-
await Promise.all(
117-
_(changedShasums)
118-
.map((hash: string, filePath: string) => _.find(localToDevicePaths, ldp => ldp.getLocalPath() === filePath))
119-
.map(localToDevicePathData => {
120-
tranferredFiles.push(localToDevicePathData);
121-
filesToChmodOnDevice.push(`"${localToDevicePathData.getDevicePath()}"`);
122-
return this.transferFile(localToDevicePathData.getLocalPath(), localToDevicePathData.getDevicePath());
123-
})
124-
.value()
125-
);
101+
102+
const transferFileAction = async (hash: string, filePath: string) => {
103+
const localToDevicePathData = _.find(localToDevicePaths, ldp => ldp.getLocalPath() === filePath);
104+
tranferredFiles.push(localToDevicePathData);
105+
filesToChmodOnDevice.push(`"${localToDevicePathData.getDevicePath()}"`);
106+
return this.transferFile(localToDevicePathData.getLocalPath(), localToDevicePathData.getDevicePath());
107+
};
108+
109+
await executeActionByChunks<string>(changedShasums, DEFAULT_CHUNK_SIZE, transferFileAction);
126110
}
127111

128112
if (filesToChmodOnDevice.length) {

‎mobile/android/android-device-hash-service.ts

+23-28
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import * as path from "path";
22
import * as temp from "temp";
33
import { cache } from "../../decorators";
4+
import { executeActionByChunks } from "../../helpers";
5+
import { DEFAULT_CHUNK_SIZE } from "../../constants";
46

57
export class AndroidDeviceHashService implements Mobile.IAndroidDeviceHashService {
68
private static HASH_FILE_NAME = "hashes";
@@ -32,40 +34,15 @@ export class AndroidDeviceHashService implements Mobile.IAndroidDeviceHashServic
3234
return null;
3335
}
3436

35-
public async uploadHashFileToDevice(data: IStringDictionary | Mobile.ILocalToDevicePathData[]): Promise<void> {
36-
let shasums: IStringDictionary = {};
37-
if (_.isArray(data)) {
38-
await Promise.all(
39-
(<Mobile.ILocalToDevicePathData[]>data).map(async localToDevicePathData => {
40-
const localPath = localToDevicePathData.getLocalPath();
41-
const stats = this.$fs.getFsStats(localPath);
42-
if (stats.isFile()) {
43-
const fileShasum = await this.$fs.getFileShasum(localPath);
44-
shasums[localPath] = fileShasum;
45-
}
46-
})
47-
);
48-
} else {
49-
shasums = <IStringDictionary>data;
50-
}
51-
52-
this.$fs.writeJson(this.hashFileLocalPath, shasums);
37+
public async uploadHashFileToDevice(data: IStringDictionary): Promise<void> {
38+
this.$fs.writeJson(this.hashFileLocalPath, data);
5339
await this.adb.executeCommand(["push", this.hashFileLocalPath, this.hashFileDevicePath]);
5440
}
5541

5642
public async updateHashes(localToDevicePaths: Mobile.ILocalToDevicePathData[]): Promise<boolean> {
5743
const oldShasums = await this.getShasumsFromDevice();
5844
if (oldShasums) {
59-
await Promise.all(
60-
_.map(localToDevicePaths, async ldp => {
61-
const localPath = ldp.getLocalPath();
62-
if (this.$fs.getFsStats(localPath).isFile()) {
63-
// TODO: Use relative to project path for key
64-
// This will speed up livesync on the same device for the same project on different PCs.
65-
oldShasums[localPath] = await this.$fs.getFileShasum(localPath);
66-
}
67-
})
68-
);
45+
await this.generateHashesFromLocalToDevicePaths(localToDevicePaths, oldShasums);
6946

7047
await this.uploadHashFileToDevice(oldShasums);
7148

@@ -75,6 +52,24 @@ export class AndroidDeviceHashService implements Mobile.IAndroidDeviceHashServic
7552
return false;
7653
}
7754

55+
public async generateHashesFromLocalToDevicePaths(localToDevicePaths: Mobile.ILocalToDevicePathData[], shasums: IStringDictionary): Promise<string[]> {
56+
const devicePaths: string[] = [];
57+
const action = async (localToDevicePathData: Mobile.ILocalToDevicePathData) => {
58+
const localPath = localToDevicePathData.getLocalPath();
59+
if (this.$fs.getFsStats(localPath).isFile()) {
60+
// TODO: Use relative to project path for key
61+
// This will speed up livesync on the same device for the same project on different PCs.
62+
shasums[localPath] = await this.$fs.getFileShasum(localPath);
63+
}
64+
65+
devicePaths.push(`"${localToDevicePathData.getDevicePath()}"`);
66+
};
67+
68+
await executeActionByChunks<Mobile.ILocalToDevicePathData>(localToDevicePaths, DEFAULT_CHUNK_SIZE, action);
69+
70+
return devicePaths;
71+
}
72+
7873
public async removeHashes(localToDevicePaths: Mobile.ILocalToDevicePathData[]): Promise<boolean> {
7974
const oldShasums = await this.getShasumsFromDevice();
8075
if (oldShasums) {

‎test/unit-tests/helpers.ts

+55
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,61 @@ describe("helpers", () => {
1515
assert.deepEqual(actualResult, testData.expectedResult, `For input ${testData.input}, the expected result is: ${testData.expectedResult}, but actual result is: ${actualResult}.`);
1616
};
1717

18+
describe("executeActionByChunks", () => {
19+
const chunkSize = 2;
20+
21+
const assertElements = (initialDataValues: any[], handledElements: any[], element: any, passedChunkSize: number) => {
22+
return new Promise(resolve => setImmediate(() => {
23+
const remainingElements = _.difference(initialDataValues, handledElements);
24+
const isFromLastChunk = (element + passedChunkSize) > initialDataValues.length;
25+
// If the element is one of the last chunk, the remainingElements must be empty.
26+
// If the element is from any other chunk, the remainingElements must contain all elements outside of this chunk.
27+
if (isFromLastChunk) {
28+
assert.isTrue(!remainingElements.length);
29+
} else {
30+
const indexOfElement = initialDataValues.indexOf(element);
31+
const chunkNumber = Math.floor(indexOfElement / passedChunkSize) + 1;
32+
const expectedRemainingElements = _.drop(initialDataValues, chunkNumber * passedChunkSize);
33+
34+
assert.deepEqual(remainingElements, expectedRemainingElements);
35+
}
36+
37+
resolve();
38+
}));
39+
};
40+
41+
it("works correctly with array", () => {
42+
const initialData = _.range(7);
43+
const handledElements: number[] = [];
44+
45+
return helpers.executeActionByChunks(initialData, chunkSize, (element: number, index: number) => {
46+
handledElements.push(element);
47+
assert.deepEqual(element, initialData[index]);
48+
assert.isTrue(initialData.indexOf(element) !== -1);
49+
return assertElements(initialData, handledElements, element, chunkSize);
50+
});
51+
});
52+
53+
it("works correctly with IDictionary", () => {
54+
const initialData: IDictionary<any> = {
55+
a: 1,
56+
b: 2,
57+
c: 3,
58+
d: 4,
59+
e: 5,
60+
f: 6
61+
};
62+
63+
const initialDataValues = _.values(initialData);
64+
const handledElements: number[] = [];
65+
return helpers.executeActionByChunks(initialData, chunkSize, (element, key) => {
66+
handledElements.push(element);
67+
assert.isTrue(initialData[key] === element);
68+
return assertElements(initialDataValues, handledElements, element, chunkSize);
69+
});
70+
});
71+
});
72+
1873
describe("getPropertyName", () => {
1974
const ES5Functions: ITestData[] = [
2075
{

0 commit comments

Comments
 (0)