Skip to content

Commit 3d4f82c

Browse files
giacomocusinatogiacomocusinato
committed
refactor: generate-protocol now fetch proto files from arduino_cli_{version}_proto.zip
- Use the CLI release proto.zip to get proto files for production versions of CLI - Extract the proto files from repo if CLI version is declared as `commitsh` or version is 1.1.0 See arduino/arduino-cli#2761
1 parent d106588 commit 3d4f82c

File tree

1 file changed

+152
-104
lines changed

1 file changed

+152
-104
lines changed

Diff for: ‎arduino-ide-extension/scripts/generate-protocol.js

+152-104
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33
(async () => {
44
const os = require('node:os');
55
const path = require('node:path');
6-
const { mkdirSync, promises: fs, rmSync } = require('node:fs');
6+
const decompress = require('decompress');
7+
const unzip = require('decompress-unzip');
8+
const { mkdirSync, promises: fs, rmSync, existsSync } = require('node:fs');
79
const { exec } = require('./utils');
810
const { glob } = require('glob');
9-
const { SemVer, gte, valid: validSemVer, gt } = require('semver');
11+
const { SemVer, gte, valid: validSemVer, eq } = require('semver');
1012
// Use a node-protoc fork until apple arm32 is supported
1113
// https://github.com/YePpHa/node-protoc/pull/10
1214
const protoc = path.dirname(require('@pingghost/protoc/protoc'));
@@ -90,152 +92,198 @@
9092
}
9193
*/
9294
const versionObject = JSON.parse(versionJson);
93-
const version = versionObject.VersionString;
9495

95-
// Clone the repository and check out the tagged version
96-
// Return folder with proto files
97-
async function getProtoPath(forceCliVersion) {
98-
const repository = await fs.mkdtemp(path.join(os.tmpdir(), 'arduino-cli-'));
96+
async function globProtos(folder, pattern = '**/*.proto') {
97+
let protos = [];
98+
try {
99+
const matches = await glob(pattern, { cwd: folder });
100+
protos = matches.map((filename) => path.join(folder, filename));
101+
} catch (error) {
102+
console.log(error.stack ?? error.message);
103+
}
104+
return protos;
105+
}
106+
107+
async function getProtosFromRepo(
108+
commitish = '',
109+
version = '',
110+
owner = 'arduino',
111+
repo = 'arduino-cli'
112+
) {
113+
const repoFolder = await fs.mkdtemp(path.join(os.tmpdir(), 'arduino-cli-'));
99114

100115
const url = `https://github.com/${owner}/${repo}.git`;
101116
console.log(`>>> Cloning repository from '${url}'...`);
102-
exec('git', ['clone', url, repository], { logStdout: true });
117+
exec('git', ['clone', url, repoFolder], { logStdout: true });
103118
console.log(`<<< Repository cloned.`);
104119

105-
let cliVersion = forceCliVersion || version;
106-
if (validSemVer(cliVersion)) {
120+
if (validSemVer(version)) {
121+
let versionTag = version;
107122
// https://github.com/arduino/arduino-cli/pull/2374
108123
if (
109124
gte(new SemVer(version, { loose: true }), new SemVer('0.35.0-rc.1'))
110125
) {
111-
cliVersion = `v${cliVersion}`;
126+
versionTag = `v${version}`;
112127
}
113-
console.log(`>>> Checking out tagged version: '${cliVersion}'...`);
114-
exec('git', ['-C', repository, 'fetch', '--all', '--tags'], {
128+
console.log(`>>> Checking out tagged version: '${versionTag}'...`);
129+
exec('git', ['-C', repoFolder, 'fetch', '--all', '--tags'], {
115130
logStdout: true,
116131
});
117132
exec(
118133
'git',
119-
['-C', repository, 'checkout', `tags/${cliVersion}`, '-b', cliVersion],
134+
['-C', repoFolder, 'checkout', `tags/${versionTag}`, '-b', versionTag],
120135
{ logStdout: true }
121136
);
122-
console.log(`<<< Checked out tagged version: '${cliVersion}'.`);
123-
} else if (forceCliVersion) {
124-
console.log(`WARN: invalid semver: '${forceCliVersion}'.`);
125-
// If the forced version is invalid, do not proceed with fallbacks.
126-
return undefined;
137+
console.log(`<<< Checked out tagged version: '${versionTag}'.`);
127138
} else if (commitish) {
128-
console.log(
129-
`>>> Checking out commitish from 'package.json': '${commitish}'...`
130-
);
131-
exec('git', ['-C', repository, 'checkout', commitish], {
139+
console.log(`>>> Checking out commitish: '${commitish}'...`);
140+
exec('git', ['-C', repoFolder, 'checkout', commitish], {
132141
logStdout: true,
133142
});
134-
console.log(
135-
`<<< Checked out commitish from 'package.json': '${commitish}'.`
136-
);
137-
} else if (versionObject.Commit) {
138-
console.log(
139-
`>>> Checking out commitish from the CLI: '${versionObject.Commit}'...`
140-
);
141-
exec('git', ['-C', repository, 'checkout', versionObject.Commit], {
142-
logStdout: true,
143-
});
144-
console.log(
145-
`<<< Checked out commitish from the CLI: '${versionObject.Commit}'.`
146-
);
143+
console.log(`<<< Checked out commitish: '${commitish}'.`);
147144
} else {
148145
console.log(
149146
`WARN: no 'git checkout'. Generating from the HEAD revision.`
150147
);
151148
}
152149

153-
return path.join(repository, 'rpc');
150+
const rpcFolder = await fs.mkdtemp(
151+
path.join(os.tmpdir(), 'arduino-cli-rpc')
152+
);
153+
154+
// Copy the the repository rpc folder so we can remove the repository
155+
await fs.cp(path.join(repoFolder, 'rpc'), path.join(rpcFolder), {
156+
recursive: true,
157+
});
158+
rmSync(repoFolder, { recursive: true, maxRetries: 5, force: true });
159+
160+
// Patch for https://github.com/arduino/arduino-cli/issues/2755
161+
// Google proto files are removed from source since v1.1.0
162+
if (!existsSync(path.join(rpcFolder, 'google'))) {
163+
// Include packaged google proto files from v1.1.1
164+
// See https://github.com/arduino/arduino-cli/pull/2761
165+
console.log(`>>> Missing google proto files. Including from v1.1.1...`);
166+
const v111ProtoFolder = await getProtosFromZip('1.1.1');
167+
168+
// Create an return a folder name google in rpcFolder
169+
const googleFolder = path.join(rpcFolder, 'google');
170+
await fs.cp(path.join(v111ProtoFolder, 'google'), googleFolder, {
171+
recursive: true,
172+
});
173+
console.log(`<<< Included google proto files from v1.1.1.`);
174+
}
175+
176+
return rpcFolder;
154177
}
155178

156-
const protoPath = await getProtoPath();
179+
async function getProtosFromZip(version) {
180+
if (!version) {
181+
console.log(`Could not download proto files: CLI version not provided.`);
182+
process.exit(1);
183+
}
184+
console.log(`>>> Downloading proto files from zip for ${version}.`);
185+
186+
const url = `https://downloads.arduino.cc/arduino-cli/arduino-cli_${version}_proto.zip`;
187+
const protos = await fs.mkdtemp(
188+
path.join(os.tmpdir(), 'arduino-cli-proto')
189+
);
157190

158-
if (!protoPath) {
159-
console.log(`Could not find the proto files folder.`);
191+
const { default: download } = await import('@xhmikosr/downloader');
192+
/** @type {import('node:buffer').Buffer} */
193+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
194+
// @ts-ignore
195+
const data = await download(url);
196+
197+
await decompress(data, protos, {
198+
plugins: [unzip()],
199+
filter: (file) => file.path.endsWith('.proto'),
200+
});
201+
202+
console.log(
203+
`<<< Finished downloading and extracting proto files for ${version}.`
204+
);
205+
206+
return protos;
207+
}
208+
209+
let protosFolder;
210+
211+
if (commitish) {
212+
protosFolder = await getProtosFromRepo(commitish, undefined, owner, repo);
213+
} else if (
214+
versionObject.VersionString &&
215+
validSemVer(versionObject.VersionString)
216+
) {
217+
const version = versionObject.VersionString;
218+
// v1.1.0 does not contains google proto files in zip
219+
// See https://github.com/arduino/arduino-cli/issues/2755
220+
const isV110 = eq(new SemVer(version, { loose: true }), '1.1.0');
221+
protosFolder = isV110
222+
? await getProtosFromRepo(undefined, version)
223+
: await getProtosFromZip(version);
224+
} else if (versionObject.Commit) {
225+
protosFolder = await getProtosFromRepo(versionObject.Commit);
226+
}
227+
228+
if (!protosFolder) {
229+
console.log(`Could not get proto files: missing commitish or version.`);
230+
process.exit(1);
231+
}
232+
233+
const protos = await globProtos(protosFolder);
234+
235+
if (!protos || protos.length === 0) {
236+
// rmSync(protosFolder, { recursive: true, maxRetries: 5, force: true });
237+
console.log(`Could not find any .proto files under ${protosFolder}.`);
160238
process.exit(1);
161239
}
162240

163241
console.log('>>> Generating TS/JS API from:');
164-
exec('git', ['-C', protoPath, 'rev-parse', '--abbrev-ref', 'HEAD'], {
165-
logStdout: true,
166-
});
167242

168243
const out = path.join(__dirname, '..', 'src', 'node', 'cli-protocol');
169244
// Must wipe the gen output folder. Otherwise, dangling service implementation remain in IDE2 code,
170245
// although it has been removed from the proto file.
171246
// For example, https://github.com/arduino/arduino-cli/commit/50a8bf5c3e61d5b661ccfcd6a055e82eeb510859.
172-
rmSync(out, { recursive: true, maxRetries: 5, force: true });
247+
// rmSync(out, { recursive: true, maxRetries: 5, force: true });
173248
mkdirSync(out, { recursive: true });
174249

175-
if (gt(new SemVer(version, { loose: true }), new SemVer('1.0.4'))) {
176-
// Patch for https://github.com/arduino/arduino-cli/issues/2755
177-
// Credit https://github.com/dankeboy36/ardunno-cli-gen/pull/9/commits/64a5ac89aae605249261c8ceff7255655ecfafca
178-
// Download the 1.0.4 version and use the missing google/rpc/status.proto file.
179-
console.log('<<< Generating missing google proto files');
180-
const v104ProtoPath = await getProtoPath('1.0.4');
181-
if (!v104ProtoPath) {
182-
console.log(`Could not find the proto files folder for version 1.0.4.`);
183-
process.exit(1);
184-
}
185-
await fs.cp(
186-
path.join(v104ProtoPath, 'google'),
187-
path.join(protoPath, 'google'),
188-
{
189-
recursive: true,
190-
}
250+
try {
251+
// Generate JS code from the `.proto` files.
252+
exec(
253+
'grpc_tools_node_protoc',
254+
[
255+
`--js_out=import_style=commonjs,binary:${out}`,
256+
`--grpc_out=generate_package_definition:${out}`,
257+
'-I',
258+
protosFolder,
259+
...protos,
260+
],
261+
{ logStdout: true }
191262
);
192-
console.log(`>>> Generated missing google file`);
193-
}
194263

195-
let protos = [];
196-
try {
197-
const matches = await glob('**/*.proto', { cwd: protoPath });
198-
protos = matches.map((filename) => path.join(protoPath, filename));
264+
// Generate the `.d.ts` files for JS.
265+
exec(
266+
path.join(protoc, `protoc${platform === 'win32' ? '.exe' : ''}`),
267+
[
268+
`--plugin=protoc-gen-ts=${path.resolve(
269+
__dirname,
270+
'..',
271+
'node_modules',
272+
'.bin',
273+
`protoc-gen-ts${platform === 'win32' ? '.cmd' : ''}`
274+
)}`,
275+
`--ts_out=generate_package_definition:${out}`,
276+
'-I',
277+
protosFolder,
278+
...protos,
279+
],
280+
{ logStdout: true }
281+
);
199282
} catch (error) {
200-
console.log(error.stack ?? error.message);
201-
}
202-
203-
if (!protos || protos.length === 0) {
204-
console.log(`Could not find any .proto files under ${protoPath}.`);
205-
process.exit(1);
283+
console.log(error);
284+
} finally {
285+
rmSync(protosFolder, { recursive: true, maxRetries: 5, force: true });
206286
}
207287

208-
// Generate JS code from the `.proto` files.
209-
exec(
210-
'grpc_tools_node_protoc',
211-
[
212-
`--js_out=import_style=commonjs,binary:${out}`,
213-
`--grpc_out=generate_package_definition:${out}`,
214-
'-I',
215-
protoPath,
216-
...protos,
217-
],
218-
{ logStdout: true }
219-
);
220-
221-
// Generate the `.d.ts` files for JS.
222-
exec(
223-
path.join(protoc, `protoc${platform === 'win32' ? '.exe' : ''}`),
224-
[
225-
`--plugin=protoc-gen-ts=${path.resolve(
226-
__dirname,
227-
'..',
228-
'node_modules',
229-
'.bin',
230-
`protoc-gen-ts${platform === 'win32' ? '.cmd' : ''}`
231-
)}`,
232-
`--ts_out=generate_package_definition:${out}`,
233-
'-I',
234-
protoPath,
235-
...protos,
236-
],
237-
{ logStdout: true }
238-
);
239-
240288
console.log('<<< Generation was successful.');
241289
})();

0 commit comments

Comments
 (0)