Skip to content

Commit f328ecd

Browse files
cmagliecmaglie
authored
[skip-changelog] legacy: Arduino preprocess subroutine refactorization (part 1) (#2186)
* legacy: Removed ReadFileAndStoreInContext command * ctags parser.Parse signature change * removing ctags-related data from builder ctx (part 1) * removing ctags-related data from builder ctx (part 2) * removing ctags-related data from builder ctx (part 3) * Clearly separate Source code processing phases * Removed Command wrapper from ContainerMergeCopySketchFiles * Moved builder.CopySketchFilesToBuildPath into proper location * Converted FilterSketchSource into a function * Moved a couple of functions in the proper builder package * Fixed lint error
1 parent 493fa83 commit f328ecd

27 files changed

+507
-477
lines changed

Diff for: ‎arduino/builder/cpp.go

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to [email protected].
15+
16+
package builder
17+
18+
import (
19+
"strings"
20+
"unicode/utf8"
21+
)
22+
23+
// QuoteCppString returns the given string as a quoted string for use with the C
24+
// preprocessor. This adds double quotes around it and escapes any
25+
// double quotes and backslashes in the string.
26+
func QuoteCppString(str string) string {
27+
str = strings.Replace(str, "\\", "\\\\", -1)
28+
str = strings.Replace(str, "\"", "\\\"", -1)
29+
return "\"" + str + "\""
30+
}
31+
32+
// ParseCppString parse a C-preprocessor string as emitted by the preprocessor. This
33+
// is a string contained in double quotes, with any backslashes or
34+
// quotes escaped with a backslash. If a valid string was present at the
35+
// start of the given line, returns the unquoted string contents, the
36+
// remainder of the line (everything after the closing "), and true.
37+
// Otherwise, returns the empty string, the entire line and false.
38+
func ParseCppString(line string) (string, string, bool) {
39+
// For details about how these strings are output by gcc, see:
40+
// https://github.com/gcc-mirror/gcc/blob/a588355ab948cf551bc9d2b89f18e5ae5140f52c/libcpp/macro.c#L491-L511
41+
// Note that the documentation suggests all non-printable
42+
// characters are also escaped, but the implementation does not
43+
// actually do this. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51259
44+
if len(line) < 1 || line[0] != '"' {
45+
return "", line, false
46+
}
47+
48+
i := 1
49+
res := ""
50+
for {
51+
if i >= len(line) {
52+
return "", line, false
53+
}
54+
55+
c, width := utf8.DecodeRuneInString(line[i:])
56+
57+
switch c {
58+
case '\\':
59+
// Backslash, next character is used unmodified
60+
i += width
61+
if i >= len(line) {
62+
return "", line, false
63+
}
64+
res += string(line[i])
65+
case '"':
66+
// Quote, end of string
67+
return res, line[i+width:], true
68+
default:
69+
res += string(c)
70+
}
71+
72+
i += width
73+
}
74+
}

Diff for: ‎arduino/builder/cpp_test.go

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package builder_test
2+
3+
import (
4+
"testing"
5+
6+
"github.com/arduino/arduino-cli/arduino/builder"
7+
"github.com/stretchr/testify/require"
8+
)
9+
10+
func TestParseCppString(t *testing.T) {
11+
_, _, ok := builder.ParseCppString(`foo`)
12+
require.Equal(t, false, ok)
13+
14+
_, _, ok = builder.ParseCppString(`"foo`)
15+
require.Equal(t, false, ok)
16+
17+
str, rest, ok := builder.ParseCppString(`"foo"`)
18+
require.Equal(t, true, ok)
19+
require.Equal(t, `foo`, str)
20+
require.Equal(t, ``, rest)
21+
22+
str, rest, ok = builder.ParseCppString(`"foo\\bar"`)
23+
require.Equal(t, true, ok)
24+
require.Equal(t, `foo\bar`, str)
25+
require.Equal(t, ``, rest)
26+
27+
str, rest, ok = builder.ParseCppString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`)
28+
require.Equal(t, true, ok)
29+
require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str)
30+
require.Equal(t, ` and "then" some`, rest)
31+
32+
str, rest, ok = builder.ParseCppString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`)
33+
require.Equal(t, true, ok)
34+
require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str)
35+
require.Equal(t, ``, rest)
36+
37+
str, rest, ok = builder.ParseCppString(`"/home/ççç/"`)
38+
require.Equal(t, true, ok)
39+
require.Equal(t, `/home/ççç/`, str)
40+
require.Equal(t, ``, rest)
41+
42+
str, rest, ok = builder.ParseCppString(`"/home/ççç/ /$sdsdd\\"`)
43+
require.Equal(t, true, ok)
44+
require.Equal(t, `/home/ççç/ /$sdsdd\`, str)
45+
require.Equal(t, ``, rest)
46+
}

Diff for: ‎arduino/builder/sketch.go

+19-12
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ import (
1919
"bytes"
2020
"fmt"
2121
"regexp"
22-
"strings"
2322

2423
"github.com/arduino/arduino-cli/arduino/sketch"
2524
"github.com/arduino/arduino-cli/i18n"
@@ -34,13 +33,20 @@ var (
3433
tr = i18n.Tr
3534
)
3635

37-
// QuoteCppString returns the given string as a quoted string for use with the C
38-
// preprocessor. This adds double quotes around it and escapes any
39-
// double quotes and backslashes in the string.
40-
func QuoteCppString(str string) string {
41-
str = strings.Replace(str, "\\", "\\\\", -1)
42-
str = strings.Replace(str, "\"", "\\\"", -1)
43-
return "\"" + str + "\""
36+
// PrepareSketchBuildPath copies the sketch source files in the build path.
37+
// The .ino files are merged together to create a .cpp file (by the way, the
38+
// .cpp file still needs to be Arduino-preprocessed to compile).
39+
func PrepareSketchBuildPath(sketch *sketch.Sketch, sourceOverrides map[string]string, buildPath *paths.Path) (offset int, mergedSource string, err error) {
40+
if offset, mergedSource, err = sketchMergeSources(sketch, sourceOverrides); err != nil {
41+
return
42+
}
43+
if err = SketchSaveItemCpp(sketch.MainFile, []byte(mergedSource), buildPath); err != nil {
44+
return
45+
}
46+
if err = sketchCopyAdditionalFiles(sketch, buildPath, sourceOverrides); err != nil {
47+
return
48+
}
49+
return
4450
}
4551

4652
// SketchSaveItemCpp saves a preprocessed .cpp sketch file on disk
@@ -59,8 +65,9 @@ func SketchSaveItemCpp(path *paths.Path, contents []byte, destPath *paths.Path)
5965
return nil
6066
}
6167

62-
// SketchMergeSources merges all the source files included in a sketch
63-
func SketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) {
68+
// sketchMergeSources merges all the .ino source files included in a sketch to produce
69+
// a single .cpp file.
70+
func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) {
6471
lineOffset := 0
6572
mergedSource := ""
6673

@@ -105,9 +112,9 @@ func SketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st
105112
return lineOffset, mergedSource, nil
106113
}
107114

108-
// SketchCopyAdditionalFiles copies the additional files for a sketch to the
115+
// sketchCopyAdditionalFiles copies the additional files for a sketch to the
109116
// specified destination directory.
110-
func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, overrides map[string]string) error {
117+
func sketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, overrides map[string]string) error {
111118
if err := destPath.MkdirAll(); err != nil {
112119
return errors.Wrap(err, tr("unable to create a folder to save the sketch files"))
113120
}

Diff for: ‎arduino/builder/sketch_test.go

+6-7
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
// Arduino software without disclosing the source code of your own applications.
1414
// To purchase a commercial license, send an email to [email protected].
1515

16-
package builder_test
16+
package builder
1717

1818
import (
1919
"fmt"
@@ -23,7 +23,6 @@ import (
2323
"strings"
2424
"testing"
2525

26-
"github.com/arduino/arduino-cli/arduino/builder"
2726
"github.com/arduino/arduino-cli/arduino/sketch"
2827
"github.com/arduino/go-paths-helper"
2928
"github.com/stretchr/testify/require"
@@ -48,7 +47,7 @@ func TestSaveSketch(t *testing.T) {
4847
t.Fatalf("unable to read golden file %s: %v", sketchFile, err)
4948
}
5049

51-
builder.SketchSaveItemCpp(paths.New(sketchName), source, tmp)
50+
SketchSaveItemCpp(paths.New(sketchName), source, tmp)
5251

5352
out, err := tmp.Join(outName).ReadFile()
5453
if err != nil {
@@ -82,7 +81,7 @@ func TestMergeSketchSources(t *testing.T) {
8281
}
8382
mergedSources := strings.ReplaceAll(string(mergedBytes), "%s", pathToGoldenSource)
8483

85-
offset, source, err := builder.SketchMergeSources(s, nil)
84+
offset, source, err := sketchMergeSources(s, nil)
8685
require.Nil(t, err)
8786
require.Equal(t, 2, offset)
8887
require.Equal(t, mergedSources, source)
@@ -94,7 +93,7 @@ func TestMergeSketchSourcesArduinoIncluded(t *testing.T) {
9493
require.NotNil(t, s)
9594

9695
// ensure not to include Arduino.h when it's already there
97-
_, source, err := builder.SketchMergeSources(s, nil)
96+
_, source, err := sketchMergeSources(s, nil)
9897
require.Nil(t, err)
9998
require.Equal(t, 1, strings.Count(source, "<Arduino.h>"))
10099
}
@@ -110,7 +109,7 @@ func TestCopyAdditionalFiles(t *testing.T) {
110109

111110
// copy the sketch over, create a fake main file we don't care about it
112111
// but we need it for `SketchLoad` to succeed later
113-
err = builder.SketchCopyAdditionalFiles(s1, tmp, nil)
112+
err = sketchCopyAdditionalFiles(s1, tmp, nil)
114113
require.Nil(t, err)
115114
fakeIno := tmp.Join(fmt.Sprintf("%s.ino", tmp.Base()))
116115
require.Nil(t, fakeIno.WriteFile([]byte{}))
@@ -125,7 +124,7 @@ func TestCopyAdditionalFiles(t *testing.T) {
125124
require.Nil(t, err)
126125

127126
// copy again
128-
err = builder.SketchCopyAdditionalFiles(s1, tmp, nil)
127+
err = sketchCopyAdditionalFiles(s1, tmp, nil)
129128
require.Nil(t, err)
130129

131130
// verify file hasn't changed

Diff for: ‎legacy/builder/builder.go

+12-3
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"reflect"
2020
"time"
2121

22+
"github.com/arduino/arduino-cli/arduino/builder"
2223
"github.com/arduino/arduino-cli/i18n"
2324
"github.com/arduino/arduino-cli/legacy/builder/phases"
2425
"github.com/arduino/arduino-cli/legacy/builder/types"
@@ -39,14 +40,18 @@ func (s *Builder) Run(ctx *types.Context) error {
3940
return err
4041
}
4142

43+
var _err error
4244
commands := []types.Command{
4345
&ContainerSetupHardwareToolsLibsSketchAndProps{},
4446

4547
&ContainerBuildOptions{},
4648

4749
&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"},
4850

49-
&ContainerMergeCopySketchFiles{},
51+
types.BareCommand(func(ctx *types.Context) error {
52+
ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
53+
return _err
54+
}),
5055

5156
utils.LogIfVerbose(false, tr("Detecting libraries used...")),
5257
&ContainerFindIncludes{},
@@ -127,14 +132,18 @@ func (s *Preprocess) Run(ctx *types.Context) error {
127132
return err
128133
}
129134

135+
var _err error
130136
commands := []types.Command{
131137
&ContainerSetupHardwareToolsLibsSketchAndProps{},
132138

133139
&ContainerBuildOptions{},
134140

135141
&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"},
136142

137-
&ContainerMergeCopySketchFiles{},
143+
types.BareCommand(func(ctx *types.Context) error {
144+
ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
145+
return _err
146+
}),
138147

139148
&ContainerFindIncludes{},
140149

@@ -148,7 +157,7 @@ func (s *Preprocess) Run(ctx *types.Context) error {
148157
}
149158

150159
// Output arduino-preprocessed source
151-
ctx.WriteStdout([]byte(ctx.Source))
160+
ctx.WriteStdout([]byte(ctx.SketchSourceAfterArduinoPreprocessing))
152161
return nil
153162
}
154163

0 commit comments

Comments
 (0)