Skip to content

Commit f51f98d

Browse files
per1234per1234
authored
Merge pull request #72 from per1234/archive-package
Make dedicated package for release archive handling
2 parents 8473338 + 7434cd7 commit f51f98d

File tree

6 files changed

+231
-91
lines changed

6 files changed

+231
-91
lines changed

‎internal/command/sync/sync.go

Lines changed: 9 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ import (
3737
"github.com/arduino/libraries-repository-engine/internal/configuration"
3838
"github.com/arduino/libraries-repository-engine/internal/feedback"
3939
"github.com/arduino/libraries-repository-engine/internal/libraries"
40+
"github.com/arduino/libraries-repository-engine/internal/libraries/archive"
4041
"github.com/arduino/libraries-repository-engine/internal/libraries/db"
4142
"github.com/arduino/libraries-repository-engine/internal/libraries/gitutils"
42-
"github.com/arduino/libraries-repository-engine/internal/libraries/hash"
4343
"github.com/go-git/go-git/v5/plumbing"
4444
"github.com/spf13/cobra"
4545
)
@@ -268,23 +268,19 @@ func syncLibraryTaggedRelease(logger *log.Logger, repo *libraries.Repository, ta
268268
releaseLog += formattedReport
269269
}
270270

271-
zipName := libraries.ZipFolderName(library)
272-
lib := filepath.Base(filepath.Clean(filepath.Join(repo.FolderPath, "..")))
273-
host := filepath.Base(filepath.Clean(filepath.Join(repo.FolderPath, "..", "..")))
274-
zipFilePath, err := libraries.ZipRepo(repo.FolderPath, filepath.Join(config.LibrariesFolder, host, lib), zipName)
271+
archiveData, err := archive.New(repo, library, config)
275272
if err != nil {
273+
return fmt.Errorf("Error while configuring library release archive: %s", err)
274+
}
275+
if err := archiveData.Create(); err != nil {
276276
return fmt.Errorf("Error while zipping library: %s", err)
277277
}
278278

279-
size, checksum, err := getSizeAndCalculateChecksum(zipFilePath)
280-
if err != nil {
281-
return fmt.Errorf("Error while calculating checksums: %s", err)
282-
}
283279
release := db.FromLibraryToRelease(library)
284-
release.URL = config.BaseDownloadURL + host + "/" + lib + "/" + zipName + ".zip"
285-
release.ArchiveFileName = zipName + ".zip"
286-
release.Size = size
287-
release.Checksum = checksum
280+
release.URL = archiveData.URL
281+
release.ArchiveFileName = archiveData.FileName
282+
release.Size = archiveData.Size
283+
release.Checksum = archiveData.Checksum
288284
release.Log = releaseLog
289285

290286
if err := libraries.UpdateLibrary(release, repo.URL, libraryDb); err != nil {
@@ -294,22 +290,6 @@ func syncLibraryTaggedRelease(logger *log.Logger, repo *libraries.Repository, ta
294290
return nil
295291
}
296292

297-
func getSizeAndCalculateChecksum(filePath string) (int64, string, error) {
298-
info, err := os.Stat(filePath)
299-
if err != nil {
300-
return -1, "", err
301-
}
302-
303-
size := info.Size()
304-
305-
checksum, err := hash.Checksum(filePath)
306-
if err != nil {
307-
return -1, "", err
308-
}
309-
310-
return size, checksum, nil
311-
}
312-
313293
func outputLogFile(logger *log.Logger, repoMetadata *libraries.Repo, buffer *bytes.Buffer) error {
314294
if config.LogsFolder == "" {
315295
return nil

‎internal/libraries/archive/archive.go

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
// This file is part of libraries-repository-engine.
2+
//
3+
// Copyright 2021 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This program is free software: you can redistribute it and/or modify
6+
// it under the terms of the GNU Affero General Public License as published
7+
// by the Free Software Foundation, either version 3 of the License, or
8+
// (at your option) any later version.
9+
//
10+
// This program is distributed in the hope that it will be useful,
11+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
// GNU Affero General Public License for more details.
14+
//
15+
// You should have received a copy of the GNU Affero General Public License
16+
// along with this program. If not, see <https://www.gnu.org/licenses/>.
17+
//
18+
// You can be released from the requirements of the above licenses by purchasing
19+
// a commercial license. Buying such a license is mandatory if you want to
20+
// modify or otherwise use the software for commercial activities involving the
21+
// Arduino software without disclosing the source code of your own applications.
22+
// To purchase a commercial license, send an email to [email protected].
23+
24+
// Package archive handles the library release archive.
25+
package archive
26+
27+
import (
28+
"net/url"
29+
"os"
30+
"path/filepath"
31+
"regexp"
32+
33+
"github.com/arduino/libraries-repository-engine/internal/configuration"
34+
"github.com/arduino/libraries-repository-engine/internal/libraries"
35+
"github.com/arduino/libraries-repository-engine/internal/libraries/hash"
36+
"github.com/arduino/libraries-repository-engine/internal/libraries/metadata"
37+
"github.com/arduino/libraries-repository-engine/internal/libraries/zip"
38+
)
39+
40+
// Archive is the type for library release archive data.
41+
type Archive struct {
42+
SourcePath string
43+
RootName string // Name of the root folder inside the archive.
44+
FileName string
45+
Path string // Full path of the archive.
46+
URL string // URL the archive will have on the download server.
47+
Size int64
48+
Checksum string
49+
}
50+
51+
// New initializes and returns an Archive object.
52+
func New(repository *libraries.Repository, libraryMetadata *metadata.LibraryMetadata, config *configuration.Config) (*Archive, error) {
53+
repositoryURLData, err := url.Parse(repository.URL)
54+
if err != nil {
55+
return nil, err
56+
}
57+
// e.g., https://github.com/arduino-libraries/Servo.git -> github.com
58+
repositoryHost := repositoryURLData.Host
59+
// e.g., https://github.com/arduino-libraries/Servo.git -> arduino-libraries
60+
repositoryParent := filepath.Base(filepath.Dir(repositoryURLData.Path))
61+
62+
// Unlike the other path components, the filename is based on library name, not repository name URL.
63+
fileName := zipFolderName(libraryMetadata) + ".zip"
64+
65+
return &Archive{
66+
SourcePath: repository.FolderPath,
67+
RootName: zipFolderName(libraryMetadata),
68+
FileName: fileName,
69+
Path: filepath.Join(config.LibrariesFolder, repositoryHost, repositoryParent, fileName),
70+
URL: config.BaseDownloadURL + repositoryHost + "/" + repositoryParent + "/" + fileName,
71+
}, nil
72+
}
73+
74+
// Create makes an archive file according to the data of the Archive object and updates the object with the size and
75+
// checksum for the resulting file.
76+
func (archive *Archive) Create() error {
77+
err := os.MkdirAll(filepath.Dir(archive.Path), os.FileMode(0755))
78+
if err != nil {
79+
return err
80+
}
81+
82+
if err := zip.Directory(archive.SourcePath, archive.RootName, archive.Path); err != nil {
83+
os.Remove(archive.Path)
84+
return err
85+
}
86+
87+
size, checksum, err := getSizeAndCalculateChecksum(archive.Path)
88+
if err != nil {
89+
return err
90+
}
91+
archive.Size = size
92+
archive.Checksum = checksum
93+
94+
return nil
95+
}
96+
97+
var zipFolderNamePattern = regexp.MustCompile("[^a-zA-Z0-9]")
98+
99+
// zipFolderName returns the name to use for the folder.
100+
func zipFolderName(library *metadata.LibraryMetadata) string {
101+
return zipFolderNamePattern.ReplaceAllString(library.Name, "_") + "-" + library.Version
102+
}
103+
104+
// getSizeAndCalculateChecksum returns the size and SHA-256 checksum for the given file.
105+
func getSizeAndCalculateChecksum(filePath string) (int64, string, error) {
106+
info, err := os.Stat(filePath)
107+
if err != nil {
108+
return -1, "", err
109+
}
110+
111+
size := info.Size()
112+
113+
checksum, err := hash.Checksum(filePath)
114+
if err != nil {
115+
return -1, "", err
116+
}
117+
118+
return size, checksum, nil
119+
}

‎internal/libraries/archive/archive_test.go

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
// This file is part of libraries-repository-engine.
2+
//
3+
// Copyright 2021 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This program is free software: you can redistribute it and/or modify
6+
// it under the terms of the GNU Affero General Public License as published
7+
// by the Free Software Foundation, either version 3 of the License, or
8+
// (at your option) any later version.
9+
//
10+
// This program is distributed in the hope that it will be useful,
11+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
// GNU Affero General Public License for more details.
14+
//
15+
// You should have received a copy of the GNU Affero General Public License
16+
// along with this program. If not, see <https://www.gnu.org/licenses/>.
17+
//
18+
// You can be released from the requirements of the above licenses by purchasing
19+
// a commercial license. Buying such a license is mandatory if you want to
20+
// modify or otherwise use the software for commercial activities involving the
21+
// Arduino software without disclosing the source code of your own applications.
22+
// To purchase a commercial license, send an email to [email protected].
23+
24+
package archive
25+
26+
import (
27+
"os"
28+
"path/filepath"
29+
"testing"
30+
31+
"github.com/arduino/libraries-repository-engine/internal/configuration"
32+
"github.com/arduino/libraries-repository-engine/internal/libraries"
33+
"github.com/arduino/libraries-repository-engine/internal/libraries/metadata"
34+
"github.com/stretchr/testify/assert"
35+
"github.com/stretchr/testify/require"
36+
)
37+
38+
var testDataPath string
39+
40+
func init() {
41+
workingDirectory, err := os.Getwd()
42+
if err != nil {
43+
panic(err)
44+
}
45+
testDataPath = filepath.Join(workingDirectory, "testdata")
46+
}
47+
48+
func TestNew(t *testing.T) {
49+
repository := libraries.Repository{
50+
FolderPath: "/qux/repos/some-repo",
51+
URL: "https://github.com/Foo/Bar.git",
52+
}
53+
libraryMetadata := metadata.LibraryMetadata{
54+
Name: "Foo Bar",
55+
Version: "1.2.3",
56+
}
57+
config := configuration.Config{
58+
LibrariesFolder: "/baz/libs/",
59+
BaseDownloadURL: "https://example/com/libraries/",
60+
}
61+
62+
archiveObject, err := New(&repository, &libraryMetadata, &config)
63+
require.NoError(t, err)
64+
assert.Equal(t, "/qux/repos/some-repo", archiveObject.SourcePath)
65+
assert.Equal(t, "Foo_Bar-1.2.3", archiveObject.RootName)
66+
assert.Equal(t, "Foo_Bar-1.2.3.zip", archiveObject.FileName)
67+
assert.Equal(t, filepath.Join("/baz/libs/github.com/Foo/Foo_Bar-1.2.3.zip"), archiveObject.Path)
68+
assert.Equal(t, "https://example/com/libraries/github.com/Foo/Foo_Bar-1.2.3.zip", archiveObject.URL)
69+
}
70+
71+
func TestCreate(t *testing.T) {
72+
archiveDir := filepath.Join(os.TempDir(), "TestCreateArchiveDir")
73+
defer os.RemoveAll(archiveDir)
74+
archivePath := filepath.Join(archiveDir, "TestCreateArchive.zip")
75+
76+
archiveObject := Archive{
77+
Path: archivePath,
78+
SourcePath: filepath.Join(testDataPath, "gitclones", "SomeRepository"),
79+
RootName: "SomeLibrary",
80+
}
81+
82+
err := archiveObject.Create()
83+
require.NoError(t, err, "This test must be run as administrator on Windows to have symlink creation privilege.")
84+
85+
assert.FileExists(t, archivePath)
86+
assert.Greater(t, archiveObject.Size, int64(0))
87+
assert.NotEmpty(t, archiveObject.Checksum)
88+
}

‎internal/libraries/archive/testdata/gitclones/SomeRepository/SomeFile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
This is some arbitrary source content to stand in for a library release. It is not intended to be a Git repository
2+
because that is not convenient to have in the repository and not relevant to testing this package.

‎internal/libraries/git_integration_test.go

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,24 @@
2121
// Arduino software without disclosing the source code of your own applications.
2222
// To purchase a commercial license, send an email to [email protected].
2323

24-
package libraries
24+
package libraries_test
2525

2626
import (
2727
"io/ioutil"
2828
"os"
2929
"path/filepath"
3030
"testing"
3131

32+
"github.com/arduino/libraries-repository-engine/internal/configuration"
33+
"github.com/arduino/libraries-repository-engine/internal/libraries"
34+
"github.com/arduino/libraries-repository-engine/internal/libraries/archive"
3235
"github.com/arduino/libraries-repository-engine/internal/libraries/db"
3336
"github.com/arduino/libraries-repository-engine/internal/libraries/gitutils"
3437
"github.com/stretchr/testify/require"
3538
)
3639

3740
func TestUpdateLibraryJson(t *testing.T) {
38-
repos, err := ListRepos("./testdata/git_test_repo.txt")
41+
repos, err := libraries.ListRepos("./testdata/git_test_repo.txt")
3942

4043
require.NoError(t, err)
4144
require.NotNil(t, repos)
@@ -51,7 +54,7 @@ func TestUpdateLibraryJson(t *testing.T) {
5154
subfolder, err := repo.AsFolder()
5255
require.NoError(t, err)
5356

54-
r, err := CloneOrFetch(repo, filepath.Join("/tmp", subfolder))
57+
r, err := libraries.CloneOrFetch(repo, filepath.Join("/tmp", subfolder))
5558
require.NoError(t, err)
5659
require.NotNil(t, r)
5760

@@ -64,19 +67,21 @@ func TestUpdateLibraryJson(t *testing.T) {
6467

6568
err = gitutils.CheckoutTag(r.Repository, tag)
6669

67-
library, err := GenerateLibraryFromRepo(r)
70+
library, err := libraries.GenerateLibraryFromRepo(r)
6871
require.NoError(t, err)
6972
require.NotNil(t, library)
7073

71-
zipFolderName := ZipFolderName(library)
74+
config := configuration.Config{LibrariesFolder: librariesRepo}
75+
archiveData, err := archive.New(r, library, &config)
76+
require.NoError(t, err)
7277

7378
release := db.FromLibraryToRelease(library)
7479

75-
zipFilePath, err := ZipRepo(r.FolderPath, librariesRepo, zipFolderName)
80+
err = archiveData.Create()
7681
require.NoError(t, err)
77-
require.NotEmpty(t, zipFilePath)
82+
require.NotEmpty(t, archiveData.Path)
7883

79-
err = UpdateLibrary(release, r.URL, libraryDb)
84+
err = libraries.UpdateLibrary(release, r.URL, libraryDb)
8085
require.NoError(t, err)
8186

8287
}

‎internal/libraries/repoarchive.go

Lines changed: 0 additions & 54 deletions
This file was deleted.

0 commit comments

Comments
 (0)