Skip to content

Commit 40e27c6

Browse files
cmagliecmaglie
authored
Merge pull request #20 from cmaglie/add_more_formats
Add more compression formats (.xz and .zst)
2 parents c73c821 + df53e1a commit 40e27c6

File tree

9 files changed

+120
-59
lines changed

9 files changed

+120
-59
lines changed

‎.github/workflows/test.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ jobs:
1616
- uses: actions/checkout@v1
1717
- uses: actions/setup-go@v1
1818
with:
19-
go-version: "1.14"
19+
go-version: "1.17"
2020
- name: Build native
2121
run: go build -v ./...
2222
shell: bash

‎extract.go

+20
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,26 @@ func Archive(ctx context.Context, body io.Reader, location string, rename Rename
4848
return extractor.Archive(ctx, body, location, rename)
4949
}
5050

51+
// Zstd extracts a .zst or .tar.zst archived stream of data in the specified location.
52+
// It accepts a rename function to handle the names of the files (see the example)
53+
func Zstd(ctx context.Context, body io.Reader, location string, rename Renamer) error {
54+
extractor := Extractor{
55+
FS: fs{},
56+
}
57+
58+
return extractor.Zstd(ctx, body, location, rename)
59+
}
60+
61+
// Xz extracts a .xz or .tar.xz archived stream of data in the specified location.
62+
// It accepts a rename function to handle the names of the files (see the example)
63+
func Xz(ctx context.Context, body io.Reader, location string, rename Renamer) error {
64+
extractor := Extractor{
65+
FS: fs{},
66+
}
67+
68+
return extractor.Xz(ctx, body, location, rename)
69+
}
70+
5171
// Bz2 extracts a .bz2 or .tar.bz2 archived stream of data in the specified location.
5272
// It accepts a rename function to handle the names of the files (see the example)
5373
func Bz2(ctx context.Context, body io.Reader, location string, rename Renamer) error {

‎extract_test.go

+3-6
Original file line numberDiff line numberDiff line change
@@ -181,9 +181,8 @@ var ExtractCases = []struct {
181181

182182
func TestArchiveFailure(t *testing.T) {
183183
err := extract.Archive(context.Background(), strings.NewReader("not an archive"), "", nil)
184-
if err == nil || err.Error() != "Not a supported archive" {
185-
t.Error("Expected error 'Not a supported archive', got", err)
186-
}
184+
require.Error(t, err)
185+
require.Contains(t, err.Error(), "Not a supported archive")
187186
}
188187

189188
func TestExtract(t *testing.T) {
@@ -321,9 +320,7 @@ func testWalk(t *testing.T, dir string, testFiles Files) {
321320
files[path] = "link"
322321
} else {
323322
data, err := ioutil.ReadFile(filepath.Join(dir, path))
324-
if err != nil {
325-
326-
}
323+
require.NoError(t, err)
327324
files[path] = strings.TrimSpace(string(data))
328325
}
329326

‎extractor.go

+52-2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ import (
1616
filetype "github.com/h2non/filetype"
1717
"github.com/h2non/filetype/types"
1818
"github.com/juju/errors"
19+
"github.com/klauspost/compress/zstd"
20+
"github.com/ulikunitz/xz"
1921
)
2022

2123
// Extractor is more sophisticated than the base functions. It allows to write over an interface
@@ -46,13 +48,61 @@ func (e *Extractor) Archive(ctx context.Context, body io.Reader, location string
4648
return e.Gz(ctx, body, location, rename)
4749
case "bz2":
4850
return e.Bz2(ctx, body, location, rename)
51+
case "xz":
52+
return e.Xz(ctx, body, location, rename)
53+
case "zst":
54+
return e.Zstd(ctx, body, location, rename)
4955
case "tar":
5056
return e.Tar(ctx, body, location, rename)
5157
default:
52-
return errors.New("Not a supported archive")
58+
return errors.New("Not a supported archive: " + kind.Extension)
5359
}
5460
}
5561

62+
func (e *Extractor) Zstd(ctx context.Context, body io.Reader, location string, rename Renamer) error {
63+
reader, err := zstd.NewReader(body)
64+
if err != nil {
65+
return errors.Annotatef(err, "opening zstd: detect")
66+
}
67+
68+
body, kind, err := match(reader)
69+
if err != nil {
70+
return errors.Annotatef(err, "extract zstd: detect")
71+
}
72+
73+
if kind.Extension == "tar" {
74+
return e.Tar(ctx, body, location, rename)
75+
}
76+
77+
err = e.copy(ctx, location, 0666, body)
78+
if err != nil {
79+
return err
80+
}
81+
return nil
82+
}
83+
84+
func (e *Extractor) Xz(ctx context.Context, body io.Reader, location string, rename Renamer) error {
85+
reader, err := xz.NewReader(body)
86+
if err != nil {
87+
return errors.Annotatef(err, "opening xz: detect")
88+
}
89+
90+
body, kind, err := match(reader)
91+
if err != nil {
92+
return errors.Annotatef(err, "extract xz: detect")
93+
}
94+
95+
if kind.Extension == "tar" {
96+
return e.Tar(ctx, body, location, rename)
97+
}
98+
99+
err = e.copy(ctx, location, 0666, body)
100+
if err != nil {
101+
return err
102+
}
103+
return nil
104+
}
105+
56106
// Bz2 extracts a .bz2 or .tar.bz2 archived stream of data in the specified location.
57107
// It accepts a rename function to handle the names of the files (see the example)
58108
func (e *Extractor) Bz2(ctx context.Context, body io.Reader, location string, rename Renamer) error {
@@ -64,7 +114,7 @@ func (e *Extractor) Bz2(ctx context.Context, body io.Reader, location string, re
64114
}
65115

66116
if kind.Extension == "tar" {
67-
return Tar(ctx, body, location, rename)
117+
return e.Tar(ctx, body, location, rename)
68118
}
69119

70120
err = e.copy(ctx, location, 0666, body)

‎extractor_test.go

+28-44
Original file line numberDiff line numberDiff line change
@@ -4,68 +4,52 @@ import (
44
"bytes"
55
"context"
66
"fmt"
7-
"io/ioutil"
87
"os"
98
"path/filepath"
109
"runtime"
1110
"testing"
1211

12+
"github.com/arduino/go-paths-helper"
1313
"github.com/codeclysm/extract/v3"
1414
"github.com/stretchr/testify/require"
1515
)
1616

17-
func TestExtractor_Tar(t *testing.T) {
18-
tmp, _ := ioutil.TempDir("", "")
19-
20-
extractor := extract.Extractor{
21-
FS: MockDisk{
22-
Base: tmp,
23-
},
17+
func TestExtractors(t *testing.T) {
18+
type archiveTest struct {
19+
name string
20+
file *paths.Path
2421
}
25-
26-
data, err := ioutil.ReadFile("testdata/archive.tar.gz")
27-
if err != nil {
28-
t.Fatal(err)
22+
testCases := []archiveTest{
23+
{"TarGz", paths.New("testdata/archive.tar.gz")},
24+
{"TarBz2", paths.New("testdata/archive.tar.bz2")},
25+
{"TarXz", paths.New("testdata/archive.tar.xz")},
26+
{"TarZstd", paths.New("testdata/archive.tar.zst")},
27+
{"Zip", paths.New("testdata/archive.zip")},
2928
}
30-
buffer := bytes.NewBuffer(data)
31-
32-
err = extractor.Gz(context.Background(), buffer, "/", nil)
33-
if err != nil {
34-
t.Error(err.Error())
35-
}
36-
37-
files := Files{
38-
"": "dir",
39-
"/archive": "dir",
40-
"/archive/folder": "dir",
41-
"/archive/folderlink": "link",
42-
"/archive/folder/file1.txt": "folder/File1",
43-
"/archive/file1.txt": "File1",
44-
"/archive/file2.txt": "File2",
45-
"/archive/link.txt": "File1",
29+
for _, test := range testCases {
30+
t.Run(test.name, func(t *testing.T) {
31+
testArchive(t, test.file)
32+
})
4633
}
47-
testWalk(t, tmp, files)
4834
}
4935

50-
func TestExtractor_Zip(t *testing.T) {
51-
tmp, _ := ioutil.TempDir("", "")
36+
func testArchive(t *testing.T, archivePath *paths.Path) {
37+
tmp, err := paths.MkTempDir("", "")
38+
require.NoError(t, err)
39+
defer tmp.RemoveAll()
5240

53-
extractor := extract.Extractor{
54-
FS: MockDisk{
55-
Base: tmp,
56-
},
57-
}
41+
data, err := archivePath.ReadFile()
42+
require.NoError(t, err)
5843

59-
data, err := ioutil.ReadFile("testdata/archive.zip")
60-
if err != nil {
61-
t.Fatal(err)
62-
}
6344
buffer := bytes.NewBuffer(data)
6445

65-
err = extractor.Zip(context.Background(), buffer, "/", nil)
66-
if err != nil {
67-
t.Error(err.Error())
46+
extractor := extract.Extractor{
47+
FS: MockDisk{
48+
Base: tmp.String(),
49+
},
6850
}
51+
err = extractor.Archive(context.Background(), buffer, "/", nil)
52+
require.NoError(t, err)
6953

7054
files := Files{
7155
"": "dir",
@@ -77,7 +61,7 @@ func TestExtractor_Zip(t *testing.T) {
7761
"/archive/file2.txt": "File2",
7862
"/archive/link.txt": "File1",
7963
}
80-
testWalk(t, tmp, files)
64+
testWalk(t, tmp.String(), files)
8165
}
8266

8367
func TestZipSlipHardening(t *testing.T) {

‎go.mod

+10-4
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,20 @@
11
module github.com/codeclysm/extract/v3
22

3-
go 1.14
3+
go 1.17
44

55
require (
66
github.com/arduino/go-paths-helper v1.2.0
7-
github.com/davecgh/go-spew v1.1.1 // indirect
8-
github.com/h2non/filetype v1.0.6
7+
github.com/h2non/filetype v1.1.3
98
github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5
9+
github.com/klauspost/compress v1.15.13
10+
github.com/stretchr/testify v1.3.0
11+
github.com/ulikunitz/xz v0.5.11
12+
)
13+
14+
require (
15+
github.com/davecgh/go-spew v1.1.1 // indirect
1016
github.com/juju/testing v0.0.0-20200510222523-6c8c298c77a0 // indirect
1117
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
12-
github.com/stretchr/testify v1.3.0
18+
github.com/pmezard/go-difflib v1.0.0 // indirect
1319
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
1420
)

‎go.sum

+6-2
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ github.com/arduino/go-paths-helper v1.2.0/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3
33
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
44
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
55
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6-
github.com/h2non/filetype v1.0.6 h1:g84/+gdkAT1hnYO+tHpCLoikm13Ju55OkN4KCb1uGEQ=
7-
github.com/h2non/filetype v1.0.6/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU=
6+
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
7+
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
88
github.com/juju/clock v0.0.0-20180524022203-d293bb356ca4/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA=
99
github.com/juju/errors v0.0.0-20150916125642-1b5e39b83d18/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
1010
github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 h1:rhqTjzJlm7EbkELJDKMTU7udov+Se0xZkWmugr6zGok=
@@ -16,6 +16,8 @@ github.com/juju/testing v0.0.0-20200510222523-6c8c298c77a0 h1:+WWUkhnTjV6RNOxkcw
1616
github.com/juju/testing v0.0.0-20200510222523-6c8c298c77a0/go.mod h1:hpGvhGHPVbNBraRLZEhoQwFLMrjK8PSlO4D3nDjKYXo=
1717
github.com/juju/utils v0.0.0-20180808125547-9dfc6dbfb02b/go.mod h1:6/KLg8Wz/y2KVGWEpkK9vMNGkOnu4k/cqs8Z1fKjTOk=
1818
github.com/juju/version v0.0.0-20161031051906-1f41e27e54f2/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U=
19+
github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0=
20+
github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
1921
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
2022
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
2123
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -26,6 +28,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
2628
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
2729
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
2830
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
31+
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
32+
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
2933
golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
3034
golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
3135
gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

‎testdata/archive.tar.xz

308 Bytes
Binary file not shown.

‎testdata/archive.tar.zst

274 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)