Skip to content

Commit f8d3c65

Browse files
silvanocerzasilvanocerza
authored
[skip changelog] Update i18n scripts to use Transifex REST API v3 (#1641)
* Add organization env var in i18n transifex script * Update i18n transifex push command to use API v3 * Update i18n transifex pull command to use API v3 * Remove duplicated code * Add success message after successfull resource upload * Reworked i18n scripts to unmarshal data to structs * Fixed wrong defer
1 parent 5d18ae0 commit f8d3c65

File tree

3 files changed

+349
-82
lines changed

3 files changed

+349
-82
lines changed

‎i18n/cmd/commands/transifex/pull_transifex.go

+198-47
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,17 @@
1616
package transifex
1717

1818
import (
19+
"bytes"
1920
"encoding/json"
2021
"fmt"
22+
"io"
2123
"io/ioutil"
2224
"net/http"
2325
"os"
24-
"path"
26+
"sync"
27+
"time"
2528

29+
"github.com/arduino/go-paths-helper"
2630
"github.com/spf13/cobra"
2731
)
2832

@@ -33,99 +37,246 @@ var pullTransifexCommand = &cobra.Command{
3337
}
3438

3539
func getLanguages() []string {
36-
req, err := http.NewRequest(
37-
"GET",
38-
fmt.Sprintf(
39-
"https://www.transifex.com/api/2/project/%s/resource/%s/stats/",
40-
project, resource,
41-
), nil)
40+
url := mainEndpoint + fmt.Sprintf("projects/o:%s:p:%s/languages", organization, project)
4241

42+
req, err := http.NewRequest("GET", url, nil)
4343
if err != nil {
4444
fmt.Println(err.Error())
4545
os.Exit(1)
4646
}
4747

48-
req.SetBasicAuth("api", apiKey)
49-
50-
resp, err := http.DefaultClient.Do(req)
48+
addHeaders(req)
5149

50+
res, err := http.DefaultClient.Do(req)
5251
if err != nil {
5352
fmt.Println(err.Error())
5453
os.Exit(1)
5554
}
5655

57-
defer resp.Body.Close()
58-
59-
b, err := ioutil.ReadAll(resp.Body)
56+
defer res.Body.Close()
57+
b, err := ioutil.ReadAll(res.Body)
6058
if err != nil {
6159
fmt.Println(err.Error())
6260
os.Exit(1)
6361
}
6462

65-
var jsonResp map[string]interface{}
66-
if err := json.Unmarshal(b, &jsonResp); err != nil {
63+
var jsonRes struct {
64+
Data []struct {
65+
Attributes struct {
66+
Code string `json:"code"`
67+
} `json:"attributes"`
68+
} `json:"data"`
69+
}
70+
if err := json.Unmarshal(b, &jsonRes); err != nil {
6771
fmt.Println(err.Error())
6872
os.Exit(1)
6973
}
7074

71-
var langs []string
72-
for key := range jsonResp {
73-
langs = append(langs, key)
75+
var languages []string
76+
for _, object := range jsonRes.Data {
77+
languages = append(languages, object.Attributes.Code)
7478
}
75-
76-
return langs
79+
return languages
7780
}
7881

79-
func pullCatalog(cmd *cobra.Command, args []string) {
80-
languages := getLanguages()
81-
fmt.Println("translations found:", languages)
82+
// startTranslationDownload notifies Transifex that we want to start downloading
83+
// the resources file for the specified languageCode.
84+
// Returns an id to monitor the download status.
85+
func startTranslationDownload(languageCode string) string {
86+
url := mainEndpoint + "resource_translations_async_downloads"
8287

83-
folder := args[0]
88+
type jsonReq struct {
89+
Data struct {
90+
Relationships struct {
91+
Language struct {
92+
Data struct {
93+
ID string `json:"id"`
94+
Type string `json:"type"`
95+
} `json:"data"`
96+
} `json:"language"`
97+
Resource struct {
98+
Data struct {
99+
ID string `json:"id"`
100+
Type string `json:"type"`
101+
} `json:"data"`
102+
} `json:"resource"`
103+
} `json:"relationships"`
104+
Type string `json:"type"`
105+
} `json:"data"`
106+
}
84107

85-
for _, lang := range languages {
108+
jsonData := jsonReq{}
109+
jsonData.Data.Type = "resource_translations_async_downloads"
110+
jsonData.Data.Relationships.Language.Data.ID = fmt.Sprintf("l:%s", languageCode)
111+
jsonData.Data.Relationships.Language.Data.Type = "languages"
112+
jsonData.Data.Relationships.Resource.Data.ID = fmt.Sprintf("o:%s:p:%s:r:%s", organization, project, resource)
113+
jsonData.Data.Relationships.Resource.Data.Type = "resources"
114+
115+
jsonBytes, err := json.Marshal(jsonData)
116+
if err != nil {
117+
fmt.Println(err)
118+
os.Exit(1)
119+
}
120+
121+
req, err := http.NewRequest(
122+
"POST",
123+
url,
124+
bytes.NewBuffer(jsonBytes),
125+
)
126+
if err != nil {
127+
fmt.Println(err)
128+
os.Exit(1)
129+
}
130+
131+
addHeaders(req)
132+
133+
res, err := http.DefaultClient.Do(req)
134+
if err != nil {
135+
fmt.Println(err)
136+
os.Exit(1)
137+
}
138+
139+
defer res.Body.Close()
140+
body, err := io.ReadAll(res.Body)
141+
if err != nil {
142+
fmt.Println(err)
143+
os.Exit(1)
144+
}
145+
146+
var jsonRes struct {
147+
Data struct {
148+
ID string `json:"id"`
149+
} `json:"data"`
150+
}
151+
if err = json.Unmarshal(body, &jsonRes); err != nil {
152+
fmt.Println(err)
153+
os.Exit(1)
154+
}
155+
return jsonRes.Data.ID
156+
}
157+
158+
// getDownloadURL checks for the download status of the languageCode file specified
159+
// by downloadID.
160+
// It return a URL to download the file when ready.
161+
func getDownloadURL(languageCode, downloadID string) string {
162+
url := mainEndpoint + "resource_translations_async_downloads/" + downloadID
163+
// The download request status must be asked from time to time, if it's
164+
// still pending we try again using exponentional backoff starting from 2.5 seconds.
165+
backoff := 2500 * time.Millisecond
166+
for {
86167

87168
req, err := http.NewRequest(
88169
"GET",
89-
fmt.Sprintf(
90-
"https://www.transifex.com/api/2/project/%s/resource/%s/translation/%s/?mode=reviewed&file=po",
91-
project, resource, lang,
92-
), nil)
93-
170+
url,
171+
nil,
172+
)
94173
if err != nil {
95-
fmt.Println(err.Error())
174+
fmt.Println(err)
96175
os.Exit(1)
97176
}
98177

99-
req.SetBasicAuth("api", apiKey)
100-
101-
resp, err := http.DefaultClient.Do(req)
178+
addHeaders(req)
102179

180+
client := http.Client{
181+
CheckRedirect: func(req *http.Request, via []*http.Request) error {
182+
// We handle redirection manually
183+
return http.ErrUseLastResponse
184+
},
185+
}
186+
res, err := client.Do(req)
103187
if err != nil {
104-
fmt.Println(err.Error())
188+
fmt.Println(err)
105189
os.Exit(1)
106190
}
107191

108-
defer resp.Body.Close()
109-
110-
b, err := ioutil.ReadAll(resp.Body)
192+
if res.StatusCode == 303 {
193+
// Return the URL to download translation file
194+
return res.Header.Get("location")
195+
}
111196

197+
body, err := io.ReadAll(res.Body)
112198
if err != nil {
113-
fmt.Println(err.Error())
199+
fmt.Println(err)
114200
os.Exit(1)
115201
}
202+
res.Body.Close()
116203

117-
os.Remove(path.Join(folder, lang+".po"))
118-
file, err := os.OpenFile(path.Join(folder, lang+".po"), os.O_CREATE|os.O_RDWR, 0644)
119-
120-
if err != nil {
121-
fmt.Println(err.Error())
204+
var jsonRes struct {
205+
Data struct {
206+
Attributes struct {
207+
Status string `json:"status"`
208+
Errors []struct {
209+
Code string `json:"code"`
210+
Detail string `json:"detail"`
211+
} `json:"errors"`
212+
} `json:"attributes"`
213+
} `json:"data"`
214+
}
215+
if err = json.Unmarshal(body, &jsonRes); err != nil {
216+
fmt.Println(err)
122217
os.Exit(1)
123218
}
124219

125-
_, err = file.Write(b)
126-
if err != nil {
127-
fmt.Println(err.Error())
220+
status := jsonRes.Data.Attributes.Status
221+
switch status {
222+
case "succeeded":
223+
return ""
224+
case "pending":
225+
fallthrough
226+
case "processing":
227+
fmt.Printf("Current status for language %s: %s\n", languageCode, status)
228+
time.Sleep(backoff)
229+
backoff = backoff * 2
230+
// Request the status again
231+
continue
232+
case "failed":
233+
for _, err := range jsonRes.Data.Attributes.Errors {
234+
fmt.Printf("%s: %s\n", err.Code, err.Detail)
235+
}
128236
os.Exit(1)
129237
}
238+
fmt.Printf("Status request for language %s failed in an unforeseen way\n", languageCode)
239+
os.Exit(1)
240+
}
241+
}
242+
243+
// download file from url and saves it in folder with the specified fileName
244+
func download(folder, fileName, url string) {
245+
fmt.Printf("Starting download of %s\n", fileName)
246+
filePath := paths.New(folder, fileName)
247+
248+
res, err := http.DefaultClient.Get(url)
249+
if err != nil {
250+
fmt.Println(err)
251+
os.Exit(1)
252+
}
253+
254+
data, err := io.ReadAll(res.Body)
255+
if err != nil {
256+
fmt.Println(err)
257+
os.Exit(1)
258+
}
259+
260+
filePath.WriteFile(data)
261+
fmt.Printf("Finished download of %s\n", fileName)
262+
}
263+
264+
func pullCatalog(cmd *cobra.Command, args []string) {
265+
languages := getLanguages()
266+
fmt.Println("translations found:", languages)
267+
268+
folder := args[0]
269+
270+
var wg sync.WaitGroup
271+
for _, lang := range languages {
272+
wg.Add(1)
273+
go func(lang string) {
274+
downloadID := startTranslationDownload(lang)
275+
url := getDownloadURL(lang, downloadID)
276+
download(folder, lang+".po", url)
277+
wg.Done()
278+
}(lang)
130279
}
280+
wg.Wait()
281+
fmt.Println("Translation files downloaded")
131282
}

0 commit comments

Comments
 (0)