@@ -32,8 +32,9 @@ import (
3232 "log"
3333 "os"
3434 "path/filepath"
35+ "runtime"
36+ "sync"
3537
36- cc "github.com/arduino/golang-concurrent-workers"
3738 "github.com/arduino/libraries-repository-engine/internal/configuration"
3839 "github.com/arduino/libraries-repository-engine/internal/feedback"
3940 "github.com/arduino/libraries-repository-engine/internal/libraries"
@@ -79,54 +80,40 @@ func syncLibraries(reposFile string) {
7980 os .Exit (1 )
8081 }
8182
82- type jobContext struct {
83- id int
84- repoMetadata * libraries.Repo
85- }
86-
8783 libraryDb := db .Init (config .LibrariesDB )
8884
89- jobQueue := make (chan * jobContext )
90-
91- pool := cc .New (4 )
92- worker := func () {
93- log .Println ("Started worker..." )
94- for job := range jobQueue {
95- buffer := & bytes.Buffer {}
96- logger := log .New (buffer , "" , log .LstdFlags | log .LUTC )
97- syncLibrary (logger , job .repoMetadata , libraryDb )
98-
99- // Output log to file
100- if err := outputLogFile (logger , job .repoMetadata , buffer ); err != nil {
101- logger .Printf ("Error writing log file: %s" , err .Error ())
102- }
103-
104- // Output log to stdout
105- fmt .Println (buffer .String ())
106- }
107- log .Println ("Completed worker!" )
108- }
109- pool .Run (worker )
110- pool .Run (worker )
111- pool .Run (worker )
112- pool .Run (worker )
113- pool .Wait ()
114-
85+ reposChan := make (chan * libraries.Repo )
11586 go func () {
116- id := 0
11787 for _ , repo := range repos {
118- jobQueue <- & jobContext {
119- id : id ,
120- repoMetadata : repo ,
121- }
122- id ++
88+ reposChan <- repo
12389 }
124- close (jobQueue )
90+ close (reposChan )
12591 }()
12692
127- for err := range pool .Errors {
128- feedback .LogError (err )
93+ // Run workers in parallel to consume repositories list
94+ var wg sync.WaitGroup
95+ for workersCount := 0 ; workersCount < runtime .NumCPU (); workersCount ++ {
96+ wg .Add (1 )
97+ go func () {
98+ log .Println ("Started worker..." )
99+ for repo := range reposChan {
100+ buffer := & bytes.Buffer {}
101+ logger := log .New (buffer , "" , log .LstdFlags | log .LUTC )
102+ syncLibrary (logger , repo , libraryDb )
103+
104+ // Output log to file
105+ if err := outputLogFile (logger , repo , buffer ); err != nil {
106+ logger .Printf ("Error writing log file: %s" , err .Error ())
107+ }
108+
109+ // Output log to stdout
110+ fmt .Println (buffer .String ())
111+ }
112+ wg .Done ()
113+ log .Println ("Completed worker!" )
114+ }()
129115 }
116+ wg .Wait ()
130117
131118 libraryIndex , err := libraryDb .OutputLibraryIndex ()
132119 if feedback .LogError (err ) {
0 commit comments