Skip to content

Commit 1409281

Browse files
zeripathzeripath
committed
Share HTML template renderers and create a watcher framework
The recovery, API, Web and package frameworks all create their own HTML Renderers. This increases the memory requirements of Gitea unnecessarily with duplicate templates being kept in memory. Further the reloading framework in dev mode for these involves locking and recompiling all of the templates on each load. This will potentially hide concurrency issues and it is inefficient. This PR stores the templates renderer in the context and stores this context in the NormalRoutes, it then creates a fsnotify.Watcher framework to watch files. The watching framework is then extended to the mailer templates which were previously not being reloaded in dev. Then the locales are simplified to a similar structure. Fix go-gitea#20210, go-gitea#20211, go-gitea#20217 Replace go-gitea#20159 Signed-off-by: Andrew Thornton <[email protected]>
1 parent 9d9bf66 commit 1409281

36 files changed

+837
-496
lines changed

‎cmd/embedded.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ func initEmbeddedExtractor(c *cli.Context) error {
123123

124124
sections["public"] = &section{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset}
125125
sections["options"] = &section{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset}
126-
sections["templates"] = &section{Path: "templates", Names: templates.AssetNames, IsDir: templates.AssetIsDir, Asset: templates.Asset}
126+
sections["templates"] = &section{Path: "templates", Names: templates.BuiltinAssetNames, IsDir: templates.BuiltinAssetIsDir, Asset: templates.BuiltinAsset}
127127

128128
for _, sec := range sections {
129129
assets = append(assets, buildAssetList(sec, pats, c)...)

‎cmd/web.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,10 @@ func runWeb(ctx *cli.Context) error {
126126
return err
127127
}
128128
}
129-
c := install.Routes()
129+
installCtx, cancel := context.WithCancel(graceful.GetManager().HammerContext())
130+
c := install.Routes(installCtx)
130131
err := listen(c, false)
132+
cancel()
131133
if err != nil {
132134
log.Critical("Unable to open listener for installer. Is Gitea already running?")
133135
graceful.GetManager().DoGracefulShutdown()
@@ -174,7 +176,7 @@ func runWeb(ctx *cli.Context) error {
174176
}
175177

176178
// Set up Chi routes
177-
c := routers.NormalRoutes()
179+
c := routers.NormalRoutes(graceful.GetManager().HammerContext())
178180
err := listen(c, true)
179181
<-graceful.GetManager().Done()
180182
log.Info("PID: %d Gitea Web Finished", os.Getpid())

‎contrib/pr/checkout.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"code.gitea.io/gitea/models/db"
2828
"code.gitea.io/gitea/models/unittest"
2929
gitea_git "code.gitea.io/gitea/modules/git"
30+
"code.gitea.io/gitea/modules/graceful"
3031
"code.gitea.io/gitea/modules/markup"
3132
"code.gitea.io/gitea/modules/markup/external"
3233
repo_module "code.gitea.io/gitea/modules/repository"
@@ -118,7 +119,7 @@ func runPR() {
118119
// routers.GlobalInit()
119120
external.RegisterRenderers()
120121
markup.Init()
121-
c := routers.NormalRoutes()
122+
c := routers.NormalRoutes(graceful.GetManager().HammerContext())
122123

123124
log.Printf("[PR] Ready for testing !\n")
124125
log.Printf("[PR] Login with user1, user2, user3, ... with pass: password\n")

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ require (
2727
github.com/emirpasic/gods v1.18.1
2828
github.com/ethantkoenig/rupture v1.0.1
2929
github.com/felixge/fgprof v0.9.2
30+
github.com/fsnotify/fsnotify v1.5.4
3031
github.com/gliderlabs/ssh v0.3.4
3132
github.com/go-ap/activitypub v0.0.0-20220615144428-48208c70483b
3233
github.com/go-ap/jsonld v0.0.0-20220615144122-1d862b15410d
@@ -160,7 +161,6 @@ require (
160161
github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect
161162
github.com/felixge/httpsnoop v1.0.2 // indirect
162163
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
163-
github.com/fsnotify/fsnotify v1.5.4 // indirect
164164
github.com/fullstorydev/grpcurl v1.8.1 // indirect
165165
github.com/fxamacker/cbor/v2 v2.4.0 // indirect
166166
github.com/go-ap/errors v0.0.0-20220615144307-e8bc4a40ae9f // indirect

‎integrations/api_activitypub_person_test.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@ import (
2323

2424
func TestActivityPubPerson(t *testing.T) {
2525
setting.Federation.Enabled = true
26-
c = routers.NormalRoutes()
26+
c = routers.NormalRoutes(context.TODO())
2727
defer func() {
2828
setting.Federation.Enabled = false
29-
c = routers.NormalRoutes()
29+
c = routers.NormalRoutes(context.TODO())
3030
}()
3131

3232
onGiteaRun(t, func(*testing.T, *url.URL) {
@@ -60,10 +60,10 @@ func TestActivityPubPerson(t *testing.T) {
6060

6161
func TestActivityPubMissingPerson(t *testing.T) {
6262
setting.Federation.Enabled = true
63-
c = routers.NormalRoutes()
63+
c = routers.NormalRoutes(context.Background())
6464
defer func() {
6565
setting.Federation.Enabled = false
66-
c = routers.NormalRoutes()
66+
c = routers.NormalRoutes(context.Background())
6767
}()
6868

6969
onGiteaRun(t, func(*testing.T, *url.URL) {
@@ -75,10 +75,10 @@ func TestActivityPubMissingPerson(t *testing.T) {
7575

7676
func TestActivityPubPersonInbox(t *testing.T) {
7777
setting.Federation.Enabled = true
78-
c = routers.NormalRoutes()
78+
c = routers.NormalRoutes(context.Background())
7979
defer func() {
8080
setting.Federation.Enabled = false
81-
c = routers.NormalRoutes()
81+
c = routers.NormalRoutes(context.Background())
8282
}()
8383

8484
srv := httptest.NewServer(c)

‎integrations/api_nodeinfo_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
package integrations
66

77
import (
8+
"context"
89
"net/http"
910
"net/url"
1011
"testing"
@@ -18,10 +19,10 @@ import (
1819

1920
func TestNodeinfo(t *testing.T) {
2021
setting.Federation.Enabled = true
21-
c = routers.NormalRoutes()
22+
c = routers.NormalRoutes(context.TODO())
2223
defer func() {
2324
setting.Federation.Enabled = false
24-
c = routers.NormalRoutes()
25+
c = routers.NormalRoutes(context.TODO())
2526
}()
2627

2728
onGiteaRun(t, func(*testing.T, *url.URL) {

‎integrations/create_no_session_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
package integrations
66

77
import (
8+
"context"
89
"net/http"
910
"net/http/httptest"
1011
"os"
@@ -57,7 +58,7 @@ func TestSessionFileCreation(t *testing.T) {
5758
oldSessionConfig := setting.SessionConfig.ProviderConfig
5859
defer func() {
5960
setting.SessionConfig.ProviderConfig = oldSessionConfig
60-
c = routers.NormalRoutes()
61+
c = routers.NormalRoutes(context.TODO())
6162
}()
6263

6364
var config session.Options
@@ -82,7 +83,7 @@ func TestSessionFileCreation(t *testing.T) {
8283

8384
setting.SessionConfig.ProviderConfig = string(newConfigBytes)
8485

85-
c = routers.NormalRoutes()
86+
c = routers.NormalRoutes(context.TODO())
8687

8788
t.Run("NoSessionOnViewIssue", func(t *testing.T) {
8889
defer PrintCurrentTest(t)()

‎integrations/integration_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ func TestMain(m *testing.M) {
8989
defer cancel()
9090

9191
initIntegrationTest()
92-
c = routers.NormalRoutes()
92+
c = routers.NormalRoutes(context.TODO())
9393

9494
// integration test settings...
9595
if setting.Cfg != nil {

‎modules/context/context.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -673,8 +673,8 @@ func Auth(authMethod auth.Method) func(*Context) {
673673
}
674674

675675
// Contexter initializes a classic context for a request.
676-
func Contexter() func(next http.Handler) http.Handler {
677-
rnd := templates.HTMLRenderer()
676+
func Contexter(ctx context.Context) func(next http.Handler) http.Handler {
677+
_, rnd := templates.HTMLRenderer(ctx)
678678
csrfOpts := getCsrfOpts()
679679
if !setting.IsProd {
680680
CsrfTokenRegenerationInterval = 5 * time.Second // in dev, re-generate the tokens more aggressively for debug purpose

‎modules/context/package.go

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
package context
66

77
import (
8+
gocontext "context"
89
"fmt"
910
"net/http"
1011

@@ -13,6 +14,7 @@ import (
1314
"code.gitea.io/gitea/models/perm"
1415
user_model "code.gitea.io/gitea/models/user"
1516
"code.gitea.io/gitea/modules/structs"
17+
"code.gitea.io/gitea/modules/templates"
1618
)
1719

1820
// Package contains owner, access mode and optional the package descriptor
@@ -101,12 +103,14 @@ func packageAssignment(ctx *Context, errCb func(int, string, interface{})) {
101103
}
102104

103105
// PackageContexter initializes a package context for a request.
104-
func PackageContexter() func(next http.Handler) http.Handler {
106+
func PackageContexter(ctx gocontext.Context) func(next http.Handler) http.Handler {
107+
_, rnd := templates.HTMLRenderer(ctx)
105108
return func(next http.Handler) http.Handler {
106109
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
107110
ctx := Context{
108-
Resp: NewResponse(resp),
109-
Data: map[string]interface{}{},
111+
Resp: NewResponse(resp),
112+
Data: map[string]interface{}{},
113+
Render: rnd,
110114
}
111115
defer ctx.Close()
112116

‎modules/options/base.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package options
6+
7+
import (
8+
"fmt"
9+
"io/fs"
10+
"os"
11+
"path/filepath"
12+
)
13+
14+
func walkAssetDir(root string, callback func(path string, name string, d fs.DirEntry, err error) error) error {
15+
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
16+
name := path[len(root):]
17+
if len(name) > 0 && name[0] == '/' {
18+
name = name[1:]
19+
}
20+
if err != nil {
21+
if os.IsNotExist(err) {
22+
return callback(path, name, d, err)
23+
}
24+
return err
25+
}
26+
if d.Name() == ".DS_Store" && d.IsDir() { // Because Macs...
27+
return fs.SkipDir
28+
}
29+
return callback(path, name, d, err)
30+
}); err != nil && !os.IsNotExist(err) {
31+
return fmt.Errorf("unable to get files for assets in %s: %w", root, err)
32+
}
33+
return nil
34+
}

‎modules/options/dynamic.go

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ package options
88

99
import (
1010
"fmt"
11+
"io/fs"
1112
"os"
1213
"path"
14+
"path/filepath"
1315

1416
"code.gitea.io/gitea/modules/log"
1517
"code.gitea.io/gitea/modules/setting"
@@ -45,7 +47,7 @@ func Dir(name string) ([]string, error) {
4547

4648
isDir, err = util.IsDir(staticDir)
4749
if err != nil {
48-
return []string{}, fmt.Errorf("Unabe to check if static directory %s is a directory. %v", staticDir, err)
50+
return []string{}, fmt.Errorf("unable to check if static directory %s is a directory. %v", staticDir, err)
4951
}
5052
if isDir {
5153
files, err := util.StatDir(staticDir, true)
@@ -64,6 +66,18 @@ func Locale(name string) ([]byte, error) {
6466
return fileFromDir(path.Join("locale", name))
6567
}
6668

69+
// WalkLocales reads the content of a specific locale from static or custom path.
70+
func WalkLocales(callback func(path string, name string, d fs.DirEntry, err error) error) error {
71+
if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
72+
return fmt.Errorf("failed to walk locales. Error: %w", err)
73+
}
74+
75+
if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
76+
return fmt.Errorf("failed to walk locales. Error: %w", err)
77+
}
78+
return nil
79+
}
80+
6781
// Readme reads the content of a specific readme from static or custom path.
6882
func Readme(name string) ([]byte, error) {
6983
return fileFromDir(path.Join("readme", name))

‎modules/options/static.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,10 @@ package options
99
import (
1010
"fmt"
1111
"io"
12+
"io/fs"
1213
"os"
1314
"path"
15+
"path/filepath"
1416

1517
"code.gitea.io/gitea/modules/log"
1618
"code.gitea.io/gitea/modules/setting"
@@ -74,6 +76,14 @@ func Locale(name string) ([]byte, error) {
7476
return fileFromDir(path.Join("locale", name))
7577
}
7678

79+
// WalkLocales reads the content of a specific locale from static or custom path.
80+
func WalkLocales(callback func(path string, name string, d fs.DirEntry, err error) error) error {
81+
if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
82+
return fmt.Errorf("failed to walk locales. Error: %w", err)
83+
}
84+
return nil
85+
}
86+
7787
// Readme reads the content of a specific readme from bindata or custom path.
7888
func Readme(name string) ([]byte, error) {
7989
return fileFromDir(path.Join("readme", name))

0 commit comments

Comments
 (0)