Merge branch 'master' into open-repository-open-cat-file-batch

Author: zeripath

Dockerfile

@@ -53,7 +53,7 @@ RUN addgroup \
     -u 1000 \
     -G git \
     git && \
-  echo "git:$(dd if=/dev/urandom bs=24 count=1 status=none | base64)" | chpasswd
+  echo "git:*" | chpasswd -e
 
 ENV USER git
 ENV GITEA_CUSTOM /data/gitea

Dockerfile.rootless

@@ -46,8 +46,7 @@ RUN addgroup \
     -s /bin/bash \
     -u 1000 \
     -G git \
-    git && \
-  echo "git:$(dd if=/dev/urandom bs=24 count=1 status=none | base64)" | chpasswd
+    git
 
 RUN mkdir -p /var/lib/gitea /etc/gitea
 RUN chown git:git /var/lib/gitea /etc/gitea

custom/conf/app.example.ini

@@ -1145,6 +1145,14 @@ RUN_AT_START = false
 NO_SUCCESS_NOTICE = false
 SCHEDULE = @every 72h
 
+; Delete all old actions from database
+[cron.delete_old_actions]
+ENABLED = false
+RUN_AT_START = false
+NO_SUCCESS_NOTICE = false
+SCHEDULE = @every 168h
+OLDER_THAN = 8760h
+
 [git]
 ; The path of git executable. If empty, Gitea searches through the PATH environment.
 PATH =

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -59,7 +59,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `MIRROR_QUEUE_LENGTH`: **1000**: Patch test queue length, increase if pull request patch
    testing starts hanging.
 - `PREFERRED_LICENSES`: **Apache License 2.0,MIT License**: Preferred Licenses to place at
-   the top of the list. Name must match file name in conf/license or custom/conf/license.
+   the top of the list. Name must match file name in options/license or custom/options/license.
 - `DISABLE_HTTP_GIT`: **false**: Disable the ability to interact with repositories over the
    HTTP protocol.
 - `USE_COMPAT_SSH_URI`: **false**: Force ssh:// clone url instead of scp-style uri when
@@ -786,6 +786,13 @@ NB: You must have `DISABLE_ROUTER_LOG` set to `false` for this option to take ef
 - `NO_SUCCESS_NOTICE`: **false**: Set to true to switch off success notices.
 - `SCHEDULE`: **@every 72h**: Cron syntax for scheduling repository archive cleanup, e.g. `@every 1h`.
 
+#### Cron -  Delete all old actions from database ('cron.delete_old_actions')
+- `ENABLED`: **false**: Enable service.
+- `RUN_AT_START`: **false**: Run tasks at start up time (if ENABLED).
+- `NO_SUCCESS_NOTICE`: **false**: Set to true to switch off success notices.
+- `SCHEDULE`: **@every 128h**: Cron syntax for scheduling a work, e.g. `@every 128h`.
+- `OLDER_THAN`: **@every 8760h**: any action older than this expression will be deleted from database, suggest using `8760h` (1 year) because that's the max length of heatmap.
+
 ## Git (`git`)
 
 - `PATH`: **""**: The path of git executable. If empty, Gitea searches through the PATH environment.

docs/content/doc/advanced/external-renderers.en-us.md

@@ -70,7 +70,7 @@ IS_INPUT_FILE = true
 [markup.restructuredtext]
 ENABLED = true
 FILE_EXTENSIONS = .rst
-RENDER_COMMAND = rst2html.py
+RENDER_COMMAND = "timeout 30s pandoc +RTS -M512M -RTS -f rst"
 IS_INPUT_FILE = false
 ```
 

integrations/mssql.ini.tmpl

@@ -10,9 +10,10 @@ PASSWD   = {{TEST_MSSQL_PASSWORD}}
 SSL_MODE = disable
 
 [indexer]
-ISSUE_INDEXER_PATH = integrations/indexers-mssql/issues.bleve
+ISSUE_INDEXER_PATH = integrations/gitea-integration-mssql/indexers/issues.bleve
+ISSUE_INDEXER_QUEUE_DIR = integrations/gitea-integration-mssql/indexers/issues.queue
 REPO_INDEXER_ENABLED = true
-REPO_INDEXER_PATH = integrations/indexers-mssql/repos.bleve
+REPO_INDEXER_PATH = integrations/gitea-integration-mssql/indexers/repos.bleve
 
 [queue.code_indexer]
 TYPE = immediate
@@ -24,8 +25,10 @@ TYPE = immediate
 ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-mssql/gitea-repositories
 
 [repository.local]
-LOCAL_COPY_PATH = tmp/local-repo-mssql
-LOCAL_WIKI_PATH = tmp/local-wiki-mssql
+LOCAL_COPY_PATH = integrations/gitea-integration-mssql/tmp/local-repo
+
+[repository.upload]
+TEMP_PATH = integrations/gitea-integration-mssql/tmp/uploads
 
 [repository.signing]
 SIGNING_KEY = none
@@ -39,14 +42,14 @@ SSH_LISTEN_HOST  = localhost
 SSH_PORT         = 2201
 START_SSH_SERVER = true
 LFS_START_SERVER = true
-LFS_CONTENT_PATH = integrations/gitea-integration-mssql/data/lfs-mssql
+LFS_CONTENT_PATH = integrations/gitea-integration-mssql/data/lfs
 OFFLINE_MODE     = false
 LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
 APP_DATA_PATH    = integrations/gitea-integration-mssql/data
 BUILTIN_SSH_SERVER_USER = git
 
 [attachment]
-PATH = integrations/gitea-integration-mssql/data
+PATH = integrations/gitea-integration-mssql/data/attachments
 
 [mailer]
 ENABLED = true

integrations/mysql.ini.tmpl

@@ -12,9 +12,9 @@ SSL_MODE = disable
 [indexer]
 ISSUE_INDEXER_TYPE = elasticsearch
 ISSUE_INDEXER_CONN_STR = http://elastic:changeme@elasticsearch:9200
-ISSUE_INDEXER_PATH = integrations/indexers-mysql/issues.bleve
+ISSUE_INDEXER_QUEUE_DIR = integrations/gitea-integration-mysql/indexers/issues.queue
 REPO_INDEXER_ENABLED = true
-REPO_INDEXER_PATH = integrations/indexers-mysql/repos.bleve
+REPO_INDEXER_PATH = integrations/gitea-integration-mysql/indexers/repos.bleve
 
 [queue.code_indexer]
 TYPE = immediate
@@ -26,8 +26,10 @@ TYPE = immediate
 ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-mysql/gitea-repositories
 
 [repository.local]
-LOCAL_COPY_PATH = tmp/local-repo-mysql
-LOCAL_WIKI_PATH = tmp/local-wiki-mysql
+LOCAL_COPY_PATH = integrations/gitea-integration-mysql/tmp/local-repo
+
+[repository.upload]
+TEMP_PATH = integrations/gitea-integration-mysql/tmp/uploads
 
 [repository.signing]
 SIGNING_KEY = none

integrations/mysql8.ini.tmpl

@@ -10,9 +10,10 @@ PASSWD   = {{TEST_MYSQL8_PASSWORD}}
 SSL_MODE = disable
 
 [indexer]
-ISSUE_INDEXER_PATH = integrations/indexers-mysql8/issues.bleve
+ISSUE_INDEXER_PATH = integrations/gitea-integration-mysql8/indexers/issues.bleve
+ISSUE_INDEXER_QUEUE_DIR = integrations/gitea-integration-mysql8/indexers/issues.queue
 REPO_INDEXER_ENABLED = true
-REPO_INDEXER_PATH = integrations/indexers-mysql8/repos.bleve
+REPO_INDEXER_PATH = integrations/gitea-integration-mysql8/indexers/repos.bleve
 
 [queue.code_indexer]
 TYPE = immediate
@@ -24,8 +25,10 @@ TYPE = immediate
 ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-mysql8/gitea-repositories
 
 [repository.local]
-LOCAL_COPY_PATH = tmp/local-repo-mysql8
-LOCAL_WIKI_PATH = tmp/local-wiki-mysql8
+LOCAL_COPY_PATH = integrations/gitea-integration-mysql8/tmp/local-repo
+
+[repository.upload]
+TEMP_PATH = integrations/gitea-integration-mysql8/tmp/uploads
 
 [repository.signing]
 SIGNING_KEY = none
@@ -39,12 +42,15 @@ SSH_LISTEN_HOST  = localhost
 SSH_PORT         = 2204
 START_SSH_SERVER = true
 LFS_START_SERVER = true
-LFS_CONTENT_PATH = data/lfs-mysql8
+LFS_CONTENT_PATH = integrations/gitea-integration-mysql8/data/lfs
 OFFLINE_MODE     = false
 LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
 APP_DATA_PATH    = integrations/gitea-integration-mysql8/data
 BUILTIN_SSH_SERVER_USER = git
 
+[attachment]
+PATH = integrations/gitea-integration-mysql8/data/attachments
+
 [mailer]
 ENABLED = false
 
@@ -60,12 +66,14 @@ DEFAULT_ALLOW_CREATE_ORGANIZATION = true
 NO_REPLY_ADDRESS                  = noreply.example.org
 
 [picture]
-DISABLE_GRAVATAR        = false
-ENABLE_FEDERATED_AVATAR = false
+DISABLE_GRAVATAR              = false
+ENABLE_FEDERATED_AVATAR       = false
+AVATAR_UPLOAD_PATH            = integrations/gitea-integration-mysql8/data/avatars
+REPOSITORY_AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql8/data/repo-avatars
 
 [session]
 PROVIDER = file
-PROVIDER_CONFIG = data/sessions-mysql8
+PROVIDER_CONFIG = integrations/gitea-integration-mysql8/data/sessions
 
 [log]
 MODE                 = test,file
@@ -80,10 +88,8 @@ COLORIZE             = true
 [log.file]
 LEVEL                = Debug
 
-
 [security]
 DISABLE_GIT_HOOKS = false
 INSTALL_LOCK   = true
 SECRET_KEY     = 9pCviYTWSb
 INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ
-

integrations/pgsql.ini.tmpl

@@ -11,9 +11,10 @@ SCHEMA   = {{TEST_PGSQL_SCHEMA}}
 SSL_MODE = disable
 
 [indexer]
-ISSUE_INDEXER_PATH = integrations/indexers-pgsql/issues.bleve
+ISSUE_INDEXER_PATH = integrations/gitea-integration-pgsql/indexers/issues.bleve
+ISSUE_INDEXER_QUEUE_DIR = integrations/gitea-integration-pgsql/indexers/issues.queue
 REPO_INDEXER_ENABLED = true
-REPO_INDEXER_PATH = integrations/indexers-pgsql/repos.bleve
+REPO_INDEXER_PATH = integrations/gitea-integration-pgsql/indexers/repos.bleve
 
 [queue.code_indexer]
 TYPE = immediate
@@ -25,8 +26,10 @@ TYPE = immediate
 ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-pgsql/gitea-repositories
 
 [repository.local]
-LOCAL_COPY_PATH = tmp/local-repo-pgsql
-LOCAL_WIKI_PATH = tmp/local-wiki-pgsql
+LOCAL_COPY_PATH = integrations/gitea-integration-pgsql/tmp/local-repo
+
+[repository.upload]
+TEMP_PATH = integrations/gitea-integration-pgsql/tmp/uploads
 
 [repository.signing]
 SIGNING_KEY = none
@@ -40,14 +43,14 @@ SSH_LISTEN_HOST  = localhost
 SSH_PORT         = 2202
 START_SSH_SERVER = true
 LFS_START_SERVER = true
-LFS_CONTENT_PATH = integrations/gitea-integration-pgsql/data/lfs-pgsql
+LFS_CONTENT_PATH = integrations/gitea-integration-pgsql/data/lfs
 OFFLINE_MODE     = false
 LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
 APP_DATA_PATH    = integrations/gitea-integration-pgsql/data
 BUILTIN_SSH_SERVER_USER = git
 
 [attachment]
-PATH = integrations/gitea-integration-pgsql/data
+PATH = integrations/gitea-integration-pgsql/data/attachments
 
 [mailer]
 ENABLED = true

integrations/sqlite.ini.tmpl

@@ -6,9 +6,10 @@ DB_TYPE = sqlite3
 PATH    = integrations/gitea-integration-sqlite/gitea.db
 
 [indexer]
-ISSUE_INDEXER_PATH   = integrations/indexers-sqlite/issues.bleve
+ISSUE_INDEXER_PATH   = integrations/gitea-integration-sqlite/indexers/issues.bleve
+ISSUE_INDEXER_QUEUE_DIR = integrations/gitea-integration-sqlite/indexers/issues.queue
 REPO_INDEXER_ENABLED = true
-REPO_INDEXER_PATH    = integrations/indexers-sqlite/repos.bleve
+REPO_INDEXER_PATH    = integrations/gitea-integration-sqlite/indexers/repos.bleve
 
 [queue.code_indexer]
 TYPE = immediate
@@ -20,8 +21,10 @@ TYPE = immediate
 ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-sqlite/gitea-repositories
 
 [repository.local]
-LOCAL_COPY_PATH = tmp/local-repo-sqlite
-LOCAL_WIKI_PATH = tmp/local-wiki-sqlite
+LOCAL_COPY_PATH = integrations/gitea-integration-sqlite/tmp/local-repo
+
+[repository.upload]
+TEMP_PATH = integrations/gitea-integration-sqlite/tmp/uploads
 
 [repository.signing]
 SIGNING_KEY = none
@@ -35,15 +38,15 @@ SSH_LISTEN_HOST  = localhost
 SSH_PORT         = 2203
 START_SSH_SERVER = true
 LFS_START_SERVER = true
-LFS_CONTENT_PATH = integrations/gitea-integration-sqlite/data
+LFS_CONTENT_PATH = integrations/gitea-integration-sqlite/data/lfs
 OFFLINE_MODE     = false
 LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
 APP_DATA_PATH    = integrations/gitea-integration-sqlite/data
 ENABLE_GZIP      = true
 BUILTIN_SSH_SERVER_USER = git
 
 [attachment]
-PATH = integrations/gitea-integration-sqlite/data
+PATH = integrations/gitea-integration-sqlite/data/attachments
 
 [mailer]
 ENABLED     = true
@@ -67,7 +70,6 @@ ENABLE_FEDERATED_AVATAR       = false
 AVATAR_UPLOAD_PATH            = integrations/gitea-integration-sqlite/data/avatars
 REPOSITORY_AVATAR_UPLOAD_PATH = integrations/gitea-integration-sqlite/data/repo-avatars
 
-
 [session]
 PROVIDER = file
 PROVIDER_CONFIG = integrations/gitea-integration-sqlite/data/sessions

models/action.go

@@ -395,3 +395,13 @@ func activityQueryCondition(opts GetFeedsOptions) (builder.Cond, error) {
 
 	return cond, nil
 }
+
+// DeleteOldActions deletes all old actions from database.
+func DeleteOldActions(olderThan time.Duration) (err error) {
+	if olderThan <= 0 {
+		return nil
+	}
+
+	_, err = x.Where("created_unix < ?", time.Now().Add(-olderThan).Unix()).Delete(&Action{})
+	return
+}

models/consistency.go

@@ -296,11 +296,15 @@ func CountOrphanedObjects(subject, refobject, joinCond string) (int64, error) {
 
 // DeleteOrphanedObjects delete subjects with have no existing refobject anymore
 func DeleteOrphanedObjects(subject, refobject, joinCond string) error {
-	_, err := x.In("id", builder.Select("`"+subject+"`.id").
+	subQuery := builder.Select("`"+subject+"`.id").
 		From("`"+subject+"`").
 		Join("LEFT", "`"+refobject+"`", joinCond).
-		Where(builder.IsNull{"`" + refobject + "`.id"})).
-		Delete("`" + subject + "`")
+		Where(builder.IsNull{"`" + refobject + "`.id"})
+	sql, args, err := builder.Delete(builder.In("id", subQuery)).From("`" + subject + "`").ToSQL()
+	if err != nil {
+		return err
+	}
+	_, err = x.Exec(append([]interface{}{sql}, args...)...)
 	return err
 }
 

models/consistency_test.go

@@ -0,0 +1,32 @@
+// Copyright 2021 Gitea. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package models
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDeleteOrphanedObjects(t *testing.T) {
+	assert.NoError(t, PrepareTestDatabase())
+
+	countBefore, err := x.Count(&PullRequest{})
+	assert.NoError(t, err)
+
+	_, err = x.Insert(&PullRequest{IssueID: 1000}, &PullRequest{IssueID: 1001}, &PullRequest{IssueID: 1003})
+	assert.NoError(t, err)
+
+	orphaned, err := CountOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id")
+	assert.NoError(t, err)
+	assert.EqualValues(t, 3, orphaned)
+
+	err = DeleteOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id")
+	assert.NoError(t, err)
+
+	countAfter, err := x.Count(&PullRequest{})
+	assert.NoError(t, err)
+	assert.EqualValues(t, countBefore, countAfter)
+}