Merge branch 'feature-projectbase' into base-activitypub

.gitignore

@@ -116,3 +116,4 @@ prime/
 
 # Manpage
 /man
+*~

.gitlab-ci-drone.yml

+---
+kind: pipeline
+name: testing-amd64
+
+services:
+  - name: mysql
+    image: mysql:5.7
+    environment:
+      MYSQL_ALLOW_EMPTY_PASSWORD: yes
+      MYSQL_DATABASE: test
+
+  - name: ldap
+    image: gitea/test-openldap:latest
+
+steps:
+  - name: build
+    pull: always
+    image: golang:1.17
+    commands:
+      - make backend
+    environment:
+      GOSUMDB: sum.golang.org
+      TAGS: bindata sqlite sqlite_unlock_notify
+
+  - name: fix-permissions
+    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
+    commands:
+      - chown -R gitea:gitea .
+
+  - name: lint-backend
+    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
+    user: gitea
+    commands:
+      - make lint-backend
+    environment:
+      GOSUMDB: sum.golang.org
+      TAGS: bindata sqlite sqlite_unlock_notify
+
+  - name: unit-test
+    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
+    user: gitea
+    commands:
+      - make unit-test-coverage
+    environment:
+      TAGS: bindata sqlite sqlite_unlock_notify
+
+  - name: test-mysql
+    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
+    user: gitea
+    commands:
+      - make test-mysql-migration integration-test-coverage
+    environment:
+      TAGS: bindata
+      TEST_LDAP: 1
+      USE_REPO_TEST_DIR: 1
+    depends_on:
+      - build
+
+  - name: test-sqlite
+    image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
+    user: gitea
+    commands:
+      - timeout -s ABRT 40m make test-sqlite-migration test-sqlite
+    environment:
+      TAGS: bindata gogit sqlite sqlite_unlock_notify
+      TEST_TAGS: bindata gogit sqlite sqlite_unlock_notify
+      USE_REPO_TEST_DIR: 1
+    depends_on:
+      - build
+
+  - name: generate-coverage
+    image: golang:1.17
+    commands:
+      - make coverage
+      - go tool cover -html=coverage.all -o coverage.html
+    environment:
+      TAGS: bindata
+    depends_on:
+      - unit-test
+      - test-mysql
+
+---
+kind: pipeline
+name: docs
+
+depends_on:
+  - compliance
+
+trigger:
+  event:
+    - push
+    - tag
+    - pull_request
+
+steps:
+  - name: build-docs
+    pull: always
+    image: plugins/hugo:latest
+    commands:
+      - apk add --no-cache make bash curl
+      - cd docs
+      - make trans-copy clean build
+
+  - name: publish-docs
+    pull: always
+    image: techknowlogick/drone-netlify:latest
+    settings:
+      path: docs/public/
+      site_id: d2260bae-7861-4c02-8646-8f6440b12672
+    environment:
+      NETLIFY_TOKEN:
+        from_secret: netlify_token
+    when:
+      branch:
+        - main
+      event:
+        - push

.gitlab-ci.yml

+jobs:
+  tags:
+    - shell
+
+  before_script:
+    - docker ps -a -q | xargs --no-run-if-empty docker rm -f
+
+  script:
+    - curl -L https://github.com/drone/drone-cli/releases/latest/download/drone_linux_amd64.tar.gz | tar zx
+    - chmod +x drone
+    - ./drone exec --pipeline testing-amd64 .gitlab-ci-drone.yml
+
+  after_script:
+    - docker run --rm --volume $(pwd):/drone/src --workdir /drone/src  gitea/test_env:linux-amd64 chown -R $(id -u) .

Makefile

@@ -363,7 +363,7 @@ coverage:
 .PHONY: unit-test-coverage
 unit-test-coverage:
 	@echo "Running unit-test-coverage $(GOTESTFLAGS) -tags '$(TEST_TAGS)'..."
-	@$(GO) test $(GOTESTFLAGS) -timeout=20m -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
+	@$(GO) test $(GOTESTFLAGS) -cpu=1 -timeout=20m -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
 
 .PHONY: vendor
 vendor:

integrations/mysql.ini.tmpl

@@ -10,8 +10,7 @@ PASSWD   = {{TEST_MYSQL_PASSWORD}}
 SSL_MODE = disable
 
 [indexer]
-ISSUE_INDEXER_TYPE = elasticsearch
-ISSUE_INDEXER_CONN_STR = http://elastic:changeme@elasticsearch:9200
+ISSUE_INDEXER_PATH = integrations/gitea-integration-mysql/indexers/issues.bleve
 ISSUE_INDEXER_QUEUE_DIR = integrations/gitea-integration-mysql/indexers/issues.queue
 REPO_INDEXER_ENABLED = true
 REPO_INDEXER_PATH = integrations/gitea-integration-mysql/indexers/repos.bleve
@@ -52,27 +51,8 @@ OFFLINE_MODE     = false
 LFS_START_SERVER = true
 LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
 
-[lfs]
-MINIO_BASE_PATH = lfs/
-
 [attachment]
-MINIO_BASE_PATH = attachments/
-
-[avatars]
-MINIO_BASE_PATH = avatars/
-
-[repo-avatars]
-MINIO_BASE_PATH = repo-avatars/
-
-[storage]
-STORAGE_TYPE = minio
-SERVE_DIRECT = false
-MINIO_ENDPOINT = minio:9000
-MINIO_ACCESS_KEY_ID = 123456
-MINIO_SECRET_ACCESS_KEY = 12345678
-MINIO_BUCKET = gitea
-MINIO_LOCATION = us-east-1
-MINIO_USE_SSL = false
+PATH = integrations/gitea-integration-mysql/data/attachments
 
 [mailer]
 ENABLED = true

integrations/projectbase_test.go

+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+	"fmt"
+	"net/url"
+	"sort"
+	"testing"
+
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/models/unittest"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/services/migrations"
+	projectbase_service "code.gitea.io/gitea/services/projectbase"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestProjectBaseUpdate(t *testing.T) {
+	onGiteaRun(t, func(t *testing.T, u *url.URL) {
+		AllowedDomains := setting.Migrations.AllowedDomains
+		setting.Migrations.AllowedDomains = "loopback"
+		AllowLocalNetworks := setting.Migrations.AllowLocalNetworks
+		setting.Migrations.AllowLocalNetworks = true
+		setting.Database.LogSQL = true
+		AppVer := setting.AppVer
+		setting.AppVer = "1.15"
+		defer func() {
+			setting.Migrations.AllowedDomains = AllowedDomains
+			setting.Migrations.AllowLocalNetworks = AllowLocalNetworks
+			setting.Database.LogSQL = false
+			setting.AppVer = AppVer
+		}()
+
+		assert.NoError(t, migrations.Init())
+
+		reponame := "repo1"
+		repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: reponame}).(*repo_model.Repository)
+		repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}).(*user_model.User)
+		session := loginUser(t, repoOwner.Name)
+		token := getTokenForLoggedInUser(t, session)
+		assert.False(t, repo.HasProjectBase())
+		assert.NoError(t, projectbase_service.InitProjectBase(repo))
+		assert.True(t, repo.HasProjectBase())
+		assert.NoError(t, projectbase_service.UpdateProjectBase(token, repo))
+
+		gitRepo, err := git.OpenRepository(repo.ProjectBasePath())
+		fmt.Println(repo.ProjectBasePath())
+		assert.NoError(t, err)
+		defer gitRepo.Close()
+		files := []string{"repo.yml", "issue.yml", "comments"}
+		filelist, err := gitRepo.LsTree("HEAD", files...)
+		assert.NoError(t, err)
+		sort.Strings(filelist)
+		var expected = append(files, "")
+		sort.Strings(expected)
+		assert.Equal(t, expected, filelist)
+	})
+}

integrations/repo_migrate_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"testing"
 
 	"code.gitea.io/gitea/modules/structs"
@@ -39,6 +40,10 @@ func testRepoMigrate(t testing.TB, session *TestSession, cloneAddr, repoName str
 }
 
 func TestRepoMigrate(t *testing.T) {
+	if os.Getenv("GITHUB_READ_TOKEN") == "" {
+		t.Skip("skipped test because GITHUB_READ_TOKEN was not in the environment")
+	}
+
 	defer prepareTestEnv(t)()
 	session := loginUser(t, "user2")
 	testRepoMigrate(t, session, "https://github.com/go-gitea/test_repo.git", "git")

models/repo.go

@@ -926,6 +926,11 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error {
 		admin_model.RemoveAllWithNotice(db.DefaultContext, "Delete repository wiki", repo.WikiPath())
 	}
 
+	// Remove projectbase files
+	if repo.HasProjectBase() {
+		admin_model.RemoveAllWithNotice(db.DefaultContext, "Delete repository projectbase", repo.ProjectBasePath())
+	}
+
 	// Remove archives
 	for i := range archivePaths {
 		admin_model.RemoveStorageWithNotice(db.DefaultContext, storage.RepoArchives, "Delete repo archive file", archivePaths[i])

models/repo/projectbase.go

+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package repo
+
+import (
+	"path/filepath"
+	"strings"
+
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/util"
+)
+
+// ProjectBaseCloneLink function
+func (repo *Repository) ProjectBaseCloneLink() *CloneLink {
+	return repo.cloneLink(".projectbase")
+}
+
+// ProjectBasePath function
+func ProjectBasePath(userName, repoName string) string {
+	return filepath.Join(user_model.UserPath(userName), strings.ToLower(repoName)+".projectbase.git")
+}
+
+// ProjectBasePath function
+func (repo *Repository) ProjectBasePath() string {
+	return ProjectBasePath(repo.OwnerName, repo.Name)
+}
+
+// HasProjectBase function
+func (repo *Repository) HasProjectBase() bool {
+	isDir, err := util.IsDir(repo.ProjectBasePath())
+	if err != nil {
+		log.Error("Unable to check if %s is a directory: %v", repo.ProjectBasePath(), err)
+	}
+	return isDir
+}

models/repo/projectbase_test.go

+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package repo
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"code.gitea.io/gitea/models/unittest"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRepository_ProjectBaseCloneLink(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+
+	repo := unittest.AssertExistsAndLoadBean(t, &Repository{ID: 1}).(*Repository)
+	cloneLink := repo.ProjectBaseCloneLink()
+	assert.Equal(t, "ssh://runuser@try.gitea.io:3000/user2/repo1.projectbase.git", cloneLink.SSH)
+	assert.Equal(t, "https://try.gitea.io/user2/repo1.projectbase.git", cloneLink.HTTPS)
+}
+
+func TestProjectBasePath(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+	expected := filepath.Join(setting.RepoRootPath, "user2/repo1.projectbase.git")
+	assert.Equal(t, expected, ProjectBasePath("user2", "repo1"))
+}
+
+func TestRepository_ProjectBasePath(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+	repo := unittest.AssertExistsAndLoadBean(t, &Repository{ID: 1}).(*Repository)
+	expected := filepath.Join(setting.RepoRootPath, "user2/repo1.projectbase.git")
+	assert.Equal(t, expected, repo.ProjectBasePath())
+}
+
+func TestRepository_HasProjectBase(t *testing.T) {
+	unittest.PrepareTestEnv(t)
+	repo1 := unittest.AssertExistsAndLoadBean(t, &Repository{ID: 1}).(*Repository)
+	assert.False(t, repo1.HasProjectBase())
+	os.MkdirAll(ProjectBasePath("user2", "repo1"), os.ModePerm)
+	assert.True(t, repo1.HasProjectBase())
+}

models/repo/repo.go

@@ -27,7 +27,7 @@ import (
 
 var (
 	reservedRepoNames    = []string{".", ".."}
-	reservedRepoPatterns = []string{"*.git", "*.wiki", "*.rss", "*.atom"}
+	reservedRepoPatterns = []string{"*.git", "*.wiki", "*.projectbase", "*.rss", "*.atom"}
 )
 
 // IsUsableRepoName returns true when repository is usable
@@ -509,11 +509,8 @@ func ComposeHTTPSCloneURL(owner, repo string) string {
 	return fmt.Sprintf("%s%s/%s.git", setting.AppURL, url.PathEscape(owner), url.PathEscape(repo))
 }
 
-func (repo *Repository) cloneLink(isWiki bool) *CloneLink {
-	repoName := repo.Name
-	if isWiki {
-		repoName += ".wiki"
-	}
+func (repo *Repository) cloneLink(suffix string) *CloneLink {
+	repoName := repo.Name + suffix
 
 	sshUser := setting.RunUser
 	if setting.SSH.StartBuiltinServer {
@@ -543,7 +540,7 @@ func (repo *Repository) cloneLink(isWiki bool) *CloneLink {
 
 // CloneLink returns clone URLs of repository.
 func (repo *Repository) CloneLink() (cl *CloneLink) {
-	return repo.cloneLink(false)
+	return repo.cloneLink("")
 }
 
 // GetOriginalURLHostname returns the hostname of a URL or the URL

models/repo/update.go

@@ -165,6 +165,18 @@ func ChangeRepositoryName(doer *user_model.User, repo *Repository, newRepoName s
 		}
 	}
 
+	projectBasePath := repo.ProjectBasePath()
+	isExist, err = util.IsExist(projectBasePath)
+	if err != nil {
+		log.Error("Unable to check if %s exists. Error: %v", projectBasePath, err)
+		return err
+	}
+	if isExist {
+		if err = util.Rename(projectBasePath, ProjectBasePath(repo.Owner.Name, newRepoName)); err != nil {
+			return fmt.Errorf("rename repository projectbase: %v", err)
+		}
+	}
+
 	ctx, committer, err := db.TxContext()
 	if err != nil {
 		return err

models/repo/wiki.go

@@ -16,7 +16,7 @@ import (
 
 // WikiCloneLink returns clone URLs of repository wiki.
 func (repo *Repository) WikiCloneLink() *CloneLink {
-	return repo.cloneLink(true)
+	return repo.cloneLink(".wiki")
 }
 
 // WikiPath returns wiki data path by given user and repository name.

models/repo_transfer.go

@@ -208,10 +208,11 @@ func CreatePendingRepositoryTransfer(doer, newOwner *user_model.User, repoID int
 func TransferOwnership(doer *user_model.User, newOwnerName string, repo *repo_model.Repository) (err error) {
 	repoRenamed := false
 	wikiRenamed := false
+	projectBaseRenamed := false
 	oldOwnerName := doer.Name
 
 	defer func() {
-		if !repoRenamed && !wikiRenamed {
+		if !repoRenamed && !wikiRenamed && !projectBaseRenamed {
 			return
 		}
 
@@ -234,6 +235,12 @@ func TransferOwnership(doer *user_model.User, newOwnerName string, repo *repo_mo
 			}
 		}
 
+		if projectBaseRenamed {
+			if err := util.Rename(repo_model.ProjectBasePath(newOwnerName, repo.Name), repo_model.ProjectBasePath(oldOwnerName, repo.Name)); err != nil {
+				log.Critical("Unable to move projectbase for repository %s/%s directory from %s back to correct place %s: %v", oldOwnerName, repo.Name, repo_model.ProjectBasePath(newOwnerName, repo.Name), repo_model.ProjectBasePath(oldOwnerName, repo.Name), err)
+			}
+		}
+
 		if recoverErr != nil {
 			log.Error("Panic within TransferOwnership: %v\n%s", recoverErr, log.Stack(2))
 			panic(recoverErr)
@@ -404,6 +411,19 @@ func TransferOwnership(doer *user_model.User, newOwnerName string, repo *repo_mo
 		wikiRenamed = true
 	}
 
+	// Rename remote projectbase repository to new path and delete local copy.
+	projectBasePath := repo_model.ProjectBasePath(oldOwner.Name, repo.Name)
+
+	if isExist, err := util.IsExist(projectBasePath); err != nil {
+		log.Error("Unable to check if %s exists. Error: %v", projectBasePath, err)
+		return err
+	} else if isExist {
+		if err := util.Rename(projectBasePath, repo_model.ProjectBasePath(newOwner.Name, repo.Name)); err != nil {
+			return fmt.Errorf("rename repository projectbase: %v", err)
+		}
+		projectBaseRenamed = true
+	}
+
 	if err := deleteRepositoryTransfer(ctx, repo.ID); err != nil {
 		return fmt.Errorf("deleteRepositoryTransfer: %v", err)
 	}

modules/context/repo.go

@@ -525,6 +525,7 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) {
 	ctx.Data["RepoSearchEnabled"] = setting.Indexer.RepoIndexerEnabled
 	ctx.Data["CloneLink"] = repo.CloneLink()
 	ctx.Data["WikiCloneLink"] = repo.WikiCloneLink()
+	ctx.Data["ProjectBaseCloneLink"] = repo.ProjectBaseCloneLink()
 
 	if ctx.IsSigned {
 		ctx.Data["IsWatchingRepo"] = repo_model.IsWatching(ctx.User.ID, repo.ID)

services/migrations/github_test.go

@@ -17,6 +17,10 @@ import (
 )
 
 func TestGitHubDownloadRepo(t *testing.T) {
+	if os.Getenv("GITHUB_READ_TOKEN") == "" {
+		t.Skip("skipped test because GITHUB_READ_TOKEN was not in the environment")
+	}
+
 	GithubLimitRateRemaining = 3 // Wait at 3 remaining since we could have 3 CI in //
 	downloader := NewGithubDownloaderV3(context.Background(), "https://github.com", "", "", os.Getenv("GITHUB_READ_TOKEN"), "go-gitea", "test_repo")
 	err := downloader.RefreshRate()

services/migrations/migrate.go

@@ -172,6 +172,11 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio
 	return downloader, nil
 }
 
+// InternalMigrateRepository function
+func InternalMigrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error {
+	return migrateRepository(downloader, uploader, opts, nil)
+}
+
 // migrateRepository will download information and then upload it to Uploader, this is a simple
 // process for small repository. For a big repository, save all the data to disk
 // before upload is better

services/projectbase/projectbase.go

+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package projectbase
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	repo_model "code.gitea.io/gitea/models/repo"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/services/migrations"
+	files_service "code.gitea.io/gitea/services/repository/files"
+)
+
+// InitProjectBase function
+func InitProjectBase(repo *repo_model.Repository) error {
+	if repo.HasProjectBase() {
+		return nil
+	}
+
+	if err := git.InitRepository(repo.ProjectBasePath(), true); err != nil {
+		return fmt.Errorf("InitRepository: %v", err)
+	} else if _, err = git.NewCommand("symbolic-ref", "HEAD", git.BranchPrefix+"master").RunInDir(repo.ProjectBasePath()); err != nil {
+		return fmt.Errorf("unable to set default branch to master: %v", err)
+	}
+	return nil
+}
+
+// UpdateProjectBase function
+func UpdateProjectBase(token string, repo *repo_model.Repository) error {
+	//
+	// Checkout repository
+	//
+	temp, err := files_service.NewTemporaryUploadRepository(repo)
+	if err != nil {
+		return err
+	}
+	basePath := temp.BasePath()
+	defer temp.Close()
+	if stdout, err := git.NewCommand("clone", "--no-checkout", "--shared", "--depth=1", repo.ProjectBasePath(), basePath).Run(); err != nil {
+		log.Error("Failed to clone projectbase repository: %s %s (%v)", repo.FullName(), stdout, err)
+		return fmt.Errorf("Failed to clone projectbase repository: %s %s (%v)", repo.FullName(), stdout, err)
+	}
+
+	//
+	// Create downloader & uploader
+	//
+	var opts = migrations.MigrateOptions{
+		GitServiceType: structs.GiteaService,
+		Issues:         true,
+		Comments:       true,
+		AuthToken:      token,
+		CloneAddr:      repo.CloneLink().HTTPS,
+	}
+	downloaderFactory := &migrations.GiteaDownloaderFactory{}
+	ctx := context.Background()
+	downloader, err := downloaderFactory.New(ctx, opts)
+	if err != nil {
+		return err
+	}
+	uploader, err := migrations.NewRepositoryDumper(ctx, basePath, repo.OwnerName, repo.Name, opts)
+	if err != nil {
+		return err
+	}
+
+	//
+	// Perform the migration
+	//
+	if err := migrations.InternalMigrateRepository(downloader, uploader, opts); err != nil {
+		if err1 := uploader.Rollback(); err1 != nil {
+			log.Error("rollback failed: %v", err1)
+		}
+		return err
+	}
+	d := filepath.Join(basePath, repo.OwnerName, repo.Name)
+	for _, f := range []string{"repo.yml", "topic.yml", "issue.yml", "comments"} {
+		if err := os.Rename(filepath.Join(d, f), filepath.Join(basePath, f)); err != nil {
+			return err
+		}
+	}
+	if err := util.RemoveAll(filepath.Join(basePath, repo.OwnerName)); err != nil {
+		return err
+	}
+
+	//
+	// Commit the migration to the repository & push it
+	//
+	if stdout, err := git.NewCommand("add", ".").RunInDir(basePath); err != nil {