Verified Commit 6cc4f2f0 authored by Loïc Dachary's avatar Loïc Dachary
Browse files

Merge branch 'feature-projectbase' into base-activitypub

parents b73a5a0d 3008323e
Pipeline #629 passed with stage
in 19 minutes and 59 seconds
......@@ -116,3 +116,4 @@ prime/
# Manpage
/man
*~
---
kind: pipeline
name: testing-amd64
services:
- name: mysql
image: mysql:5.7
environment:
MYSQL_ALLOW_EMPTY_PASSWORD: yes
MYSQL_DATABASE: test
- name: ldap
image: gitea/test-openldap:latest
steps:
- name: build
pull: always
image: golang:1.17
commands:
- make backend
environment:
GOSUMDB: sum.golang.org
TAGS: bindata sqlite sqlite_unlock_notify
- name: fix-permissions
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
commands:
- chown -R gitea:gitea .
- name: lint-backend
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
- make lint-backend
environment:
GOSUMDB: sum.golang.org
TAGS: bindata sqlite sqlite_unlock_notify
- name: unit-test
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
- make unit-test-coverage
environment:
TAGS: bindata sqlite sqlite_unlock_notify
- name: test-mysql
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
- make test-mysql-migration integration-test-coverage
environment:
TAGS: bindata
TEST_LDAP: 1
USE_REPO_TEST_DIR: 1
depends_on:
- build
- name: test-sqlite
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
- timeout -s ABRT 40m make test-sqlite-migration test-sqlite
environment:
TAGS: bindata gogit sqlite sqlite_unlock_notify
TEST_TAGS: bindata gogit sqlite sqlite_unlock_notify
USE_REPO_TEST_DIR: 1
depends_on:
- build
- name: generate-coverage
image: golang:1.17
commands:
- make coverage
- go tool cover -html=coverage.all -o coverage.html
environment:
TAGS: bindata
depends_on:
- unit-test
- test-mysql
---
kind: pipeline
name: docs
depends_on:
- compliance
trigger:
event:
- push
- tag
- pull_request
steps:
- name: build-docs
pull: always
image: plugins/hugo:latest
commands:
- apk add --no-cache make bash curl
- cd docs
- make trans-copy clean build
- name: publish-docs
pull: always
image: techknowlogick/drone-netlify:latest
settings:
path: docs/public/
site_id: d2260bae-7861-4c02-8646-8f6440b12672
environment:
NETLIFY_TOKEN:
from_secret: netlify_token
when:
branch:
- main
event:
- push
jobs:
tags:
- shell
before_script:
- docker ps -a -q | xargs --no-run-if-empty docker rm -f
script:
- curl -L https://github.com/drone/drone-cli/releases/latest/download/drone_linux_amd64.tar.gz | tar zx
- chmod +x drone
- ./drone exec --pipeline testing-amd64 .gitlab-ci-drone.yml
after_script:
- docker run --rm --volume $(pwd):/drone/src --workdir /drone/src gitea/test_env:linux-amd64 chown -R $(id -u) .
......@@ -363,7 +363,7 @@ coverage:
.PHONY: unit-test-coverage
unit-test-coverage:
@echo "Running unit-test-coverage $(GOTESTFLAGS) -tags '$(TEST_TAGS)'..."
@$(GO) test $(GOTESTFLAGS) -timeout=20m -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
@$(GO) test $(GOTESTFLAGS) -cpu=1 -timeout=20m -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
.PHONY: vendor
vendor:
......
......@@ -10,8 +10,7 @@ PASSWD = {{TEST_MYSQL_PASSWORD}}
SSL_MODE = disable
[indexer]
ISSUE_INDEXER_TYPE = elasticsearch
ISSUE_INDEXER_CONN_STR = http://elastic:changeme@elasticsearch:9200
ISSUE_INDEXER_PATH = integrations/gitea-integration-mysql/indexers/issues.bleve
ISSUE_INDEXER_QUEUE_DIR = integrations/gitea-integration-mysql/indexers/issues.queue
REPO_INDEXER_ENABLED = true
REPO_INDEXER_PATH = integrations/gitea-integration-mysql/indexers/repos.bleve
......@@ -52,27 +51,8 @@ OFFLINE_MODE = false
LFS_START_SERVER = true
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
[lfs]
MINIO_BASE_PATH = lfs/
[attachment]
MINIO_BASE_PATH = attachments/
[avatars]
MINIO_BASE_PATH = avatars/
[repo-avatars]
MINIO_BASE_PATH = repo-avatars/
[storage]
STORAGE_TYPE = minio
SERVE_DIRECT = false
MINIO_ENDPOINT = minio:9000
MINIO_ACCESS_KEY_ID = 123456
MINIO_SECRET_ACCESS_KEY = 12345678
MINIO_BUCKET = gitea
MINIO_LOCATION = us-east-1
MINIO_USE_SSL = false
PATH = integrations/gitea-integration-mysql/data/attachments
[mailer]
ENABLED = true
......
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package integrations
import (
"fmt"
"net/url"
"sort"
"testing"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/services/migrations"
projectbase_service "code.gitea.io/gitea/services/projectbase"
"github.com/stretchr/testify/assert"
)
func TestProjectBaseUpdate(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
AllowedDomains := setting.Migrations.AllowedDomains
setting.Migrations.AllowedDomains = "loopback"
AllowLocalNetworks := setting.Migrations.AllowLocalNetworks
setting.Migrations.AllowLocalNetworks = true
setting.Database.LogSQL = true
AppVer := setting.AppVer
setting.AppVer = "1.15"
defer func() {
setting.Migrations.AllowedDomains = AllowedDomains
setting.Migrations.AllowLocalNetworks = AllowLocalNetworks
setting.Database.LogSQL = false
setting.AppVer = AppVer
}()
assert.NoError(t, migrations.Init())
reponame := "repo1"
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: reponame}).(*repo_model.Repository)
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}).(*user_model.User)
session := loginUser(t, repoOwner.Name)
token := getTokenForLoggedInUser(t, session)
assert.False(t, repo.HasProjectBase())
assert.NoError(t, projectbase_service.InitProjectBase(repo))
assert.True(t, repo.HasProjectBase())
assert.NoError(t, projectbase_service.UpdateProjectBase(token, repo))
gitRepo, err := git.OpenRepository(repo.ProjectBasePath())
fmt.Println(repo.ProjectBasePath())
assert.NoError(t, err)
defer gitRepo.Close()
files := []string{"repo.yml", "issue.yml", "comments"}
filelist, err := gitRepo.LsTree("HEAD", files...)
assert.NoError(t, err)
sort.Strings(filelist)
var expected = append(files, "")
sort.Strings(expected)
assert.Equal(t, expected, filelist)
})
}
......@@ -8,6 +8,7 @@ import (
"fmt"
"net/http"
"net/http/httptest"
"os"
"testing"
"code.gitea.io/gitea/modules/structs"
......@@ -39,6 +40,10 @@ func testRepoMigrate(t testing.TB, session *TestSession, cloneAddr, repoName str
}
func TestRepoMigrate(t *testing.T) {
if os.Getenv("GITHUB_READ_TOKEN") == "" {
t.Skip("skipped test because GITHUB_READ_TOKEN was not in the environment")
}
defer prepareTestEnv(t)()
session := loginUser(t, "user2")
testRepoMigrate(t, session, "https://github.com/go-gitea/test_repo.git", "git")
......
......@@ -926,6 +926,11 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error {
admin_model.RemoveAllWithNotice(db.DefaultContext, "Delete repository wiki", repo.WikiPath())
}
// Remove projectbase files
if repo.HasProjectBase() {
admin_model.RemoveAllWithNotice(db.DefaultContext, "Delete repository projectbase", repo.ProjectBasePath())
}
// Remove archives
for i := range archivePaths {
admin_model.RemoveStorageWithNotice(db.DefaultContext, storage.RepoArchives, "Delete repo archive file", archivePaths[i])
......
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repo
import (
"path/filepath"
"strings"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/util"
)
// ProjectBaseCloneLink function
func (repo *Repository) ProjectBaseCloneLink() *CloneLink {
return repo.cloneLink(".projectbase")
}
// ProjectBasePath function
func ProjectBasePath(userName, repoName string) string {
return filepath.Join(user_model.UserPath(userName), strings.ToLower(repoName)+".projectbase.git")
}
// ProjectBasePath function
func (repo *Repository) ProjectBasePath() string {
return ProjectBasePath(repo.OwnerName, repo.Name)
}
// HasProjectBase function
func (repo *Repository) HasProjectBase() bool {
isDir, err := util.IsDir(repo.ProjectBasePath())
if err != nil {
log.Error("Unable to check if %s is a directory: %v", repo.ProjectBasePath(), err)
}
return isDir
}
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repo
import (
"os"
"path/filepath"
"testing"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/setting"
"github.com/stretchr/testify/assert"
)
func TestRepository_ProjectBaseCloneLink(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())
repo := unittest.AssertExistsAndLoadBean(t, &Repository{ID: 1}).(*Repository)
cloneLink := repo.ProjectBaseCloneLink()
assert.Equal(t, "ssh://runuser@try.gitea.io:3000/user2/repo1.projectbase.git", cloneLink.SSH)
assert.Equal(t, "https://try.gitea.io/user2/repo1.projectbase.git", cloneLink.HTTPS)
}
func TestProjectBasePath(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())
expected := filepath.Join(setting.RepoRootPath, "user2/repo1.projectbase.git")
assert.Equal(t, expected, ProjectBasePath("user2", "repo1"))
}
func TestRepository_ProjectBasePath(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())
repo := unittest.AssertExistsAndLoadBean(t, &Repository{ID: 1}).(*Repository)
expected := filepath.Join(setting.RepoRootPath, "user2/repo1.projectbase.git")
assert.Equal(t, expected, repo.ProjectBasePath())
}
func TestRepository_HasProjectBase(t *testing.T) {
unittest.PrepareTestEnv(t)
repo1 := unittest.AssertExistsAndLoadBean(t, &Repository{ID: 1}).(*Repository)
assert.False(t, repo1.HasProjectBase())
os.MkdirAll(ProjectBasePath("user2", "repo1"), os.ModePerm)
assert.True(t, repo1.HasProjectBase())
}
......@@ -27,7 +27,7 @@ import (
var (
reservedRepoNames = []string{".", ".."}
reservedRepoPatterns = []string{"*.git", "*.wiki", "*.rss", "*.atom"}
reservedRepoPatterns = []string{"*.git", "*.wiki", "*.projectbase", "*.rss", "*.atom"}
)
// IsUsableRepoName returns true when repository is usable
......@@ -509,11 +509,8 @@ func ComposeHTTPSCloneURL(owner, repo string) string {
return fmt.Sprintf("%s%s/%s.git", setting.AppURL, url.PathEscape(owner), url.PathEscape(repo))
}
func (repo *Repository) cloneLink(isWiki bool) *CloneLink {
repoName := repo.Name
if isWiki {
repoName += ".wiki"
}
func (repo *Repository) cloneLink(suffix string) *CloneLink {
repoName := repo.Name + suffix
sshUser := setting.RunUser
if setting.SSH.StartBuiltinServer {
......@@ -543,7 +540,7 @@ func (repo *Repository) cloneLink(isWiki bool) *CloneLink {
// CloneLink returns clone URLs of repository.
func (repo *Repository) CloneLink() (cl *CloneLink) {
return repo.cloneLink(false)
return repo.cloneLink("")
}
// GetOriginalURLHostname returns the hostname of a URL or the URL
......
......@@ -165,6 +165,18 @@ func ChangeRepositoryName(doer *user_model.User, repo *Repository, newRepoName s
}
}
projectBasePath := repo.ProjectBasePath()
isExist, err = util.IsExist(projectBasePath)
if err != nil {
log.Error("Unable to check if %s exists. Error: %v", projectBasePath, err)
return err
}
if isExist {
if err = util.Rename(projectBasePath, ProjectBasePath(repo.Owner.Name, newRepoName)); err != nil {
return fmt.Errorf("rename repository projectbase: %v", err)
}
}
ctx, committer, err := db.TxContext()
if err != nil {
return err
......
......@@ -16,7 +16,7 @@ import (
// WikiCloneLink returns clone URLs of repository wiki.
func (repo *Repository) WikiCloneLink() *CloneLink {
return repo.cloneLink(true)
return repo.cloneLink(".wiki")
}
// WikiPath returns wiki data path by given user and repository name.
......
......@@ -208,10 +208,11 @@ func CreatePendingRepositoryTransfer(doer, newOwner *user_model.User, repoID int
func TransferOwnership(doer *user_model.User, newOwnerName string, repo *repo_model.Repository) (err error) {
repoRenamed := false
wikiRenamed := false
projectBaseRenamed := false
oldOwnerName := doer.Name
defer func() {
if !repoRenamed && !wikiRenamed {
if !repoRenamed && !wikiRenamed && !projectBaseRenamed {
return
}
......@@ -234,6 +235,12 @@ func TransferOwnership(doer *user_model.User, newOwnerName string, repo *repo_mo
}
}
if projectBaseRenamed {
if err := util.Rename(repo_model.ProjectBasePath(newOwnerName, repo.Name), repo_model.ProjectBasePath(oldOwnerName, repo.Name)); err != nil {
log.Critical("Unable to move projectbase for repository %s/%s directory from %s back to correct place %s: %v", oldOwnerName, repo.Name, repo_model.ProjectBasePath(newOwnerName, repo.Name), repo_model.ProjectBasePath(oldOwnerName, repo.Name), err)
}
}
if recoverErr != nil {
log.Error("Panic within TransferOwnership: %v\n%s", recoverErr, log.Stack(2))
panic(recoverErr)
......@@ -404,6 +411,19 @@ func TransferOwnership(doer *user_model.User, newOwnerName string, repo *repo_mo
wikiRenamed = true
}
// Rename remote projectbase repository to new path and delete local copy.
projectBasePath := repo_model.ProjectBasePath(oldOwner.Name, repo.Name)
if isExist, err := util.IsExist(projectBasePath); err != nil {
log.Error("Unable to check if %s exists. Error: %v", projectBasePath, err)
return err
} else if isExist {
if err := util.Rename(projectBasePath, repo_model.ProjectBasePath(newOwner.Name, repo.Name)); err != nil {
return fmt.Errorf("rename repository projectbase: %v", err)
}
projectBaseRenamed = true
}
if err := deleteRepositoryTransfer(ctx, repo.ID); err != nil {
return fmt.Errorf("deleteRepositoryTransfer: %v", err)
}
......
......@@ -525,6 +525,7 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) {
ctx.Data["RepoSearchEnabled"] = setting.Indexer.RepoIndexerEnabled
ctx.Data["CloneLink"] = repo.CloneLink()
ctx.Data["WikiCloneLink"] = repo.WikiCloneLink()
ctx.Data["ProjectBaseCloneLink"] = repo.ProjectBaseCloneLink()
if ctx.IsSigned {
ctx.Data["IsWatchingRepo"] = repo_model.IsWatching(ctx.User.ID, repo.ID)
......
......@@ -17,6 +17,10 @@ import (
)
func TestGitHubDownloadRepo(t *testing.T) {
if os.Getenv("GITHUB_READ_TOKEN") == "" {
t.Skip("skipped test because GITHUB_READ_TOKEN was not in the environment")
}
GithubLimitRateRemaining = 3 // Wait at 3 remaining since we could have 3 CI in //
downloader := NewGithubDownloaderV3(context.Background(), "https://github.com", "", "", os.Getenv("GITHUB_READ_TOKEN"), "go-gitea", "test_repo")
err := downloader.RefreshRate()
......
......@@ -172,6 +172,11 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio
return downloader, nil
}
// InternalMigrateRepository function
func InternalMigrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error {
return migrateRepository(downloader, uploader, opts, nil)
}
// migrateRepository will download information and then upload it to Uploader, this is a simple
// process for small repository. For a big repository, save all the data to disk
// before upload is better
......
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package projectbase
import (
"context"
"fmt"
"os"
"path/filepath"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/services/migrations"
files_service "code.gitea.io/gitea/services/repository/files"
)
// InitProjectBase function
func InitProjectBase(repo *repo_model.Repository) error {
if repo.HasProjectBase() {
return nil
}
if err := git.InitRepository(repo.ProjectBasePath(), true); err != nil {
return fmt.Errorf("InitRepository: %v", err)
} else if _, err = git.NewCommand("symbolic-ref", "HEAD", git.BranchPrefix+"master").RunInDir(repo.ProjectBasePath()); err != nil {
return fmt.Errorf("unable to set default branch to master: %v", err)
}
return nil
}
// UpdateProjectBase function
func UpdateProjectBase(token string, repo *repo_model.Repository) error {
//
// Checkout repository
//
temp, err := files_service.NewTemporaryUploadRepository(repo)
if err != nil {
return err
}
basePath := temp.BasePath()
defer temp.Close()
if stdout, err := git.NewCommand("clone", "--no-checkout", "--shared", "--depth=1", repo.ProjectBasePath(), basePath).Run(); err != nil {
log.Error("Failed to clone projectbase repository: %s %s (%v)", repo.FullName(), stdout, err)
return fmt.Errorf("Failed to clone projectbase repository: %s %s (%v)", repo.FullName(), stdout, err)
}
//
// Create downloader & uploader
//
var opts = migrations.MigrateOptions{
GitServiceType: structs.GiteaService,
Issues: true,
Comments: true,
AuthToken: token,
CloneAddr: repo.CloneLink().HTTPS,
}
downloaderFactory := &migrations.GiteaDownloaderFactory{}
ctx := context.Background()
downloader, err := downloaderFactory.New(ctx, opts)
if err != nil {
return err
}
uploader, err := migrations.NewRepositoryDumper(ctx, basePath, repo.OwnerName, repo.Name, opts)
if err != nil {
return err
}
//
// Perform the migration
//
if err := migrations.InternalMigrateRepository(downloader, uploader, opts); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1)
}
return err
}
d := filepath.Join(basePath, repo.OwnerName, repo.Name)
for _, f := range []string{"repo.yml", "topic.yml", "issue.yml", "comments"} {
if err := os.Rename(filepath.Join(d, f), filepath.Join(basePath, f)); err != nil {
return err
}
}
if err := util.RemoveAll(filepath.Join(basePath, repo.OwnerName)); err != nil {
return err
}
//
// Commit the migration to the repository & push it
//
if stdout, err := git.NewCommand("add", ".").RunInDir(basePath); err != nil {