Wiki mirroring implementation (#3233)
* Implement wiki mirroring, add Update mirrors operation to admin dashboard * bindata.go update after merge * Implement checking Git repo endpoint existence, support for BB included * Remove admin dashboard operation Fix bindata.go * Apply gofmt to repo model file * Try to remove bindata from PR * Revert accepted wiki names change in favor of better system * Remove unused imports
This commit is contained in:
parent
3380c946e1
commit
0885784f13
101
models/repo.go
101
models/repo.go
|
@ -659,6 +659,51 @@ type MigrateRepoOptions struct {
|
||||||
RemoteAddr string
|
RemoteAddr string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isGitRepoURL(repoURL string, timeout time.Duration) bool {
|
||||||
|
cmd := git.NewCommand("ls-remote")
|
||||||
|
cmd.AddArguments("-q", "-h", repoURL, "HEAD")
|
||||||
|
res, err := cmd.RunTimeout(timeout)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if strings.Contains(res, "fatal") || strings.Contains(res, "not found") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func wikiRemoteURL(remote string, timeout time.Duration) string {
|
||||||
|
wikiRemoteStd := remote
|
||||||
|
wikiRemoteBitBucket := remote
|
||||||
|
/*
|
||||||
|
GitHub, GitLab, Gogs: NAME.wiki.git
|
||||||
|
BitBucket: NAME.git/wiki
|
||||||
|
*/
|
||||||
|
gitSuffixed := strings.HasSuffix(remote, ".git")
|
||||||
|
if gitSuffixed {
|
||||||
|
wikiRemoteStd = wikiRemoteStd[:len(wikiRemoteStd)-4]
|
||||||
|
wikiRemoteBitBucket += "/wiki"
|
||||||
|
} else {
|
||||||
|
wikiRemoteBitBucket += ".git/wiki"
|
||||||
|
}
|
||||||
|
wikiRemoteStd += ".wiki.git"
|
||||||
|
isBB := strings.Contains(remote, "bitbucket")
|
||||||
|
if isBB {
|
||||||
|
if isGitRepoURL(wikiRemoteBitBucket, timeout) {
|
||||||
|
return wikiRemoteBitBucket
|
||||||
|
} else if isGitRepoURL(wikiRemoteStd, timeout) {
|
||||||
|
return wikiRemoteStd
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if isGitRepoURL(wikiRemoteStd, timeout) {
|
||||||
|
return wikiRemoteStd
|
||||||
|
} else if isGitRepoURL(wikiRemoteBitBucket, timeout) {
|
||||||
|
return wikiRemoteBitBucket
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// MigrateRepository migrates a existing repository from other project hosting.
|
// MigrateRepository migrates a existing repository from other project hosting.
|
||||||
func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
|
func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
|
||||||
repo, err := CreateRepository(u, CreateRepoOptions{
|
repo, err := CreateRepository(u, CreateRepoOptions{
|
||||||
|
@ -676,6 +721,7 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
|
||||||
os.MkdirAll(tmpDir, os.ModePerm)
|
os.MkdirAll(tmpDir, os.ModePerm)
|
||||||
|
|
||||||
repoPath := RepoPath(u.Name, opts.Name)
|
repoPath := RepoPath(u.Name, opts.Name)
|
||||||
|
wikiPath := WikiPath(u.Name, opts.Name)
|
||||||
|
|
||||||
if u.IsOrganization() {
|
if u.IsOrganization() {
|
||||||
t, err := u.GetOwnerTeam()
|
t, err := u.GetOwnerTeam()
|
||||||
|
@ -687,15 +733,28 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
|
||||||
repo.NumWatches = 1
|
repo.NumWatches = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
gitTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second
|
||||||
os.RemoveAll(repoPath)
|
os.RemoveAll(repoPath)
|
||||||
if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{
|
if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{
|
||||||
Mirror: true,
|
Mirror: true,
|
||||||
Quiet: true,
|
Quiet: true,
|
||||||
Timeout: time.Duration(setting.Git.Timeout.Migrate) * time.Second,
|
Timeout: gitTimeout,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return repo, fmt.Errorf("Clone: %v", err)
|
return repo, fmt.Errorf("Clone: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
wikiRemotePath := wikiRemoteURL(opts.RemoteAddr, gitTimeout)
|
||||||
|
if wikiRemotePath != "" {
|
||||||
|
os.RemoveAll(wikiPath)
|
||||||
|
if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
|
||||||
|
Mirror: true,
|
||||||
|
Quiet: true,
|
||||||
|
Timeout: gitTimeout,
|
||||||
|
}); err != nil {
|
||||||
|
log.Info("Clone wiki failed: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Check if repository is empty.
|
// Check if repository is empty.
|
||||||
_, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1")
|
_, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -735,14 +794,22 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
|
||||||
return repo, UpdateRepository(repo, false)
|
return repo, UpdateRepository(repo, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
return CleanUpMigrateInfo(repo, repoPath)
|
return CleanUpMigrateInfo(repo)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Finish migrating repository with things that don't need to be done for mirrors.
|
// Finish migrating repository with things that don't need to be done for mirrors.
|
||||||
func CleanUpMigrateInfo(repo *Repository, repoPath string) (*Repository, error) {
|
func CleanUpMigrateInfo(repo *Repository) (*Repository, error) {
|
||||||
|
repoPath := repo.RepoPath()
|
||||||
|
hasWiki := repo.HasWiki()
|
||||||
|
|
||||||
if err := createUpdateHook(repoPath); err != nil {
|
if err := createUpdateHook(repoPath); err != nil {
|
||||||
return repo, fmt.Errorf("createUpdateHook: %v", err)
|
return repo, fmt.Errorf("createUpdateHook: %v", err)
|
||||||
}
|
}
|
||||||
|
if hasWiki {
|
||||||
|
if err := createUpdateHook(repoPath); err != nil {
|
||||||
|
return repo, fmt.Errorf("createUpdateHook: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Clean up mirror info which prevents "push --all".
|
// Clean up mirror info which prevents "push --all".
|
||||||
// This also removes possible user credentials.
|
// This also removes possible user credentials.
|
||||||
|
@ -755,6 +822,17 @@ func CleanUpMigrateInfo(repo *Repository, repoPath string) (*Repository, error)
|
||||||
if err = cfg.SaveToIndent(configPath, "\t"); err != nil {
|
if err = cfg.SaveToIndent(configPath, "\t"); err != nil {
|
||||||
return repo, fmt.Errorf("save config file: %v", err)
|
return repo, fmt.Errorf("save config file: %v", err)
|
||||||
}
|
}
|
||||||
|
if hasWiki {
|
||||||
|
wikiConfigPath := filepath.Join(repo.WikiPath(), "config")
|
||||||
|
cfg, err = ini.Load(wikiConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
return repo, fmt.Errorf("open wiki config file: %v", err)
|
||||||
|
}
|
||||||
|
cfg.DeleteSection("remote \"origin\"")
|
||||||
|
if err = cfg.SaveToIndent(wikiConfigPath, "\t"); err != nil {
|
||||||
|
return repo, fmt.Errorf("save wiki config file: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return repo, UpdateRepository(repo, false)
|
return repo, UpdateRepository(repo, false)
|
||||||
}
|
}
|
||||||
|
@ -1695,6 +1773,8 @@ func MirrorUpdate() {
|
||||||
}
|
}
|
||||||
|
|
||||||
repoPath := m.Repo.RepoPath()
|
repoPath := m.Repo.RepoPath()
|
||||||
|
wikiPath := m.Repo.WikiPath()
|
||||||
|
timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
|
||||||
|
|
||||||
gitArgs := []string{"remote", "update"}
|
gitArgs := []string{"remote", "update"}
|
||||||
if m.EnablePrune {
|
if m.EnablePrune {
|
||||||
|
@ -1702,8 +1782,7 @@ func MirrorUpdate() {
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, stderr, err := process.ExecDir(
|
if _, stderr, err := process.ExecDir(
|
||||||
time.Duration(setting.Git.Timeout.Mirror)*time.Second,
|
timeout, repoPath, fmt.Sprintf("MirrorUpdate: %s", repoPath),
|
||||||
repoPath, fmt.Sprintf("MirrorUpdate: %s", repoPath),
|
|
||||||
"git", gitArgs...); err != nil {
|
"git", gitArgs...); err != nil {
|
||||||
desc := fmt.Sprintf("Fail to update mirror repository(%s): %s", repoPath, stderr)
|
desc := fmt.Sprintf("Fail to update mirror repository(%s): %s", repoPath, stderr)
|
||||||
log.Error(4, desc)
|
log.Error(4, desc)
|
||||||
|
@ -1712,6 +1791,18 @@ func MirrorUpdate() {
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if m.Repo.HasWiki() {
|
||||||
|
if _, stderr, err := process.ExecDir(
|
||||||
|
timeout, wikiPath, fmt.Sprintf("MirrorUpdate: %s", wikiPath),
|
||||||
|
"git", "remote", "update", "--prune"); err != nil {
|
||||||
|
desc := fmt.Sprintf("Fail to update mirror wiki repository(%s): %s", wikiPath, stderr)
|
||||||
|
log.Error(4, desc)
|
||||||
|
if err = CreateRepositoryNotice(desc); err != nil {
|
||||||
|
log.Error(4, "CreateRepositoryNotice: %v", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
m.NextUpdate = time.Now().Add(time.Duration(m.Interval) * time.Hour)
|
m.NextUpdate = time.Now().Add(time.Duration(m.Interval) * time.Hour)
|
||||||
mirrors = append(mirrors, m)
|
mirrors = append(mirrors, m)
|
||||||
|
|
|
@ -223,6 +223,7 @@ func RepoAssignment(args ...bool) macaron.Handler {
|
||||||
ctx.Data["IsRepositoryOwner"] = ctx.Repo.IsOwner()
|
ctx.Data["IsRepositoryOwner"] = ctx.Repo.IsOwner()
|
||||||
ctx.Data["IsRepositoryAdmin"] = ctx.Repo.IsAdmin()
|
ctx.Data["IsRepositoryAdmin"] = ctx.Repo.IsAdmin()
|
||||||
ctx.Data["IsRepositoryWriter"] = ctx.Repo.IsWriter()
|
ctx.Data["IsRepositoryWriter"] = ctx.Repo.IsWriter()
|
||||||
|
ctx.Data["IsRepositoryMirror"] = repo.IsMirror
|
||||||
|
|
||||||
ctx.Data["DisableSSH"] = setting.SSH.Disabled
|
ctx.Data["DisableSSH"] = setting.SSH.Disabled
|
||||||
ctx.Data["CloneLink"] = repo.CloneLink()
|
ctx.Data["CloneLink"] = repo.CloneLink()
|
||||||
|
|
|
@ -174,7 +174,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) {
|
||||||
}
|
}
|
||||||
repo.IsMirror = false
|
repo.IsMirror = false
|
||||||
|
|
||||||
if _, err := models.CleanUpMigrateInfo(repo, models.RepoPath(ctx.Repo.Owner.Name, repo.Name)); err != nil {
|
if _, err := models.CleanUpMigrateInfo(repo); err != nil {
|
||||||
ctx.Handle(500, "CleanUpMigrateInfo", err)
|
ctx.Handle(500, "CleanUpMigrateInfo", err)
|
||||||
return
|
return
|
||||||
} else if err = models.DeleteMirrorByRepoID(ctx.Repo.Repository.ID); err != nil {
|
} else if err = models.DeleteMirrorByRepoID(ctx.Repo.Repository.ID); err != nil {
|
||||||
|
|
|
@ -6,7 +6,7 @@
|
||||||
<span class="mega-octicon octicon-book"></span>
|
<span class="mega-octicon octicon-book"></span>
|
||||||
<h2>{{.i18n.Tr "repo.wiki.welcome"}}</h2>
|
<h2>{{.i18n.Tr "repo.wiki.welcome"}}</h2>
|
||||||
<p>{{.i18n.Tr "repo.wiki.welcome_desc"}}</p>
|
<p>{{.i18n.Tr "repo.wiki.welcome_desc"}}</p>
|
||||||
{{if .IsRepositoryWriter}}
|
{{if and .IsRepositoryWriter (not .IsRepositoryMirror)}}
|
||||||
<a class="ui green button" href="{{.RepoLink}}/wiki/_new">{{.i18n.Tr "repo.wiki.create_first_page"}}</a>
|
<a class="ui green button" href="{{.RepoLink}}/wiki/_new">{{.i18n.Tr "repo.wiki.create_first_page"}}</a>
|
||||||
{{end}}
|
{{end}}
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -46,7 +46,7 @@
|
||||||
</div>
|
</div>
|
||||||
<div class="ui dividing header">
|
<div class="ui dividing header">
|
||||||
{{.title}}
|
{{.title}}
|
||||||
{{if .IsRepositoryWriter}}
|
{{if and .IsRepositoryWriter (not .IsRepositoryMirror)}}
|
||||||
<div class="ui right">
|
<div class="ui right">
|
||||||
<a class="ui small button" href="{{.RepoLink}}/wiki/{{.PageURL}}/_edit">{{.i18n.Tr "repo.wiki.edit_page_button"}}</a>
|
<a class="ui small button" href="{{.RepoLink}}/wiki/{{.PageURL}}/_edit">{{.i18n.Tr "repo.wiki.edit_page_button"}}</a>
|
||||||
<a class="ui green small button" href="{{.RepoLink}}/wiki/_new">{{.i18n.Tr "repo.wiki.new_page_button"}}</a>
|
<a class="ui green small button" href="{{.RepoLink}}/wiki/_new">{{.i18n.Tr "repo.wiki.new_page_button"}}</a>
|
||||||
|
|
Loading…
Reference in New Issue
Block a user