Skip to content

Commit dd08853

Browse files
lunnylunny
authored
Dump github/gitlab/gitea repository data to a local directory and restore to gitea (#12244)
* Dump github/gitlab repository data to a local directory * Fix lint * Adjust directory structure * Allow migration special units * Allow migration ignore release assets * Fix lint * Add restore repository * stage the changes * Merge * Fix lint * Update the interface * Add some restore methods * Finish restore * Add comments * Fix restore * Add a token flag * Fix bug * Fix test * Fix test * Fix bug * Fix bug * Fix lint * Fix restore * refactor downloader * fmt * Fix bug isEnd detection on getIssues * Refactor maxPerPage * Remove unused codes * Remove unused codes * Fix bug * Fix restore * Fix dump * Uploader should not depend downloader * use release attachment name but not id * Fix restore bug * Fix lint * Fix restore bug * Add a method of DownloadFunc for base.Release to make uploader not depend on downloader * fix Release yml marshal * Fix trace information * Fix bug when dump & restore * Save relative path on yml file * Fix bug * Use relative path * Update docs * Use git service string but not int * Recognize clone addr to service type
1 parent 212fa34 commit dd08853

29 files changed

+1484
-225
lines changed

‎cmd/dump_repo.go‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
// Copyright 2020 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package cmd
6+
7+
import (
8+
"context"
9+
"errors"
10+
"strings"
11+
12+
"code.gitea.io/gitea/modules/convert"
13+
"code.gitea.io/gitea/modules/log"
14+
"code.gitea.io/gitea/modules/migrations"
15+
"code.gitea.io/gitea/modules/migrations/base"
16+
"code.gitea.io/gitea/modules/setting"
17+
"code.gitea.io/gitea/modules/structs"
18+
19+
"github.com/urfave/cli"
20+
)
21+
22+
// CmdDumpRepository represents the available dump repository sub-command.
23+
varCmdDumpRepository= cli.Command{
24+
Name: "dump-repo",
25+
Usage: "Dump the repository from git/github/gitea/gitlab",
26+
Description: "This is a command for dumping the repository data.",
27+
Action: runDumpRepository,
28+
Flags: []cli.Flag{
29+
cli.StringFlag{
30+
Name: "git_service",
31+
Value: "",
32+
Usage: "Git service, git, github, gitea, gitlab. If clone_addr could be recognized, this could be ignored.",
33+
},
34+
cli.StringFlag{
35+
Name: "repo_dir, r",
36+
Value: "./data",
37+
Usage: "Repository dir path to store the data",
38+
},
39+
cli.StringFlag{
40+
Name: "clone_addr",
41+
Value: "",
42+
Usage: "The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL",
43+
},
44+
cli.StringFlag{
45+
Name: "auth_username",
46+
Value: "",
47+
Usage: "The username to visit the clone_addr",
48+
},
49+
cli.StringFlag{
50+
Name: "auth_password",
51+
Value: "",
52+
Usage: "The password to visit the clone_addr",
53+
},
54+
cli.StringFlag{
55+
Name: "auth_token",
56+
Value: "",
57+
Usage: "The personal token to visit the clone_addr",
58+
},
59+
cli.StringFlag{
60+
Name: "owner_name",
61+
Value: "",
62+
Usage: "The data will be stored on a directory with owner name if not empty",
63+
},
64+
cli.StringFlag{
65+
Name: "repo_name",
66+
Value: "",
67+
Usage: "The data will be stored on a directory with repository name if not empty",
68+
},
69+
cli.StringFlag{
70+
Name: "units",
71+
Value: "",
72+
Usage: `Which items will be migrated, one or more units should be separated as comma.
73+
wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`,
74+
},
75+
},
76+
}
77+
78+
funcrunDumpRepository(ctx*cli.Context) error{
79+
iferr:=initDB(); err!=nil{
80+
returnerr
81+
}
82+
83+
log.Trace("AppPath: %s", setting.AppPath)
84+
log.Trace("AppWorkPath: %s", setting.AppWorkPath)
85+
log.Trace("Custom path: %s", setting.CustomPath)
86+
log.Trace("Log path: %s", setting.LogRootPath)
87+
setting.InitDBConfig()
88+
89+
var (
90+
serviceType structs.GitServiceType
91+
cloneAddr=ctx.String("clone_addr")
92+
serviceStr=ctx.String("git_service")
93+
)
94+
95+
ifstrings.HasPrefix(strings.ToLower(cloneAddr), "https://github.com/"){
96+
serviceStr="github"
97+
} elseifstrings.HasPrefix(strings.ToLower(cloneAddr), "https://gitlab.com/"){
98+
serviceStr="gitlab"
99+
} elseifstrings.HasPrefix(strings.ToLower(cloneAddr), "https://gitea.com/"){
100+
serviceStr="gitea"
101+
}
102+
ifserviceStr==""{
103+
returnerrors.New("git_service missed or clone_addr cannot be recognized")
104+
}
105+
serviceType=convert.ToGitServiceType(serviceStr)
106+
107+
varopts= base.MigrateOptions{
108+
GitServiceType: serviceType,
109+
CloneAddr: cloneAddr,
110+
AuthUsername: ctx.String("auth_username"),
111+
AuthPassword: ctx.String("auth_password"),
112+
AuthToken: ctx.String("auth_token"),
113+
RepoName: ctx.String("repo_name"),
114+
}
115+
116+
iflen(ctx.String("units")) ==0{
117+
opts.Wiki=true
118+
opts.Issues=true
119+
opts.Milestones=true
120+
opts.Labels=true
121+
opts.Releases=true
122+
opts.Comments=true
123+
opts.PullRequests=true
124+
opts.ReleaseAssets=true
125+
} else{
126+
units:=strings.Split(ctx.String("units"), ",")
127+
for_, unit:=rangeunits{
128+
switchstrings.ToLower(unit){
129+
case"wiki":
130+
opts.Wiki=true
131+
case"issues":
132+
opts.Issues=true
133+
case"milestones":
134+
opts.Milestones=true
135+
case"labels":
136+
opts.Labels=true
137+
case"releases":
138+
opts.Releases=true
139+
case"release_assets":
140+
opts.ReleaseAssets=true
141+
case"comments":
142+
opts.Comments=true
143+
case"pull_requests":
144+
opts.PullRequests=true
145+
}
146+
}
147+
}
148+
149+
iferr:=migrations.DumpRepository(
150+
context.Background(),
151+
ctx.String("repo_dir"),
152+
ctx.String("owner_name"),
153+
opts,
154+
); err!=nil{
155+
log.Fatal("Failed to dump repository: %v", err)
156+
returnerr
157+
}
158+
159+
log.Trace("Dump finished!!!")
160+
161+
returnnil
162+
}

‎cmd/restore_repo.go‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
// Copyright 2020 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package cmd
6+
7+
import (
8+
"context"
9+
"strings"
10+
11+
"code.gitea.io/gitea/modules/log"
12+
"code.gitea.io/gitea/modules/migrations"
13+
"code.gitea.io/gitea/modules/migrations/base"
14+
"code.gitea.io/gitea/modules/setting"
15+
"code.gitea.io/gitea/modules/storage"
16+
pull_service "code.gitea.io/gitea/services/pull"
17+
18+
"github.com/urfave/cli"
19+
)
20+
21+
// CmdRestoreRepository represents the available restore a repository sub-command.
22+
varCmdRestoreRepository= cli.Command{
23+
Name: "restore-repo",
24+
Usage: "Restore the repository from disk",
25+
Description: "This is a command for restoring the repository data.",
26+
Action: runRestoreRepository,
27+
Flags: []cli.Flag{
28+
cli.StringFlag{
29+
Name: "repo_dir, r",
30+
Value: "./data",
31+
Usage: "Repository dir path to restore from",
32+
},
33+
cli.StringFlag{
34+
Name: "owner_name",
35+
Value: "",
36+
Usage: "Restore destination owner name",
37+
},
38+
cli.StringFlag{
39+
Name: "repo_name",
40+
Value: "",
41+
Usage: "Restore destination repository name",
42+
},
43+
cli.StringFlag{
44+
Name: "units",
45+
Value: "",
46+
Usage: `Which items will be restored, one or more units should be separated as comma.
47+
wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`,
48+
},
49+
},
50+
}
51+
52+
funcrunRestoreRepository(ctx*cli.Context) error{
53+
iferr:=initDB(); err!=nil{
54+
returnerr
55+
}
56+
57+
log.Trace("AppPath: %s", setting.AppPath)
58+
log.Trace("AppWorkPath: %s", setting.AppWorkPath)
59+
log.Trace("Custom path: %s", setting.CustomPath)
60+
log.Trace("Log path: %s", setting.LogRootPath)
61+
setting.InitDBConfig()
62+
63+
iferr:=storage.Init(); err!=nil{
64+
returnerr
65+
}
66+
67+
iferr:=pull_service.Init(); err!=nil{
68+
returnerr
69+
}
70+
71+
varopts= base.MigrateOptions{
72+
RepoName: ctx.String("repo_name"),
73+
}
74+
75+
iflen(ctx.String("units")) ==0{
76+
opts.Wiki=true
77+
opts.Issues=true
78+
opts.Milestones=true
79+
opts.Labels=true
80+
opts.Releases=true
81+
opts.Comments=true
82+
opts.PullRequests=true
83+
opts.ReleaseAssets=true
84+
} else{
85+
units:=strings.Split(ctx.String("units"), ",")
86+
for_, unit:=rangeunits{
87+
switchstrings.ToLower(unit){
88+
case"wiki":
89+
opts.Wiki=true
90+
case"issues":
91+
opts.Issues=true
92+
case"milestones":
93+
opts.Milestones=true
94+
case"labels":
95+
opts.Labels=true
96+
case"releases":
97+
opts.Releases=true
98+
case"release_assets":
99+
opts.ReleaseAssets=true
100+
case"comments":
101+
opts.Comments=true
102+
case"pull_requests":
103+
opts.PullRequests=true
104+
}
105+
}
106+
}
107+
108+
iferr:=migrations.RestoreRepository(
109+
context.Background(),
110+
ctx.String("repo_dir"),
111+
ctx.String("owner_name"),
112+
ctx.String("repo_name"),
113+
); err!=nil{
114+
log.Fatal("Failed to restore repository: %v", err)
115+
returnerr
116+
}
117+
118+
returnnil
119+
}

‎docs/content/doc/usage/command-line.en-us.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,3 +441,28 @@ Manage running server operations:
441441
-`--host value`, `-H value`: Mail server host (defaults to: 127.0.0.1:25)
442442
-`--send-to value`, `-s value`: Email address(es) to send to
443443
-`--subject value`, `-S value`: Subject header of sent emails
444+
445+
### dump-repo
446+
447+
Dump-repo dumps repository data from git/github/gitea/gitlab:
448+
449+
- Options:
450+
-`--git_service service` : Git service, it could be `git`, `github`, `gitea`, `gitlab`, If clone_addr could be recognized, this could be ignored.
451+
-`--repo_dir dir`, `-r dir`: Repository dir path to store the data
452+
-`--clone_addr addr`: The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL. i.e. https://github.com/lunny/tango.git
453+
-`--auth_username lunny`: The username to visit the clone_addr
454+
-`--auth_password <password>`: The password to visit the clone_addr
455+
-`--auth_token <token>`: The personal token to visit the clone_addr
456+
-`--owner_name lunny`: The data will be stored on a directory with owner name if not empty
457+
-`--repo_name tango`: The data will be stored on a directory with repository name if not empty
458+
-`--units <units>`: Which items will be migrated, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.
459+
460+
### restore-repo
461+
462+
Restore-repo restore repository data from disk dir:
463+
464+
- Options:
465+
-`--repo_dir dir`, `-r dir`: Repository dir path to restore from
466+
-`--owner_name lunny`: Restore destination owner name
467+
-`--repo_name tango`: Restore destination repository name
468+
-`--units <units>`: Which items will be restored, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.

‎main.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ arguments - which can alternatively be run by running the subcommand web.`
7272
cmd.Cmdembedded,
7373
cmd.CmdMigrateStorage,
7474
cmd.CmdDocs,
75+
cmd.CmdDumpRepository,
76+
cmd.CmdRestoreRepository,
7577
}
7678
// Now adjust these commands to add our global configuration options
7779

‎models/admin.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,3 +132,16 @@ func DeleteNoticesByIDs(ids []int64) error{
132132
Delete(new(Notice))
133133
returnerr
134134
}
135+
136+
// GetAdminUser returns the first administrator
137+
funcGetAdminUser() (*User, error){
138+
varadminUser
139+
has, err:=x.Where("is_admin=?", true).Get(&admin)
140+
iferr!=nil{
141+
returnnil, err
142+
} elseif!has{
143+
returnnil, ErrUserNotExist{}
144+
}
145+
146+
return&admin, nil
147+
}

‎models/task.go‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -211,10 +211,6 @@ func FinishMigrateTask(task *Task) error{
211211
if_, err:=sess.ID(task.ID).Cols("status", "end_time").Update(task); err!=nil{
212212
returnerr
213213
}
214-
task.Repo.Status=RepositoryReady
215-
if_, err:=sess.ID(task.RepoID).Cols("status").Update(task.Repo); err!=nil{
216-
returnerr
217-
}
218214

219215
returnsess.Commit()
220216
}

‎modules/migrations/base/comment.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@ import "time"
99

1010
// Comment is a standard comment information
1111
typeCommentstruct{
12-
IssueIndexint64
13-
PosterIDint64
14-
PosterNamestring
15-
PosterEmailstring
12+
IssueIndexint64`yaml:"issue_index"`
13+
PosterIDint64`yaml:"poster_id"`
14+
PosterNamestring`yaml:"poster_name"`
15+
PosterEmailstring`yaml:"poster_email"`
1616
Created time.Time
1717
Updated time.Time
1818
Contentstring

‎modules/migrations/base/downloader.go‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,13 @@ package base
77

88
import (
99
"context"
10-
"io"
1110
"time"
1211

1312
"code.gitea.io/gitea/modules/structs"
1413
)
1514

16-
// AssetDownloader downloads an asset (attachment) for a release
17-
typeAssetDownloaderinterface{
18-
GetAsset(relTagstring, relID, idint64) (io.ReadCloser, error)
19-
}
20-
2115
// Downloader downloads the site repo informations
2216
typeDownloaderinterface{
23-
AssetDownloader
2417
SetContext(context.Context)
2518
GetRepoInfo() (*Repository, error)
2619
GetTopics() ([]string, error)

0 commit comments

Comments
(0)