diff --git a/README.md b/README.md index 3924068dfe5cbd7f57404b833d0872b0ed097f7f..96c7702ae795ea83afc6f581ed96ec6ebdedd997 100644 --- a/README.md +++ b/README.md @@ -194,11 +194,12 @@ pbkdf2:sha256:150000$w38eLeRm$ebb5069ba3b4dda39a698bd1d9d7f5f848af3bd93b11e0cde2 --branch ：需要进行跟踪的仓库的分支名称 \ --scm_repo ：被跟踪的上游仓库的仓库名称，--version_control为github的格式：组织/仓库；--version_control为git的格式：仓库URL，不支持需要配置SSH key公私钥对才能克隆的URL \ --scm_branch： 被跟踪的上游仓库的仓库的分支 \ +--scm_commit：指定跟踪的起始commit，选填，默认从当前最新commit开始跟踪 \ --enabled ：是否自动跟踪该仓库 例如： ```shell script -patch-tracking-cli add --server 127.0.0.1:5001 --user admin --password Test@123 --version_control github --repo https://gitee.com/testPatchTrack/testPatch1 --branch master --scm_repo BJMX/testPatch01 --scm_branch test --enabled true +patch-tracking-cli add --server 127.0.0.1:5001 --user admin --password Test@123 --version_control github --repo https://gitee.com/testPatchTrack/testPatch1 --branch master --scm_repo BJMX/testPatch01 --scm_branch test --scm_commit --enabled true ``` ### 指定文件添加 @@ -234,6 +235,13 @@ repo ：需要进行跟踪的仓库URL，不支持需要配置SSH key公私钥 branch ：需要进行跟踪的仓库的分支名称 \ enabled ：是否自动跟踪该仓库 +如果指定起始的commit，yaml文件内容增加一行 +```shell script +scm_commit: +``` + +>scm_commit ：用户指定的跟踪仓库/分支的起始commit + ### 指定目录添加 在指定的目录，例如`test_yaml`下放入多个`xxx.yaml`文件，执行如下命令，记录指定目录下所有yaml文件的跟踪项。 diff --git a/patch_tracking/cli/patch_tracking_cli.py b/patch_tracking/cli/patch_tracking_cli.py index 2c54058d30a2a702898d92cfc61614b47686cf90..2f9c2912bb18dcb5df3e4c20fb71b6fba6e14f3b 100755 --- a/patch_tracking/cli/patch_tracking_cli.py +++ b/patch_tracking/cli/patch_tracking_cli.py @@ -10,7 +10,6 @@ import pandas import requests from requests.auth import HTTPBasicAuth from requests.packages.urllib3.exceptions import InsecureRequestWarning -from requests.packages.urllib3.exceptions import HTTPError requests.packages.urllib3.disable_warnings(InsecureRequestWarning) pandas.set_option('display.max_rows', None) @@ -156,6 +155,7 @@ def params_input_track(params, file_path=None): branch = params['branch'] scm_repo = params['scm_repo'] scm_branch = params['scm_branch'] + scm_commit = params['scm_commit'] version_control = params['version_control'].lower() enabled = params['enabled'].lower() server = params['server'] @@ -185,6 +185,7 @@ def params_input_track(params, file_path=None): 'version_control': version_control, 'scm_repo': scm_repo, 'scm_branch': scm_branch, + 'scm_commit': scm_commit, 'repo': repo, 'branch': branch, 'enabled': enabled @@ -248,6 +249,7 @@ def add(args): 'branch': args.branch, 'scm_repo': args.scm_repo, 'scm_branch': args.scm_branch, + 'scm_commit': args.scm_commit, 'version_control': args.version_control, 'enabled': args.enabled, 'server': args.server, @@ -331,6 +333,8 @@ def file_input_track(file_path, args): value = item.lstrip(k).strip().strip(":").strip().strip("\n") params.update({k: value}) params.update({'server': args.server, 'user': args.user, 'password': args.password}) + if "scm_commit" not in params: + params.update({'scm_commit': None}) ret = params_input_track(params, file_path) if ret[0] == 'success': print('Tracking successfully {} for {}'.format(ret[1], file_path)) @@ -388,7 +392,7 @@ authentication_parser.add_argument('--password', required=True, help='authentica ADD_USAGE = """ patch_tracking_cli add --server SERVER --user USER --password PASSWORD --version_control github --scm_repo SCM_REPO --scm_branch SCM_BRANCH - --repo REPO --branch BRANCH --enabled True + --repo REPO --branch BRANCH --enabled True [--scm_commit SHA] patch_tracking_cli add --server SERVER --user USER --password PASSWORD --file FILE patch_tracking_cli add --server SERVER --user USER --password PASSWORD --dir DIR""" parser_add = subparsers.add_parser( @@ -398,6 +402,7 @@ parser_add.set_defaults(func=add) parser_add.add_argument("--version_control", choices=["github", "git"], help="upstream version control system") parser_add.add_argument("--scm_repo", help="upstream scm repository") parser_add.add_argument("--scm_branch", help="upstream scm branch") +parser_add.add_argument("--scm_commit", help="upstream scm commit sha") parser_add.add_argument("--repo", help="source package repository") parser_add.add_argument("--branch", help="source package branch") parser_add.add_argument("--enabled", choices=["True", "true", "False", "false"], help="whether tracing is enabled") diff --git a/patch_tracking/util/upstream/git.py b/patch_tracking/util/upstream/git.py index d5604ea3d670750635d27d55b01b9eec7e24c946..a7e1485645ad2d19b49670f1844fd12341a34c74 100644 --- a/patch_tracking/util/upstream/git.py +++ b/patch_tracking/util/upstream/git.py @@ -6,6 +6,7 @@ import git import git.exc from flask import current_app from sqlalchemy.exc import SQLAlchemyError +import time from patch_tracking.api.business import update_tracking import patch_tracking.util.upstream.upstream as upstream @@ -60,28 +61,41 @@ class Git(upstream.Upstream): def git_fetch(self, repo): """git fetch""" logging.info("Fetching repo %s.", repo) - try: - if os.path.exists(repo): - repo = git.Repo(repo) - repo.remote().fetch() - logging.info("Fetch repo %s finish.", repo) - else: + if os.path.exists(repo): + repo = git.Repo(repo) + count = 10 + while count > 0: + try: + repo.remote().fetch() + logging.info("Fetch repo %s finish.", repo) + break + except git.exc.GitError as err: + logging.warning("Fetching repo %s failed. Error: %s", repo, err) + count -= 1 + time.sleep(1) + if count == 0: + logging.error("Fetching repo %s failed.", repo) + return False + else: + try: self.git_clone() logging.info("Cloned repo done %s.", repo) - return True - except git.exc.GitError as err: - logging.error("Fetching repo %s failed. Error: %s", repo, err) - return False + except git.exc.GitError as err: + logging.error("Fetching repo %s failed. Error: %s", repo, err) + return False + return True def git_latest_sha(self): """ get latest commit id """ repo_path = os.path.join(self.base_path, self.repo_dir_name) - logging.info("Getting latest commit id of repo: %s branch: %s .", repo_path, self.track.scm_branch) try: repo = git.Repo(repo_path) sha = repo.commit(self.track.scm_branch).hexsha + logging.info( + "Getting latest commit id of repo: %s branch: %s sha: %s .", repo_path, self.track.scm_branch, sha + ) return sha except git.exc.GitError as err: logging.error( @@ -89,17 +103,53 @@ class Git(upstream.Upstream): ) return False - def get_commit_list(self, repo, start_commit): + def get_all_commit_list(self): + """get all commits of a repo/branch""" + repo_path = os.path.join(self.base_path, self.repo_dir_name) + logging.info("Getting all commit id of repo: %s.", repo_path) + try: + repo = git.Repo(repo_path) + repo.git.symbolic_ref("HEAD", "refs/heads/" + self.track.scm_branch) + all_commit_list = [str(item) for item in repo.iter_commits()] + logging.info("Get all commit id of repo: %s branch: %s.", repo_path, self.track.scm_branch) + return all_commit_list + except git.exc.GitError as err: + logging.error("Get all commit id of repo: %s failed. Error: %s", repo_path, err) + return False + + def check_commit_exist(self, commit): + """check commit exist""" + repo_path = os.path.join(self.base_path, self.repo_dir_name) + branch = self.track.scm_branch + repo = git.Repo(repo_path) + try: + ret = repo.git.branch("--contains", commit) + if ret: + ret = [r.split(" ")[-1] for r in ret.split("\n")] + if branch in ret: + return True + return False + except git.exc.GitCommandError as err: + logging.error("Error: %s.", err) + return False + + def get_commit_list(self, start_commit, latest_commit): """get commit list""" commit_list = list() + if start_commit == latest_commit: + return commit_list fetch_ret = self.git_fetch(os.path.join(self.base_path, self.repo_dir_name)) if fetch_ret: - repo = git.Repo(repo) - all_commit_list = list(repo.iter_commits()) - commit_list = list() - for item in all_commit_list: - if str(item) != start_commit: - commit_list.append(str(item)) + if not self.check_commit_exist(start_commit): + logging.error( + "Commit sha: %s not exist in repo: %s branch: %s.", start_commit, self.track.scm_repo, + self.track.scm_branch + ) + return commit_list + all_commit_list = self.get_all_commit_list() + for commit in all_commit_list: + if commit != start_commit: + commit_list.append(commit) else: break commit_list.append(start_commit) @@ -162,7 +212,7 @@ class Git(upstream.Upstream): return None - commit_list = self.get_commit_list(repo, self.track.scm_commit) + commit_list = self.get_commit_list(self.track.scm_commit, latest_commit) if not commit_list: return None diff --git a/patch_tracking/util/upstream/github.py b/patch_tracking/util/upstream/github.py index 28bf8fb2ba027ef1d15e83e7108a3a6e0e7105b8..fc971ee9f6fd8b42ad732ee279f741551083c5c7 100644 --- a/patch_tracking/util/upstream/github.py +++ b/patch_tracking/util/upstream/github.py @@ -62,7 +62,7 @@ class GitHub(upstream.Upstream): 'Accept': 'application/json' } - def api_request(self, url): + def api_request(self, url, params=None): """ request GitHub API """ @@ -70,7 +70,7 @@ class GitHub(upstream.Upstream): count = 30 while count > 0: try: - response = requests.get(url, headers=self.headers) + response = requests.get(url, headers=self.headers, params=params) return response except exceptions.ConnectionError as err: logger.warning(err) @@ -111,6 +111,20 @@ class GitHub(upstream.Upstream): ret_dict['api_ret'] = 'fail to connect github by api.' return ret_dict + def get_all_commit_list(self): + """ + get latest 100 commit + """ + url = '/'.join(['https://api.github.com/repos', self.track.scm_repo, 'commits']) + params = {"sha": self.track.scm_branch, "page": 0, "per_page": 100} + all_commits = list() + ret = self.api_request(url, params=params) + for item in ret.json(): + all_commits.append(item) + + logger.debug('[Patch Tracking] Successful get all commits.') + return all_commits + def get_latest_commit_id(self): """ get latest commit_ID, commit_message, commit_date @@ -161,6 +175,15 @@ class GitHub(upstream.Upstream): """ get patch list """ + all_commits_info = self.get_all_commit_list() + all_commits = [item["sha"] for item in all_commits_info] + + if self.track.scm_commit not in all_commits: + logger.error( + '[Patch Tracking] Scm repo commit : %s not found in latest 100 commits of scm_repo: %s scm_branch: %s.', + self.track.scm_commit, self.track.scm_repo, self.track.scm_branch + ) + return None commit_list = list() status, result = self.get_latest_commit_id() if status != 'success':