diff --git a/patch_tracking/api/tracking.py b/patch_tracking/api/tracking.py index 4a8a24cf78e244656c4a4b587eb2b00b9ebf4268..c34edf83bf5503c477da27272854706bee105775 100644 --- a/patch_tracking/api/tracking.py +++ b/patch_tracking/api/tracking.py @@ -22,10 +22,14 @@ def delete(): input_params = request.args keys = list(input_params.keys()) + param_error = False if not keys or "repo" not in keys: - return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR) + param_error = True if len(set(keys) - {"repo", "branch"}) != 0: + param_error = True + + if param_error: return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR) try: @@ -35,19 +39,16 @@ def delete(): delete_tracking(input_params['repo'], input_params['branch']) logger.info('Delete tracking repo: %s, branch: %s', input_params['repo'], input_params['branch']) return ResponseCode.ret_message(code=ResponseCode.SUCCESS) - else: - logger.info( - 'Delete tracking repo: %s, branch: %s not found.', input_params['repo'], input_params['branch'] - ) - return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_NOT_FOUND) - else: - if Tracking.query.filter(Tracking.repo == input_params['repo']).first(): - delete_tracking(input_params['repo']) - logger.info('Delete tracking repo: %s', input_params['repo']) - return ResponseCode.ret_message(code=ResponseCode.SUCCESS) - else: - logger.info('Delete tracking repo: %s not found.', input_params['repo']) - return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_NOT_FOUND) + + logger.info('Delete tracking repo: %s, branch: %s not found.', input_params['repo'], input_params['branch']) + return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_NOT_FOUND) + if Tracking.query.filter(Tracking.repo == input_params['repo']).first(): + delete_tracking(input_params['repo']) + logger.info('Delete tracking repo: %s', input_params['repo']) + return ResponseCode.ret_message(code=ResponseCode.SUCCESS) + + logger.info('Delete tracking repo: %s not found.', input_params['repo']) + return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_NOT_FOUND) except SQLAlchemyError as err: return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_ERROR, data=err) @@ -98,7 +99,7 @@ def post(): if len(required_params) > 1 or (len(required_params) == 1 and required_params[0] != 'scm_commit'): return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR) - if data['version_control'] != 'github': + if data['version_control'] not in ["github", "git"]: return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR) track = Tracking.query.filter_by(repo=data['repo'], branch=data['branch']).first() diff --git a/patch_tracking/app.py b/patch_tracking/app.py index 84aaad191ea4574ef5da3b4c8002d35843149866..479a0ec2e955b20d1836357acd1d18bf2529bc4e 100644 --- a/patch_tracking/app.py +++ b/patch_tracking/app.py @@ -9,7 +9,6 @@ from patch_tracking.api.issue import issue from patch_tracking.api.tracking import tracking from patch_tracking.database import db from patch_tracking.task import task -from patch_tracking.util import github_api, gitee_api logging.config.fileConfig('logging.conf', disable_existing_loggers=False) @@ -17,22 +16,6 @@ app = Flask(__name__) logger = logging.getLogger(__name__) -def check_token(): - """ check gitee/github token """ - gitee_token = app.config['GITEE_ACCESS_TOKEN'] - github_token = app.config['GITHUB_ACCESS_TOKEN'] - - github_ret = github_api.get_user_info(github_token) - if not github_ret[0]: - logger.error(github_ret[1]) - sys.exit(1) - - gitee_ret = gitee_api.get_user_info(gitee_token) - if not gitee_ret[0]: - logger.error(gitee_ret[1]) - sys.exit(1) - - def check_listen(listen_param): """ check LISTEN """ check_ret = True @@ -56,7 +39,7 @@ def check_settings_conf(): check settings.conf """ setting_error = False - required_settings = ['LISTEN', 'GITHUB_ACCESS_TOKEN', 'GITEE_ACCESS_TOKEN', 'SCAN_DB_INTERVAL', 'USER', 'PASSWORD'] + required_settings = ['LISTEN', 'GITEE_ACCESS_TOKEN', 'SCAN_DB_INTERVAL', 'USER', 'PASSWORD'] for setting in required_settings: if setting in app.config: if app.config[setting] == "": @@ -94,11 +77,10 @@ except (SyntaxError, NameError): logger.error('settings.conf content format error.') sys.exit(1) -check_token() - app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///db.sqlite?check_same_thread=False&timeout=30' app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False app.config['SCHEDULER_EXECUTORS'] = {'default': {'type': 'threadpool', 'max_workers': 100}} +app.config['GIT_BASE_PATH'] = "GIT_REPO_PATH" app.register_blueprint(issue, url_prefix="/issue") app.register_blueprint(tracking, url_prefix="/tracking") diff --git a/patch_tracking/task/task.py b/patch_tracking/task/task.py index 27408b437a578a6e49436888e86f882b8ae31ebd..3465645f01d8b71f0b0cf3f448bab581133abf50 100644 --- a/patch_tracking/task/task.py +++ b/patch_tracking/task/task.py @@ -5,16 +5,48 @@ import datetime import logging from patch_tracking.task import scheduler from patch_tracking.database.models import Tracking -from patch_tracking.util.github_api import GitHubApi +from patch_tracking.util.upstream.github import GitHub +from patch_tracking.util.upstream.git import Git from patch_tracking.api.business import update_tracking +from patch_tracking.util.upstream.github import get_user_info as github_get_user_info +from patch_tracking.util.gitee_api import get_user_info as gitee_get_user_info logger = logging.getLogger(__name__) +def check_token(app): + """ check gitee/github token """ + gitee_token = app.config['GITEE_ACCESS_TOKEN'] + github_token = app.config['GITHUB_ACCESS_TOKEN'] + token_error = False + try: + github_ret = github_get_user_info(github_token) + if not github_ret[0]: + logger.error(github_ret[1]) + logger.error('github token is bad credentials.') + token_error = True + except UnicodeEncodeError: + logger.error('github token is bad credentials.') + token_error = True + + gitee_ret = gitee_get_user_info(gitee_token) + if not gitee_ret[0]: + logger.error(gitee_ret[1]) + logger.error('gitee token is bad credentials.') + token_error = True + + if token_error: + return False + return True + + def init(app): """ scheduler jobs init """ + if not check_token(app): + logger.error('[Patch Tracking] Token Error. Stop tracking task.') + return scan_db_interval = app.config['SCAN_DB_INTERVAL'] scheduler.init_app(app) scheduler.add_job( @@ -55,13 +87,35 @@ def check_empty_commit_id(flask_app): """ with flask_app.app_context(): new_track = get_track_from_db() - github_api = GitHubApi() for item in new_track: if item.scm_commit: continue - status, result = github_api.get_latest_commit(item.scm_repo, item.scm_branch) - if status == 'success': - commit_id = result['latest_commit'] + if item.version_control == "github": + github_api = GitHub(item) + status, result = github_api.get_latest_commit_id() + if status == 'success': + commit_id = result['latest_commit'] + data = { + 'version_control': item.version_control, + 'repo': item.repo, + 'branch': item.branch, + 'enabled': item.enabled, + 'scm_commit': commit_id, + 'scm_branch': item.scm_branch, + 'scm_repo': item.scm_repo + } + update_tracking(data) + else: + logger.error( + 'Check empty CommitID: Fail to get latest commit id of scm_repo: %s scm_branch: %s. \ + Return val: %s', item.scm_repo, item.scm_branch, result + ) + elif item.version_control == "git": + git_api = Git(item) + commit_id = git_api.git_latest_sha() + if not commit_id: + return None + data = { 'version_control': item.version_control, 'repo': item.repo, @@ -78,6 +132,8 @@ def check_empty_commit_id(flask_app): item.scm_repo, item.scm_branch, result ) + return None + def get_track_from_db(): """ diff --git a/patch_tracking/task/task_apscheduler.py b/patch_tracking/task/task_apscheduler.py index 63d0f4228203c698670578452c9e835baf4297f9..be6e49456c796060b433ea70186ae8d8901a7527 100644 --- a/patch_tracking/task/task_apscheduler.py +++ b/patch_tracking/task/task_apscheduler.py @@ -9,11 +9,11 @@ import random from sqlalchemy.exc import SQLAlchemyError from patch_tracking.util.gitee_api import create_branch, upload_patch, create_gitee_issue from patch_tracking.util.gitee_api import create_pull_request, get_path_content, upload_spec, create_spec -from patch_tracking.util.github_api import GitHubApi from patch_tracking.database.models import Tracking from patch_tracking.api.business import update_tracking, create_issue from patch_tracking.task import scheduler from patch_tracking.util.spec import Spec +from patch_tracking.util.upstream import Factory logger = logging.getLogger(__name__) @@ -25,9 +25,10 @@ def upload_patch_to_gitee(track): cur_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S%f") with scheduler.app.app_context(): logger.info('[Patch Tracking %s] track.scm_commit_id: %s.', cur_time, track.scm_commit) - patch = get_scm_patch(track) + git_api = Factory.create(track) + patch = get_scm_patch(track, git_api) if patch: - issue = create_patch_issue_pr(patch, cur_time) + issue = create_patch_issue_pr(patch, cur_time, git_api) if issue: create_issue_db(issue) else: @@ -36,50 +37,12 @@ def upload_patch_to_gitee(track): logger.debug('[Patch Tracking %s] No new commit.', cur_time) -def get_all_commit_info(scm_repo, db_commit, latest_commit): - """ - get all commit information between to commits - """ - commit_list = list() - github_api = GitHubApi() - - while db_commit != latest_commit: - status, result = github_api.get_commit_info(scm_repo, latest_commit) - logger.debug('get_commit_info: %s %s', status, result) - if status == 'success': - if 'parent' in result: - ret = github_api.get_patch(scm_repo, latest_commit, latest_commit) - logger.debug('get patch api ret: %s', ret) - if ret['status'] == 'success': - result['patch_content'] = ret['api_ret'] - # inverted insert commit_list - commit_list.insert(0, result) - else: - logger.error('Get scm: %s commit: %s patch failed. Result: %s', scm_repo, latest_commit, result) - - latest_commit = result['parent'] - else: - logger.info( - '[Patch Tracking] Successful get scm commit from %s to %s ID/message/time/patch.', db_commit, - latest_commit - ) - break - else: - logger.error( - '[Patch Tracking] Get scm: %s commit: %s ID/message/time failed. Result: %s', scm_repo, latest_commit, - result - ) - - return commit_list - - -def get_scm_patch(track): +def get_scm_patch(track, git_api): """ Traverse the Tracking data table to get the patch file of enabled warehouse. Different warehouse has different acquisition methods :return: """ - github_api = GitHubApi() scm_dict = dict( scm_repo=track.scm_repo, scm_branch=track.scm_branch, @@ -89,75 +52,40 @@ def get_scm_patch(track): branch=track.branch, version_control=track.version_control ) - status, result = github_api.get_latest_commit(scm_dict['scm_repo'], scm_dict['scm_branch']) - logger.debug( - 'repo: %s branch: %s. get_latest_commit: %s %s', scm_dict['scm_repo'], scm_dict['scm_branch'], status, result - ) - if status == 'success': - commit_id = result['latest_commit'] - if not scm_dict['scm_commit']: - data = { - 'version_control': scm_dict['version_control'], - 'repo': scm_dict['repo'], - 'branch': scm_dict['branch'], - 'enabled': scm_dict['enabled'], - 'scm_commit': commit_id, - 'scm_branch': scm_dict['scm_branch'], - 'scm_repo': scm_dict['scm_repo'] - } - update_tracking(data) - logger.info( - '[Patch Tracking] Scm_repo: %s Scm_branch: %s.Get latest commit ID: %s From commit ID: None.', - scm_dict['scm_repo'], scm_dict['scm_branch'], result['latest_commit'] - ) - else: - if commit_id != scm_dict['scm_commit']: - commit_list = get_all_commit_info(scm_dict['scm_repo'], scm_dict['scm_commit'], commit_id) - scm_dict['commit_list'] = commit_list - return scm_dict - logger.info( - '[Patch Tracking] Scm_repo: %s Scm_branch: %s.Get latest commit ID: %s From commit ID: %s. ' - 'Nothing need to do.', scm_dict['scm_repo'], scm_dict['scm_branch'], commit_id, scm_dict['scm_commit'] - ) - else: - logger.error( - '[Patch Tracking] Fail to get latest commit id of scm_repo: %s scm_branch: %s. Return val: %s', - scm_dict['scm_repo'], scm_dict['scm_branch'], result - ) + commit_list = git_api.get_scm_patch() + if commit_list: + scm_dict['commit_list'] = commit_list + return scm_dict + logger.info('repo: %s branch: %s. get_latest_commit is None.', scm_dict['scm_repo'], scm_dict['scm_branch']) + return None -def create_patch_issue_pr(patch, cur_time): +def create_patch_issue_pr(patch, cur_time, git_api): """ Create temporary branches, submit files, and create PR and issue :return: """ issue_dict = dict() - if not patch: - return None - + gitee_repo = patch["repo"].replace("https://gitee.com/", "") issue_dict['repo'] = patch['repo'] issue_dict['branch'] = patch['branch'] new_branch = 'patch-tracking/' + cur_time - result = create_branch(patch['repo'], patch['branch'], new_branch) + result = create_branch(gitee_repo, patch['branch'], new_branch) if result == 'success': logger.info('[Patch Tracking %s] Successful create branch: %s', cur_time, new_branch) else: logger.error('[Patch Tracking %s] Fail to create branch: %s', cur_time, new_branch) return None patch_lst = list() - issue_table = "| Commit | Datetime | Message |\n| ------ | ------ | ------ |\n" for latest_commit in patch['commit_list']: scm_commit_url = '/'.join(['https://github.com', patch['scm_repo'], 'commit', latest_commit['commit_id']]) latest_commit['message'] = latest_commit['message'].replace("\r", "").replace("\n", "
") - issue_table += '| [{}]({}) | {} | {} |'.format( - latest_commit['commit_id'][0:7], scm_commit_url, latest_commit['time'], latest_commit['message'] - ) + '\n' patch_file_content = latest_commit['patch_content'] post_data = { - 'repo': patch['repo'], + 'repo': gitee_repo, 'branch': new_branch, 'latest_commit_id': latest_commit['commit_id'], 'patch_file_content': str(patch_file_content), @@ -183,25 +111,22 @@ def create_patch_issue_pr(patch, cur_time): logger.error('[Patch Tracking %s] Fail to upload spec file. Result: %s', cur_time, result) return None + issue_table = git_api.issue_table(patch['commit_list']) logger.debug(issue_table) - result = create_gitee_issue(patch['repo'], patch['branch'], issue_table, cur_time) + result = create_gitee_issue(gitee_repo, patch['branch'], issue_table, cur_time) if result[0] == 'success': issue_num = result[1] logger.info('[Patch Tracking %s] Successfully create issue: %s', cur_time, issue_num) retry_count = 10 while retry_count > 0: - ret = create_pull_request(patch['repo'], patch['branch'], new_branch, issue_num, cur_time) + ret = create_pull_request(gitee_repo, patch['branch'], new_branch, issue_num, cur_time) if ret == 'success': logger.info('[Patch Tracking %s] Successfully create PR of issue: %s.', cur_time, issue_num) break - else: - logger.warning( - '[Patch Tracking %s] Fail to create PR of issue: %s. Result: %s', cur_time, issue_num, ret - ) - retry_count -= 1 - time.sleep(random.random() * 2) - continue + logger.warning('[Patch Tracking %s] Fail to create PR of issue: %s. Result: %s', cur_time, issue_num, ret) + retry_count -= 1 + time.sleep(random.random() * 5) if retry_count == 0: logger.error('[Patch Tracking %s] Fail to create PR of issue: %s.', cur_time, issue_num) return None @@ -233,25 +158,25 @@ def upload_spec_to_repo(patch, patch_lst, cur_time): update and upload spec file """ new_branch = 'patch-tracking/' + cur_time - - _, repo_name = patch['repo'].split('/') + gitee_repo = patch["repo"].replace("https://gitee.com/", "") + _, repo_name = gitee_repo.split('/') spec_file = repo_name + '.spec' patch_file_lst = [patch + '.patch' for patch in patch_lst] - log_title = "{} patch-tracking".format(cur_time) + log_title = "{} patch-tracking".format(datetime.datetime.now().strftime("%a %b %d %Y")) log_content = "append patch file of upstream repository from <{}> to <{}>".format(patch_lst[0], patch_lst[-1]) - ret = get_path_content(patch['repo'], patch['branch'], spec_file) + ret = get_path_content(gitee_repo, patch['branch'], spec_file) if 'content' in ret: spec_content = str(base64.b64decode(ret['content']), encoding='utf-8') spec_sha = ret['sha'] new_spec = modify_spec(log_title, log_content, patch_file_lst, spec_content) - result = update_spec_to_repo(patch['repo'], new_branch, cur_time, new_spec, spec_sha) + result = update_spec_to_repo(gitee_repo, new_branch, cur_time, new_spec, spec_sha) else: spec_content = '' new_spec = modify_spec(log_title, log_content, patch_file_lst, spec_content) - result = create_spec_to_repo(patch['repo'], new_branch, cur_time, new_spec) + result = create_spec_to_repo(gitee_repo, new_branch, cur_time, new_spec) return result diff --git a/patch_tracking/util/gitee_api.py b/patch_tracking/util/gitee_api.py index 4a0af0025560298fd529f9df81da43a8e363e208..07b479eb02243a34fd82fee0affea8a4d239a51b 100644 --- a/patch_tracking/util/gitee_api.py +++ b/patch_tracking/util/gitee_api.py @@ -72,7 +72,7 @@ def get_user_info(token): if ret: if ret_info.status_code == 200: return True, ret_info.text - return False, ret_info.json() + return False, ret_info.text return False, ret_info diff --git a/patch_tracking/util/github_api.py b/patch_tracking/util/github_api.py deleted file mode 100644 index 214d290f99931f6f210e473e2bbb295e8a7fb84c..0000000000000000000000000000000000000000 --- a/patch_tracking/util/github_api.py +++ /dev/null @@ -1,156 +0,0 @@ -""" -functionality of invoking GitHub API -""" -import time -import logging -import requests -from requests import exceptions -from flask import current_app - -logger = logging.getLogger(__name__) - - -def get_user_info(token): - """ - get user info - """ - url = "https://api.github.com/user" - count = 30 - token = 'token ' + token - headers = { - 'User-Agent': 'Mozilla/5.0', - 'Authorization': token, - 'Content-Type': 'application/json', - 'Connection': 'close', - 'method': 'GET', - 'Accept': 'application/json' - } - while count > 0: - try: - ret = requests.get(url, headers=headers) - if ret.status_code == 200: - return True, ret.text - return False, ret.json() - except exceptions.ConnectionError as err: - logger.warning(err) - time.sleep(10) - count -= 1 - continue - except UnicodeEncodeError: - return False, 'github token is bad credentials.' - except IOError as error: - return False, error - if count == 0: - logger.error('Fail to connnect to github: %s after retry 30 times.', url) - return False, 'connect error' - - -class GitHubApi(object): - """ - Encapsulates GitHub functionality - """ - def __init__(self): - github_token = current_app.config['GITHUB_ACCESS_TOKEN'] - token = 'token ' + github_token - self.headers = { - 'User-Agent': 'Mozilla/5.0', - 'Authorization': token, - 'Content-Type': 'application/json', - 'Connection': 'close', - 'method': 'GET', - 'Accept': 'application/json' - } - - def api_request(self, url): - """ - request GitHub API - """ - logger.debug("Connect url: %s", url) - count = 30 - while count > 0: - try: - response = requests.get(url, headers=self.headers) - return response - except exceptions.ConnectionError as err: - logger.warning(err) - time.sleep(10) - count -= 1 - continue - except IOError as err: - logger.error(err) - return False - if count == 0: - logger.error('Fail to connnect to github: %s after retry 30 times.', url) - return False - - def get_commit_info(self, repo_url, commit_id): - """ - get commit info - """ - res_dict = dict() - api_url = 'https://api.github.com/repos' - url = '/'.join([api_url, repo_url, 'commits', commit_id]) - ret = self.api_request(url) - if ret: - if ret.status_code == 200: - res_dict['commit_id'] = commit_id - res_dict['message'] = ret.json()['commit']['message'] - res_dict['time'] = ret.json()['commit']['author']['date'] - if 'parents' in ret.json() and ret.json()['parents']: - res_dict['parent'] = ret.json()['parents'][0]['sha'] - return 'success', res_dict - - logger.error('%s failed. Return val: %s', url, ret) - return 'error', ret.json() - return 'error', 'connect error' - - def get_latest_commit(self, repo_url, branch): - """ - get latest commit_ID, commit_message, commit_date - :param repo_url: - :param branch: - :return: res_dict - """ - api_url = 'https://api.github.com/repos' - url = '/'.join([api_url, repo_url, 'branches', branch]) - ret = self.api_request(url) - res_dict = dict() - if ret: - if ret.status_code == 200: - res_dict['latest_commit'] = ret.json()['commit']['sha'] - res_dict['message'] = ret.json()['commit']['commit']['message'] - res_dict['time'] = ret.json()['commit']['commit']['committer']['date'] - return 'success', res_dict - - logger.error('%s failed. Return val: %s', url, ret) - return 'error', ret.json() - - return 'error', 'connect error' - - def get_patch(self, repo_url, scm_commit, last_commit): - """ - get patch - """ - api_url = 'https://github.com' - if scm_commit != last_commit: - commit = scm_commit + '...' + last_commit + '.diff' - else: - commit = scm_commit + '^...' + scm_commit + '.diff' - ret_dict = dict() - - url = '/'.join([api_url, repo_url, 'compare', commit]) - ret = self.api_request(url) - if ret: - if ret.status_code == 200: - patch_content = ret.text - ret_dict['status'] = 'success' - ret_dict['api_ret'] = patch_content - else: - logger.error('%s failed. Return val: %s', url, ret) - ret_dict['status'] = 'error' - ret_dict['api_ret'] = ret.text - else: - ret_dict['status'] = 'error' - ret_dict['api_ret'] = 'fail to connect github by api.' - - return ret_dict diff --git a/patch_tracking/util/upstream/git.py b/patch_tracking/util/upstream/git.py index 05206924b65044eb712125e80ed0cb79bd09927f..d5604ea3d670750635d27d55b01b9eec7e24c946 100644 --- a/patch_tracking/util/upstream/git.py +++ b/patch_tracking/util/upstream/git.py @@ -2,10 +2,10 @@ import logging import os from datetime import datetime -from flask import current_app -from sqlalchemy.exc import SQLAlchemyError import git import git.exc +from flask import current_app +from sqlalchemy.exc import SQLAlchemyError from patch_tracking.api.business import update_tracking import patch_tracking.util.upstream.upstream as upstream @@ -78,14 +78,14 @@ class Git(upstream.Upstream): get latest commit id """ repo_path = os.path.join(self.base_path, self.repo_dir_name) - logging.info("Getting latest commit id of repo: %s branch: %s .", repo_path, self.track.branch) + logging.info("Getting latest commit id of repo: %s branch: %s .", repo_path, self.track.scm_branch) try: repo = git.Repo(repo_path) - sha = repo.commit(self.track.branch).hexsha + sha = repo.commit(self.track.scm_branch).hexsha return sha except git.exc.GitError as err: logging.error( - "Get latest commit id of repo: %s branch: %s failed. Error: %s", repo_path, self.track.branch, err + "Get latest commit id of repo: %s branch: %s failed. Error: %s", repo_path, self.track.scm_branch, err ) return False diff --git a/patch_tracking/util/upstream/github.py b/patch_tracking/util/upstream/github.py index 1b1b40f0054f1b4f34b70539049d38269491ee6f..28bf8fb2ba027ef1d15e83e7108a3a6e0e7105b8 100644 --- a/patch_tracking/util/upstream/github.py +++ b/patch_tracking/util/upstream/github.py @@ -1,7 +1,9 @@ """github upstream""" import logging +import time +import requests from flask import current_app -from patch_tracking.util.github_api import GitHubApi +from requests import exceptions from patch_tracking.api.business import update_tracking from sqlalchemy.exc import SQLAlchemyError import patch_tracking.util.upstream.upstream as upstream @@ -9,34 +11,151 @@ import patch_tracking.util.upstream.upstream as upstream logger = logging.getLogger(__name__) +def get_user_info(token): + """ + get user info + """ + url = "https://api.github.com/user" + count = 30 + token = 'token ' + token + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Authorization': token, + 'Content-Type': 'application/json', + 'Connection': 'close', + 'method': 'GET', + 'Accept': 'application/json' + } + while count > 0: + try: + ret = requests.get(url, headers=headers) + if ret.status_code == 200: + return True, ret.text + return False, ret.text + except exceptions.ConnectionError as err: + logger.warning(err) + time.sleep(10) + count -= 1 + continue + except UnicodeEncodeError: + return False, 'github token is bad credentials.' + except IOError as error: + return False, error + if count == 0: + logger.error('Fail to connnect to github: %s after retry 30 times.', url) + return False, 'connect error' + + class GitHub(upstream.Upstream): """ GitHub type """ def __init__(self, track): super().__init__(track) - self.github_api = GitHubApi() - self.token = current_app.config['GITHUB_ACCESS_TOKEN'] + _token = 'token ' + current_app.config['GITHUB_ACCESS_TOKEN'] + self.headers = { + 'User-Agent': 'Mozilla/5.0', + 'Authorization': _token, + 'Content-Type': 'application/json', + 'Connection': 'close', + 'method': 'GET', + 'Accept': 'application/json' + } + + def api_request(self, url): + """ + request GitHub API + """ + logger.debug("Connect url: %s", url) + count = 30 + while count > 0: + try: + response = requests.get(url, headers=self.headers) + return response + except exceptions.ConnectionError as err: + logger.warning(err) + time.sleep(10) + count -= 1 + continue + except IOError as err: + logger.error(err) + return False + if count == 0: + logger.error('Fail to connnect to github: %s after retry 30 times.', url) + return False + + def get_patch(self, repo_url, scm_commit, last_commit): + """ + get patch + """ + api_url = 'https://github.com' + if scm_commit != last_commit: + commit = scm_commit + '...' + last_commit + '.diff' + else: + commit = scm_commit + '^...' + scm_commit + '.diff' + ret_dict = dict() + + url = '/'.join([api_url, repo_url, 'compare', commit]) + ret = self.api_request(url) + if ret: + if ret.status_code == 200: + patch_content = ret.text + ret_dict['status'] = 'success' + ret_dict['api_ret'] = patch_content + else: + logger.error('%s failed. Return val: %s', url, ret) + ret_dict['status'] = 'error' + ret_dict['api_ret'] = ret.text + else: + ret_dict['status'] = 'error' + ret_dict['api_ret'] = 'fail to connect github by api.' + return ret_dict def get_latest_commit_id(self): """ - get latest commit id + get latest commit_ID, commit_message, commit_date + :return: res_dict """ - status, result = self.github_api.get_latest_commit(self.track.scm_repo, self.track.scm_branch) - logger.debug( - 'repo: %s branch: %s. get_latest_commit: %s %s', self.track.scm_repo, self.track.scm_branch, status, result - ) + api_url = 'https://api.github.com/repos' + url = '/'.join([api_url, self.track.scm_repo, 'branches', self.track.scm_branch]) + ret = self.api_request(url) + res_dict = dict() + if ret: + if ret.status_code == 200: + res_dict['latest_commit'] = ret.json()['commit']['sha'] + res_dict['message'] = ret.json()['commit']['commit']['message'] + res_dict['time'] = ret.json()['commit']['commit']['committer']['date'] + logger.debug( + 'repo: %s branch: %s. get_latest_commit: %s %s', self.track.scm_repo, self.track.scm_branch, + 'success', res_dict + ) + return 'success', res_dict - return status, result + logger.error('%s failed. Return val: %s', url, ret) + return 'error', ret.json() + return 'error', 'connect error' - def get_commit_info(self, commit_id): + def get_commit_info(self, repo_url, commit_id): """ get commit info """ - status, result = self.github_api.get_commit_info(self.track.scm_repo, commit_id) - logger.debug('get_commit_info: %s %s', status, result) + res_dict = dict() + api_url = 'https://api.github.com/repos' + url = '/'.join([api_url, repo_url, 'commits', commit_id]) + ret = self.api_request(url) + if ret: + if ret.status_code == 200: + res_dict['commit_id'] = commit_id + res_dict['message'] = ret.json()['commit']['message'] + res_dict['time'] = ret.json()['commit']['author']['date'] + if 'parents' in ret.json() and ret.json()['parents']: + res_dict['parent'] = ret.json()['parents'][0]['sha'] + logger.debug('get_commit_info: %s %s', 'success', res_dict) + return 'success', res_dict - return status, result + logger.error('%s failed. Return val: %s', url, ret) + return 'error', ret.json() + return 'error', 'connect error' def get_patch_list(self): """ @@ -70,11 +189,11 @@ class GitHub(upstream.Upstream): return None while self.track.scm_commit != latest_commit: - status, result = self.get_commit_info(latest_commit) + status, result = self.get_commit_info(self.track.scm_repo, latest_commit) logger.debug('get_commit_info: %s %s', status, result) if status == 'success': if 'parent' in result: - ret = self.github_api.get_patch(self.track.scm_repo, latest_commit, latest_commit) + ret = self.get_patch(self.track.scm_repo, latest_commit, latest_commit) logger.debug('get patch api ret: %s', ret) if ret['status'] == 'success': result['patch_content'] = ret['api_ret'] @@ -97,7 +216,6 @@ class GitHub(upstream.Upstream): '[Patch Tracking] Get scm: %s commit: %s ID/message/time failed. Result: %s', self.track.scm_repo, latest_commit, result ) - return commit_list def get_scm_patch(self): @@ -114,5 +232,4 @@ class GitHub(upstream.Upstream): issue_table += '| [{}]({}) | {} | {} |'.format( latest_commit['commit_id'][0:7], scm_commit_url, latest_commit['time'], latest_commit['message'] ) + '\n' - return issue_table