diff options
author | Magnus Granberg <zorry@gentoo.org> | 2023-02-21 02:39:54 +0100 |
---|---|---|
committer | Magnus Granberg <zorry@gentoo.org> | 2023-02-21 02:39:54 +0100 |
commit | 021a017b6e7ba89d28fac0e40db2ebb8baad9ce2 (patch) | |
tree | 0ced1dd2f151d73c4c75d127adc1842d7ea64107 /buildbot_gentoo_ci | |
parent | Use command shell to build docker images (diff) | |
download | tinderbox-cluster-021a017b6e7ba89d28fac0e40db2ebb8baad9ce2.tar.gz tinderbox-cluster-021a017b6e7ba89d28fac0e40db2ebb8baad9ce2.tar.bz2 tinderbox-cluster-021a017b6e7ba89d28fac0e40db2ebb8baad9ce2.zip |
Get all search pattern that match
Signed-off-by: Magnus Granberg <zorry@gentoo.org>
Diffstat (limited to 'buildbot_gentoo_ci')
-rw-r--r-- | buildbot_gentoo_ci/steps/bugs.py | 4 | ||||
-rw-r--r-- | buildbot_gentoo_ci/steps/logs.py | 178 |
2 files changed, 19 insertions, 163 deletions
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py index 801fc98..6c85aac 100644 --- a/buildbot_gentoo_ci/steps/bugs.py +++ b/buildbot_gentoo_ci/steps/bugs.py @@ -96,8 +96,8 @@ class GetBugs(BuildStep): for match_word in match_search_text: if match_word in match_bug_text: matches = matches + 1 - if matches >= 10: - print(f"Bug: {str(bug['id'])} Summary: {bug['summary']}") + print(f"Bug: {str(bug['id'])} Matched words: {str(matches)} Summary: {bug['summary']}") + if matches >= 5: match = {} match['match'] = True match['id'] = bug['id'] diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py index 2a52308..e4fc951 100644 --- a/buildbot_gentoo_ci/steps/logs.py +++ b/buildbot_gentoo_ci/steps/logs.py @@ -19,6 +19,7 @@ from buildbot.process.results import FAILURE from buildbot.process.results import WARNINGS from buildbot.process.results import SKIPPED from buildbot.plugins import steps +from buildbot.plugins import util from buildbot_gentoo_ci.steps import minio from buildbot_gentoo_ci.steps import master as master_steps @@ -35,11 +36,15 @@ def PersOutputOfLogParser(rc, stdout, stderr): for k, v in json.loads(line).items(): summary_log_dict[int(k)] = { 'text' : v['text'], - 'type' : v['type'], - 'status' : v['status'], - 'id' : v['id'], - 'search_pattern' : v['search_pattern'] + 'pattern_infos' : [], } + for s in v['pattern_info']: + summary_log_dict[int(k)]['pattern_infos'].append({ + 'type' : s['type'], + 'status' : s['status'], + 'id' : s['id'], + 'search_pattern' : s['search_pattern'], + }) build_summery_output['summary_log_dict'] = summary_log_dict #FIXME: Handling of stderr output return { @@ -140,6 +145,8 @@ class SetupParserBuildLoger(BuildStep): command.append(log_cpv['full_logname']) command.append('-u') command.append(self.getProperty('project_data')['uuid']) + command.append('-d') + command.append(util.Secret("log_parser_database")) self.aftersteps_list.append(steps.SetPropertyFromCommand( name = 'RunBuildLogParser', haltOnFailure = True, @@ -152,159 +159,6 @@ class SetupParserBuildLoger(BuildStep): yield self.build.addStepsAfterCurrentStep(self.aftersteps_list) return SUCCESS -class ParserBuildLog(BuildStep): - - name = 'ParserBuildLog' - description = 'Running' - descriptionDone = 'Ran' - descriptionSuffix = None - haltOnFailure = True - flunkOnFailure = True - - def __init__(self, **kwargs): - self.logfile_text_dict = {} - self.summery_dict = {} - self.index = 1 - self.log_search_pattern_list = [] - self.max_text_lines = 0 - super().__init__(**kwargs) - - #FIXME: ansifilter - def ansiFilter(self, text): - return text - - @defer.inlineCallbacks - def get_log_search_pattern(self): - # get pattern from the projects - # add that to log_search_pattern_list - for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])): - # check if the search pattern is vaild - try: - re.compile(project_pattern['search']) - except re.error: - print("Non valid regex pattern") - print(project_pattern) - else: - self.log_search_pattern_list.append(project_pattern) - # get the default project pattern - # add if not pattern is in project ignore - self.project_pattern_ignore = yield self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('project_data')['uuid']) - for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('default_project_data')['uuid'])): - if not project_pattern['search'] in self.project_pattern_ignore: - # check if the search pattern is vaild - try: - re.compile(project_pattern['search']) - except re.error: - print("Non valid regex pattern") - print(project_pattern) - else: - self.log_search_pattern_list.append(project_pattern) - - def search_buildlog(self, tmp_index): - # get text line to search - text_line = self.ansiFilter(self.logfile_text_dict[tmp_index]) - # loop true the pattern list for match - for search_pattern in self.log_search_pattern_list: - search_hit = False - if search_pattern['search_type'] == 'in': - if search_pattern['search'] in text_line: - search_hit = True - if search_pattern['search_type'] == 'startswith': - if text_line.startswith(search_pattern['search']): - search_hit = True - if search_pattern['search_type'] == 'endswith': - if text_line.endswith(search_pattern['search']): - search_hit = True - if search_pattern['search_type'] == 'search': - if re.search(search_pattern['search'], text_line): - search_hit = True - # add the line if the pattern match - if search_hit: - print(text_line) - print(search_pattern) - print(tmp_index) - self.summery_dict[tmp_index] = {} - self.summery_dict[tmp_index]['text'] = text_line - self.summery_dict[tmp_index]['type'] = search_pattern['type'] - self.summery_dict[tmp_index]['status'] = search_pattern['status'] - self.summery_dict[tmp_index]['search_pattern_id'] = search_pattern['id'] - # add upper text lines if requested - # max 5 - if search_pattern['start'] != 0: - i = tmp_index - search_pattern['start'] - 1 - match = True - while match: - i = i + 1 - if i < (tmp_index - 9) or i == tmp_index: - match = False - else: - if not i in self.summery_dict: - self.summery_dict[i] = {} - self.summery_dict[i]['text'] = self.ansiFilter(self.logfile_text_dict[i]) - self.summery_dict[i]['type'] = 'info' - self.summery_dict[i]['status'] = 'info' - # add lower text lines if requested - # max 5 - if search_pattern['end'] != 0: - i = tmp_index - end = tmp_index + search_pattern['end'] - match = True - while match: - i = i + 1 - if i > self.max_text_lines or i > end: - match = False - else: - if not i in self.summery_dict: - self.summery_dict[i] = {} - self.summery_dict[i]['text'] = self.ansiFilter(self.logfile_text_dict[i]) - self.summery_dict[i]['type'] = 'info' - self.summery_dict[i]['status'] = 'info' - else: - # we add all line that start with ' * ' as info - # we add all line that start with '>>>' but not '>>> /' as info - if text_line.startswith(' * ') or (text_line.startswith('>>>') and not text_line.startswith('>>> /')): - if not tmp_index in self.summery_dict: - self.summery_dict[tmp_index] = {} - self.summery_dict[tmp_index]['text'] = text_line - self.summery_dict[tmp_index]['type'] = 'info' - self.summery_dict[tmp_index]['status'] = 'info' - - @defer.inlineCallbacks - def run(self): - self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] - yield self.get_log_search_pattern() - # open the log file - # read it to a buffer - # make a dict of the buffer - # maybe use mulitiprocces to speed up the search - print(self.getProperty('log_build_data')) - if self.getProperty('faild_cpv'): - log_cpv = self.getProperty('log_build_data')[self.getProperty('faild_cpv')] - else: - log_cpv = self.getProperty('log_build_data')[self.getProperty('cpv')] - file_path = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']) ,log_cpv['full_logname']) - #FIXME: decode it to utf-8 - with io.TextIOWrapper(io.BufferedReader(gzip.open(file_path, 'rb'))) as f: - for text_line in f: - self.logfile_text_dict[self.index] = text_line.strip('\n') - # run the parse patten on the line - # have a buffer on 10 before we run pattern check - if self.index >= 10: - yield self.search_buildlog(self.index - 9) - # remove text line that we don't need any more - if self.index >= 20: - del self.logfile_text_dict[self.index - 19] - self.index = self.index + 1 - self.max_text_lines = self.index - f.close() - # check last 10 lines in logfile_text_dict - yield self.search_buildlog(self.index - 10) - print(self.summery_dict) - # remove all lines with ignore in the dict - # setProperty summery_dict - self.setProperty("summary_log_dict", self.summery_dict, 'summary_log_dict') - return SUCCESS - class MakeIssue(BuildStep): name = 'MakeIssue' @@ -353,8 +207,9 @@ class MakeIssue(BuildStep): text_phase_list = [] for k, v in sorted(self.summary_log_dict.items()): # get the issue error - if v['type'] == self.error_dict['phase'] and v['status'] == 'error': - text_issue_list.append(v['text']) + for s in v['pattern_infos']: + if s['type'] == self.error_dict['phase'] and s['status'] == 'error': + text_issue_list.append(v['text']) # add the issue error if text_issue_list != []: self.error_dict['title_issue'] = text_issue_list[0].replace('*', '').strip() @@ -379,8 +234,9 @@ class MakeIssue(BuildStep): for k, v in sorted(self.summary_log_dict.items()): self.summary_log_list.append(v['text']) #self.error_dict['hash'].update(v['text'].encode('utf-8')) - if v['status'] == 'warning': - warning = True + for s in v['pattern_infos']: + if s['status'] == 'warning': + warning = True # check if the build did fail if v['text'].startswith(' * ERROR:') and v['text'].endswith(' phase):'): # get phase error |