Get all search pattern that match

Signed-off-by: Magnus Granberg <zorry@gentoo.org>

2 files changed, 19 insertions, 163 deletions

diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 801fc98..6c85aac 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -96,8 +96,8 @@ class GetBugs(BuildStep):
             for match_word in match_search_text:
                 if match_word in match_bug_text:
                     matches = matches + 1
-            if matches >= 10:
-                print(f"Bug: {str(bug['id'])} Summary: {bug['summary']}")
+            print(f"Bug: {str(bug['id'])} Matched words: {str(matches)} Summary: {bug['summary']}")
+            if matches >= 5:
                 match = {}
                 match['match'] = True
                 match['id'] = bug['id']
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 2a52308..e4fc951 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -19,6 +19,7 @@ from buildbot.process.results import FAILURE
 from buildbot.process.results import WARNINGS
 from buildbot.process.results import SKIPPED
 from buildbot.plugins import steps
+from buildbot.plugins import util
 
 from buildbot_gentoo_ci.steps import minio
 from buildbot_gentoo_ci.steps import master as master_steps
@@ -35,11 +36,15 @@ def PersOutputOfLogParser(rc, stdout, stderr):
             for k, v in json.loads(line).items():
                 summary_log_dict[int(k)] = {
                                     'text' : v['text'],
-                                    'type' : v['type'],
-                                    'status' : v['status'],
-                                    'id' : v['id'],
-                                    'search_pattern' : v['search_pattern']
+                                    'pattern_infos' : [],
                                     }
+                for s in v['pattern_info']:
+                    summary_log_dict[int(k)]['pattern_infos'].append({
+                                                            'type' : s['type'],
+                                                            'status' : s['status'],
+                                                            'id' : s['id'],
+                                                            'search_pattern' : s['search_pattern'],
+                                                            })
     build_summery_output['summary_log_dict'] = summary_log_dict
     #FIXME: Handling of stderr output
     return {
@@ -140,6 +145,8 @@ class SetupParserBuildLoger(BuildStep):
         command.append(log_cpv['full_logname'])
         command.append('-u')
         command.append(self.getProperty('project_data')['uuid'])
+        command.append('-d')
+        command.append(util.Secret("log_parser_database"))
         self.aftersteps_list.append(steps.SetPropertyFromCommand(
                                                             name = 'RunBuildLogParser',
                                                             haltOnFailure = True,
@@ -152,159 +159,6 @@ class SetupParserBuildLoger(BuildStep):
         yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
         return SUCCESS
 
-class ParserBuildLog(BuildStep):
-
-    name = 'ParserBuildLog'
-    description = 'Running'
-    descriptionDone = 'Ran'
-    descriptionSuffix = None
-    haltOnFailure = True
-    flunkOnFailure = True
-
-    def __init__(self, **kwargs):
-        self.logfile_text_dict = {}
-        self.summery_dict = {}
-        self.index = 1
-        self.log_search_pattern_list = []
-        self.max_text_lines = 0
-        super().__init__(**kwargs)
-
-    #FIXME: ansifilter
-    def ansiFilter(self, text):
-        return text
-
-    @defer.inlineCallbacks
-    def get_log_search_pattern(self):
-        # get pattern from the projects
-        # add that to log_search_pattern_list
-        for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])):
-            # check if the search pattern is vaild
-            try:
-                re.compile(project_pattern['search'])
-            except re.error:
-                print("Non valid regex pattern")
-                print(project_pattern)
-            else:
-                self.log_search_pattern_list.append(project_pattern)
-        # get the default project pattern
-        # add if not pattern is in project ignore
-        self.project_pattern_ignore = yield self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('project_data')['uuid'])
-        for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('default_project_data')['uuid'])):
-            if not project_pattern['search'] in self.project_pattern_ignore:
-                # check if the search pattern is vaild
-                try:
-                    re.compile(project_pattern['search'])
-                except re.error:
-                    print("Non valid regex pattern")
-                    print(project_pattern)
-                else:
-                    self.log_search_pattern_list.append(project_pattern)
-
-    def search_buildlog(self, tmp_index):
-        # get text line to search
-        text_line = self.ansiFilter(self.logfile_text_dict[tmp_index])
-        # loop true the pattern list for match
-        for search_pattern in self.log_search_pattern_list:
-            search_hit = False
-            if search_pattern['search_type'] == 'in':
-                if search_pattern['search'] in text_line:
-                    search_hit = True
-            if search_pattern['search_type'] == 'startswith':
-                if text_line.startswith(search_pattern['search']):
-                    search_hit = True
-            if search_pattern['search_type'] == 'endswith':
-                if text_line.endswith(search_pattern['search']):
-                    search_hit = True
-            if search_pattern['search_type'] == 'search':
-                if re.search(search_pattern['search'], text_line):
-                    search_hit = True
-            # add the line if the pattern match
-            if search_hit:
-                print(text_line)
-                print(search_pattern)
-                print(tmp_index)
-                self.summery_dict[tmp_index] = {}
-                self.summery_dict[tmp_index]['text'] = text_line
-                self.summery_dict[tmp_index]['type'] = search_pattern['type']
-                self.summery_dict[tmp_index]['status'] = search_pattern['status']
-                self.summery_dict[tmp_index]['search_pattern_id'] = search_pattern['id']
-                # add upper text lines if requested
-                # max 5
-                if search_pattern['start'] != 0:
-                    i = tmp_index - search_pattern['start'] - 1
-                    match = True
-                    while match:
-                        i = i + 1
-                        if i < (tmp_index - 9) or i == tmp_index:
-                            match = False
-                        else:
-                            if not i in self.summery_dict:
-                                self.summery_dict[i] = {}
-                                self.summery_dict[i]['text'] = self.ansiFilter(self.logfile_text_dict[i])
-                                self.summery_dict[i]['type'] = 'info'
-                                self.summery_dict[i]['status'] = 'info'
-                # add lower text lines if requested
-                # max 5
-                if search_pattern['end'] != 0:
-                    i = tmp_index
-                    end = tmp_index + search_pattern['end']
-                    match = True
-                    while match:
-                        i = i + 1
-                        if i > self.max_text_lines or i > end:
-                            match = False
-                        else:
-                            if not i in self.summery_dict:
-                                self.summery_dict[i] = {}
-                                self.summery_dict[i]['text'] = self.ansiFilter(self.logfile_text_dict[i])
-                                self.summery_dict[i]['type'] = 'info'
-                                self.summery_dict[i]['status'] = 'info'
-            else:
-                # we add all line that start with ' * ' as info
-                # we add all line that start with '>>>' but not '>>> /' as info
-                if text_line.startswith(' * ') or (text_line.startswith('>>>') and not text_line.startswith('>>> /')):
-                    if not tmp_index in self.summery_dict:
-                        self.summery_dict[tmp_index] = {}
-                        self.summery_dict[tmp_index]['text'] = text_line
-                        self.summery_dict[tmp_index]['type'] = 'info'
-                        self.summery_dict[tmp_index]['status'] = 'info'
-
-    @defer.inlineCallbacks
-    def run(self):
-        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
-        yield self.get_log_search_pattern()
-        # open the log file
-        # read it to a buffer
-        # make a dict of the buffer
-        # maybe use mulitiprocces to speed up the search
-        print(self.getProperty('log_build_data'))
-        if self.getProperty('faild_cpv'):
-            log_cpv = self.getProperty('log_build_data')[self.getProperty('faild_cpv')]
-        else:
-            log_cpv = self.getProperty('log_build_data')[self.getProperty('cpv')]
-        file_path = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']) ,log_cpv['full_logname'])
-        #FIXME: decode it to utf-8
-        with io.TextIOWrapper(io.BufferedReader(gzip.open(file_path, 'rb'))) as f:
-            for text_line in f:
-                self.logfile_text_dict[self.index] = text_line.strip('\n')
-                # run the parse patten on the line
-                # have a buffer on 10 before we run pattern check
-                if self.index >= 10:
-                    yield self.search_buildlog(self.index - 9)
-                # remove text line that we don't need any more
-                if self.index >= 20:
-                    del self.logfile_text_dict[self.index - 19]
-                self.index = self.index + 1
-                self.max_text_lines = self.index
-            f.close()
-        # check last 10 lines in logfile_text_dict
-        yield self.search_buildlog(self.index - 10)
-        print(self.summery_dict)
-        # remove all lines with ignore in the dict
-        # setProperty summery_dict
-        self.setProperty("summary_log_dict", self.summery_dict, 'summary_log_dict')
-        return SUCCESS
-
 class MakeIssue(BuildStep):
 
     name = 'MakeIssue'
@@ -353,8 +207,9 @@ class MakeIssue(BuildStep):
         text_phase_list = []
         for k, v in sorted(self.summary_log_dict.items()):
             # get the issue error
-            if v['type'] == self.error_dict['phase'] and v['status'] == 'error':
-                text_issue_list.append(v['text'])
+            for s in v['pattern_infos']:
+                if s['type'] == self.error_dict['phase'] and s['status'] == 'error':
+                    text_issue_list.append(v['text'])
         # add the issue error
         if text_issue_list != []:
             self.error_dict['title_issue'] = text_issue_list[0].replace('*', '').strip()
@@ -379,8 +234,9 @@ class MakeIssue(BuildStep):
         for k, v in sorted(self.summary_log_dict.items()):
             self.summary_log_list.append(v['text'])
             #self.error_dict['hash'].update(v['text'].encode('utf-8'))
-            if v['status'] == 'warning':
-                warning = True
+            for s in v['pattern_infos']:
+                if s['status'] == 'warning':
+                    warning = True
             # check if the build did fail
             if v['text'].startswith(' * ERROR:') and v['text'].endswith(' phase):'):
                 # get phase error