aboutsummaryrefslogtreecommitdiff
blob: 50e3f3f9a771475d84b7c5a00a8c0c78332f33f8 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

import os
import re
import gzip
import io
import hashlib
import json

from portage.versions import catpkgsplit

from twisted.internet import defer
from twisted.python import log

from buildbot.process.buildstep import BuildStep
from buildbot.process.properties import Properties
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import WARNINGS
from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
from buildbot.plugins import util

#from buildbot_gentoo_ci.steps import minio
from buildbot_gentoo_ci.steps import master as master_steps
from buildbot_gentoo_ci.steps import bugs
from buildbot_gentoo_ci.utils.regex import stripQuotesAndMore, finishTitle

def PersOutputOfLogParser(rc, stdout, stderr):
    build_summery_output = {}
    build_summery_output['rc'] = rc
    summary_log_dict = {}
    # split the lines
    for line in stdout.split('\n'):
        #FIXME: check if line start with {"[1-9]": {
        if line.startswith('{'):
            for k, v in json.loads(line).items():
                summary_log_dict[int(k)] = {
                                    'text' : v['text'],
                                    'pattern_infos' : [],
                                    }
                for s in v['pattern_info']:
                    summary_log_dict[int(k)]['pattern_infos'].append({
                                                            'search_type' : s['search_type'],
                                                            'status' : s['status'],
                                                            'line' : s['line'],
                                                            'search_pattern' : s['search_pattern'],
                                                            'phase' : s['phase'],
                                                            'uuid' : s['uuid'],
                                                            'description' : s['description'],
                                                            })
    build_summery_output['summary_log_dict'] = summary_log_dict
    #FIXME: Handling of stderr output
    return {
        'build_summery_output' : build_summery_output
        }

def PersOutputOfEmergeInfo(rc, stdout, stderr):
    #FIXME: line for package info
    emerge_info_output = {}
    emerge_info_output['rc'] = rc
    emerge_info_list = []
    emerge_package_info = []
    for line in stdout.split('\n'):
        if line.startswith('['):
            emerge_package_info.append(line)
        else:
            emerge_info_list.append(line)
    emerge_info_output['emerge_info'] = emerge_info_list
    emerge_info_output['emerge_package_info'] = emerge_package_info
    return {
        'emerge_info_output' : emerge_info_output
        }

class SetupPropertys(BuildStep):
    
    name = 'SetupPropertys'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        # set this in config
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        project_data = yield self.gentooci.db.projects.getProjectByUuid(self.getProperty('project_build_data')['project_uuid'])
        default_project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project']['update_db'])
        version_data = yield self.gentooci.db.versions.getVersionByUuid(self.getProperty('project_build_data')['version_uuid'])
        self.setProperty("project_data", project_data, 'project_data')
        self.setProperty("default_project_data", default_project_data, 'default_project_data')
        self.setProperty("version_data", version_data, 'version_data')
        self.setProperty("status", 'completed', 'status')
        if isinstance(self.getProperty('faild_cpv'), str):
            log_cpv = self.getProperty('faild_cpv')
        else:
            log_cpv = self.getProperty('cpv')
        self.setProperty("log_cpv", log_cpv, 'log_cpv')
        self.setProperty("bgo", False, 'bgo')
        self.descriptionDone = 'Runing log checker on ' + log_cpv
        logsdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']))
        self.setProperty("logsdir", logsdir, 'logsdir')
        return SUCCESS

class SetupParserBuildLoger(BuildStep):

    name = 'SetupParserBuildLoger'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        self.aftersteps_list = []
        log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')]
        build_log_file_compressed = log_cpv['full_logname'] + '.xz'
        mastersrc_log = yield os.path.join(self.getProperty('logsdir'), build_log_file_compressed)
        log_py = 'log_parser.py'
        mastersrc_py = yield os.path.join(self.master.basedir, log_py)
        repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(self.getProperty("project_data")['settings_repository_uuid'])
        # Git clone settings repo
        self.aftersteps_list.append(steps.GitLab(repourl=repository_data['url'],
            name = 'RunGit',
            descriptionDone=repository_data['name'],
            mode=repository_data['mode'],
            method=repository_data['method'],
            submodules=True,
            alwaysUseLatest=repository_data['alwaysuselatest']
        ))
        # Upload logfile to worker
        self.aftersteps_list.append(steps.FileDownload(
                                                    mastersrc=mastersrc_log,
                                                    workerdest=build_log_file_compressed
                                                    ))
        # Upload log parser py code
        self.aftersteps_list.append(steps.FileDownload(
                                                    mastersrc=mastersrc_py,
                                                    workerdest=log_py
                                                    ))
        #Untar the log
        shell_commad_list = []
        shell_commad_list.append('xz')
        shell_commad_list.append('-dv')
        shell_commad_list.append(build_log_file_compressed)
        self.aftersteps_list.append(
            steps.ShellCommand(
                            name = 'Decompress build log with xz',
                            command = shell_commad_list,
        ))
        # Run the log parser code
        command = []
        command.append('python3')
        command.append(log_py)
        command.append('-f')
        command.append(log_cpv['full_logname'])
        command.append('-u')
        command.append(self.getProperty('project_data')['uuid'])
        command.append('-c')
        #FIXME: set it by images/flavors
        command.append('8')
        #FIXME: debug
        #command.append('-d')
        self.aftersteps_list.append(steps.SetPropertyFromCommand(
                                                            name = 'RunBuildLogParser',
                                                            haltOnFailure = True,
                                                            flunkOnFailure = True,
                                                            command=command,
                                                            strip=False,
                                                            extract_fn=PersOutputOfLogParser,
                                                            timeout=3600
                                                            ))
        yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
        return SUCCESS

class MakeIssue(BuildStep):

    name = 'MakeIssue'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def getNiceErrorLine(self, full_line):
        new_words = []
        for word in full_line.split(' '):
            new_words.append(finishTitle(stripQuotesAndMore(word)))
        return ' '.join(new_words)

    def BuildWordList(self):
        word_list = []
        word_list.append(self.error_dict['cpv'])
        c = catpkgsplit(self.error_dict['cpv'])[0]
        p = catpkgsplit(self.error_dict['cpv'])[1]
        cp = c + '/' + p
        word_list.append(cp)
        for word in self.error_dict['title_phase'].split(' '):
            word_list.append(word)
        for word in self.error_dict['title_issue_nice'].split(' '):
            word_list.append(word)
        return word_list

    def ClassifyIssue(self):
        # get the title for the issue
        text_issue_list = []
        text_phase_list = []
        for k, v in sorted(self.summary_log_dict.items()):
            # get the issue error
            for s in v['pattern_infos']:
                if s['search_type'] == self.error_dict['phase'] and s['status'] == 'error':
                    text_issue_list.append(v['text'])
        # add the issue error
        self.error_dict['cpv'] = self.getProperty('log_cpv')
        if text_issue_list != []:
            title_line = text_issue_list[0].replace('*', '').strip()
            self.error_dict['title_issue'] = title_line
            self.error_dict['title_issue_nice'] = self.getNiceErrorLine(title_line)
            self.error_dict['title_found'] = True
        else:
            self.error_dict['title_issue'] = 'title_issue : None'
            self.error_dict['title_issue_nice'] = 'title_issue_nice : None'
            self.error_dict['title_found'] = False
        self.error_dict['title_phase'] = 'fails to '+ self.error_dict['phase']
        #set the error title
        self.error_dict['title'] = ' '.join([self.error_dict['title_phase'] + ':', self.error_dict['title_issue']])
        self.error_dict['title_nice'] = ' '.join([self.error_dict['title_phase'] + ':', self.error_dict['title_issue_nice']])
        self.error_dict['words'] = self.BuildWordList()

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        self.summary_log_dict = self.getProperty('build_summery_output')['summary_log_dict']
        error = False
        warning = False
        self.summary_log_list = []
        self.error_dict = {}
        self.aftersteps_list = []
        #self.error_dict['hash'] = hashlib.sha256()
        for k, v in sorted(self.summary_log_dict.items()):
            self.summary_log_list.append(v['text'])
            #self.error_dict['hash'].update(v['text'].encode('utf-8'))
            for s in v['pattern_infos']:
                if s['status'] == 'warning':
                    warning = True
            # check if the build did fail
            if v['text'].startswith(' * ERROR:') and v['text'].endswith(' phase):'):
                # get phase error
                phase_error = v['text'].split(' (')[1].split(' phase')[0]
                self.error_dict['phase'] = phase_error
                error = True
        #FIXME: write summary_log_list to a file
        # add issue/bug/pr report
        if error:
            self.ClassifyIssue()
            print(self.error_dict)
            self.setProperty("status", 'failed', 'status')
            self.setProperty("error_dict", self.error_dict, 'error_dict')
            self.aftersteps_list.append(bugs.GetBugs())
        if warning:
            self.setProperty("status", 'warning', 'status')
            self.setProperty("bgo", False, 'bgo')
        self.setProperty("summary_log_list", self.summary_log_list, 'summary_log_list')
        if self.aftersteps_list is not []:
            yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
        return SUCCESS

class Summarylog(BuildStep):

    name = 'Summarylog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        #setup the log
        log = yield self.addLog('summary')
        # add line for line
        for line in self.getProperty('summary_log_list'):
            yield log.addStdout(line + '\n')
        return SUCCESS

class ReadEmergeInfoLog(BuildStep):

    name = 'ReadEmergeInfoLog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    #@defer.inlineCallbacks
    def run(self):
        emerge_info_output = {}
        emerge_info_list = []
        emerge_package_info = []
        # Read the file and add it to a property
        with open(os.path.join(self.getProperty('logsdir'), 'emerge_info.txt'), encoding='utf-8') as source:
            emerge_info = source.read()
        # set emerge_info_output Property
        for line in emerge_info.split('\n'):
            if line.startswith('['):
                emerge_package_info.append(line)
            else:
                emerge_info_list.append(line)
        emerge_info_output['emerge_info'] = emerge_info_list
        emerge_info_output['emerge_package_info'] = emerge_package_info
        self.setProperty("emerge_info_output", emerge_info_output, 'emerge_info_output')
        return SUCCESS

class setEmergeInfoLog(BuildStep):

    name = 'setEmergeInfoLog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        #setup the log
        log = yield self.addLog('emerge_info')
        #FIXME: add emerge info to db
        # add line for line
        for line in self.getProperty('emerge_info_output')['emerge_info']:
            yield log.addStdout(line + '\n')
        return SUCCESS

class setPackageInfoLog(BuildStep):

    name = 'setPackageInfoLog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        #setup the log
        log = yield self.addLog('package_info')
        #FIXME: add package info to db
        # add line for line
        for line in self.getProperty('emerge_info_output')['emerge_package_info']:
            yield log.addStdout(line + '\n')
        return SUCCESS


class Upload(BuildStep):

    name = 'Upload'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')]
        bucket = self.getProperty('project_data')['uuid'] + '-' + 'logs'
        file_path = yield os.path.join(self.getProperty('logsdir') ,log_cpv['full_logname'])
        aftersteps_list = []
        aftersteps_list.append(minio.putFileToMinio(file_path, log_cpv['full_logname'], bucket))
        yield self.build.addStepsAfterCurrentStep(aftersteps_list)
        return SUCCESS

class ParserPkgCheckLog(BuildStep):

    name = 'ParserPkgCheckLog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnWarnings = False
    flunkOnFailure = False
    warnOnWarnings = False

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        if self.getProperty("pkg_check_log_data") is None:
            return SKIPPED
        returnstatus = SUCCESS
        error = False
        warning = False
        log = yield self.addLog('Pkgcheck')
        print(self.getProperty("pkg_check_log_data"))
        for a in self.getProperty("pkg_check_log_data"):
            status = ''
            print(a)
            if isinstance(a, dict):
                for k, i in a.items():
                    if k.startswith('_'):
                        if k == '_info':
                            status = 'INFO: '
                        if k == '_error':
                            status = 'ERROR: '
                            error = True
                        if k == '_warning':
                            status = 'WARNING: '
                            warning = True
                        if k == '_style':
                            status = 'STYLE: '
                        if isinstance(i, dict):
                            for b, c in i.items():
                                yield log.addStdout(status + b + c + '\n')
                    else:
                        yield log.addStdout(i + '\n')
        if error:
            returnstatus = FAILURE
        if warning and not error:
            returnstatus = WARNINGS
        return returnstatus

class SetupBugReportSteps(BuildStep):
    name = 'SetupBugReportSteps'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        bug_config = self.gentooci.config.project['bug_config']
        # add bug enable profile and repo
        if bug_config['enable'] and self.getProperty('status') == 'failed' and self.getProperty('error_dict')['title_found'] and not self.getProperty('bgo'):
            aftersteps_list = []
            #Post
            p = Properties()
            p.master = self.master
            separator = ' '
            separator2 = '\n'
            bug_args = {}
            bug_args['user'] = bug_config['user']
            bug_args['passwd'] = yield p.render(util.Secret("bugs_password"))
            bug_params = {}
            title = separator.join([bug_config['extra_summery'], self.getProperty('error_dict')['cpv'], self.getProperty('error_dict')['title_nice']])
            # bug title max 170
            if len(title) >= 170:
                title = title[:167] + '...'
            bug_params['summary'] = title
            # max 16384
            description_list = self.getProperty('summary_log_list')
            del description_list[-6:]
            description_text = separator2.join(description_list)
            bug_params['description'] = description_text + separator
            bug_params['assigned_to'] = bug_config['user']
            #params['cc'] = cc
            bug_params['url'] = f"https://ci.dev.gentoo.org:8010/#/builders/7/builds/{self.getProperty('buildnumber')}"
            print(bug_params)
            # post
            aftersteps_list.append(bugs.Post(bug_args, bug_params))
            # add buildbot info
            bug_params2 = {}
            bug_params2['comment'] = separator2.join(bug_config['buildbot_comment'])
            aftersteps_list.append(bugs.Modify(bug_args, bug_params2))
            # add build info
            comment_list = []
            comment_list.append(f"Was trying to build {self.getProperty('cpv')} Commit: {self.getProperty('revision')} Repository: {self.getProperty('repository')}")
            if self.getProperty('faild_cpv') != self.getProperty('cpv'):
                comment_list.append(f"Depend: {self.getProperty('faild_cpv')} fail to build")
            comment_list.append(f"Package info:")
            for line in self.getProperty('emerge_info_output')['emerge_package_info']:
                comment_list.append(f"{line}")
            comment_list.append(f"Emerge info:")
            for line in self.getProperty('emerge_info_output')['emerge_info']:
                comment_list.append(f"{line}")
            bug_params3 = {}
            bug_params3['comment'] = separator2.join(comment_list)
            aftersteps_list.append(bugs.Modify(bug_args, bug_params3))
            # Attach build log
            file_list = []
            with os.scandir(self.getProperty('logsdir')) as entries:
                for entry in entries:
                    file_list.append(entry.name)
            print(file_list)
            for filename in file_list:
                bug_params4 = {}
                bug_params4['filename'] = yield os.path.join(self.getProperty('logsdir'), filename)
                #bug_params4['comment'] = filename
                aftersteps_list.append(bugs.Attach(bug_args, bug_params4))
            yield self.build.addStepsAfterCurrentStep(aftersteps_list)
            return SUCCESS
        return SKIPPED

class setBuildStatus(BuildStep):

    name = 'setBuildStatus'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        project_build_data = self.getProperty('project_build_data')
        bugid = 0
        if self.getProperty('bgo'):
            bugid = self.getProperty('bgo')['id']
        yield self.gentooci.db.builds.setBugIdBuilds(
                                                    project_build_data['id'],
                                                    bugid
                                                    )
        yield self.gentooci.db.builds.setStatusBuilds(
                                                    project_build_data['id'],
                                                    self.getProperty('status')
                                                    )
        if self.getProperty('status') == 'failed':
            return FAILURE
        if self.getProperty('status') == 'warning':
            return WARNINGS
        return SUCCESS