report-pr-errors

#!/usr/bin/env python
from __future__ import print_function

from argparse import ArgumentParser, Namespace
import atexit
try:
  from commands import getstatusoutput
except ImportError:
  from subprocess import getstatusoutput
from glob import glob
from os.path import dirname, join
import re
import os
import sys
import datetime

from alibot_helpers.github_utilities import calculateMessageHash, github_token
from alibot_helpers.github_utilities import setGithubStatus, parseGithubRef
from alibot_helpers.github_utilities import GithubCachedClient, PickledCache
from alibot_helpers.utilities import to_unicode

# Allow uploading logs to S3
try:
    from boto3.s3.transfer import S3Transfer
    import boto3
except:
    pass

def parse_args():
    parser = ArgumentParser()
    parser.add_argument("--work-dir", "-w", default="sw", dest="workDir")

    parser.add_argument("--default", default="release")

    parser.add_argument("--devel-prefix", "-z",
                        dest="develPrefix",
                        default="")

    parser.add_argument("--pr",
                        required=True,
                        help=("Pull request which was checked in "
                              "<org>/<project>#<nr>@ref format"))

    parser.add_argument("--no-comments",
                        action="store_true",
                        dest="noComments",
                        default=False,
                        help="Use Details button, do not post a comment")

    parser.add_argument("--success",
                        action="store_true",
                        dest="success",
                        default=False,
                        help="Signal a green status, not error")

    parser.add_argument("--status", "-s",
                        required=True,
                        help="Check which had the error")

    parser.add_argument("--dry-run", "-n",
                        action="store_true",
                        default=False,
                        help="Do not actually comment")

    parser.add_argument("--limit", "-l",
                        default=50,
                        help="Max number of lines from the report")

    parser.add_argument("--message", "-m",
                        dest="message",
                        help="Message to be posted")

    parser.add_argument("--logs-dest",
                        dest="logsDest",
                        default="rsync://repo.marathon.mesos/store/logs",
                        help="Destination store for logs. Either rsync://<rsync server enpoint> or s3://<bucket>.<server>")

    parser.add_argument("--log-url",
                        dest="logsUrl",
                        default="https://ali-ci.cern.ch/repo/logs",
                        help="Destination path for logs")

    parser.add_argument("--debug", "-d",
                        action="store_true",
                        default=False,
                        help="Turn on debug output")

    args = parser.parse_args()
    if "#" not in args.pr:
        parser.error("You need to specify a pull request")
    if "@" not in args.pr:
        parser.error("You need to specify a commit this error refers to")
    return args


class Logs(object):
    def __init__(self, args, is_branch):
        self.work_dir = args.workDir
        self.develPrefix = args.develPrefix
        self.limit = args.limit
        self.norm_status = re.sub('[^a-zA-Z0-9_-]', '_', args.status)
        self.full_log = self.constructFullLogName(args.pr)
        self.is_branch = is_branch
        self.full_log_latest = self.constructFullLogName(args.pr, latest=True)
        self.dest = args.logsDest
        self.url = join(args.logsUrl, self.full_log)

    def parse(self):
        self.find()
        self.grep()
        self.cat(self.full_log)
        if self.is_branch: self.cat(self.full_log_latest, no_delete=True)
        if self.dest.startswith("rsync:"):
            self.rsync(self.dest)
        elif self.dest.startswith("s3"):
            self.s3Upload(self.dest)
        else:
            print("Unknown destination url %s" % self.dest)

    def constructFullLogName(self, pr, latest=False):
        # file to which we cat all the individual logs
        pr = parse_pr(pr)
        return join(pr.repo_name, pr.id, "latest" if latest else pr.commit, self.norm_status, "fullLog.txt")

    def find(self):
        # Python's glob * matches at least one char
        search_paths = [join(self.work_dir, "BUILD/*latest*/log"),
                        join(self.work_dir, "BUILD/*latest/log")]
        print("Searching all logs matching: %s" % ", ".join(search_paths), file=sys.stderr)
        globbed = []
        for sp in search_paths:
            globbed.extend(glob(sp))

        suffix = ("latest" + "-" + self.develPrefix).strip("-")
        logs = {x for x in globbed if dirname(x).endswith(suffix)}

        print("Found:\n%s" % "\n".join(logs), file=sys.stderr)
        self.logs = logs

    def grep(self):
        """Grep for errors in the build logs, or, if none are found,
        return the last N lines where N is the limit argument.
        """
        error_log = ""
        for log in self.logs:
            cmd = "cat %s | grep -e ': error:' -A 3 -B 3 " % log
            cmd += "|| tail -n %s %s" % (self.limit, log)
            err, out = getstatusoutput(cmd)
            if err:
                print("Error while parsing logs", file=sys.stderr)
                print(out, file=sys.stderr)
                continue

            error_log += log + "\n"
            error_log += out

        error_log = "\n".join(error_log.split("\n")[0:self.limit])
        error_log.strip(" \n\t")
        self.error_log = error_log

    def cat(self, tgtFile, no_delete=False):
        cmd = "%s && mkdir -p `dirname copy-logs/%s`" % ("true" if no_delete else "rm -rf copy-logs", tgtFile)
        err, out = getstatusoutput(cmd)
        if err:
            print(out, file=sys.stderr)

        cmd = "echo 'Builing on: '`hostname` >> copy-logs/%s" % tgtFile
        print(cmd, file=sys.stderr)
        err, out = getstatusoutput(cmd)
        print(out, file=sys.stderr)

        if len(self.logs):
          cmd = "echo 'The following files are present in the log:' >> copy-logs/%s" % tgtFile
          print(cmd, file=sys.stderr)
          err, out = getstatusoutput(cmd)
          print(out, file=sys.stderr)
        else:
          cmd = "echo 'No logs found. Please check actual builders log in aurora.\nSee http://alisw.github.io/infrastructure-pr-testing for more instructions.' >> copy-logs/%s" % tgtFile
          print(cmd, file=sys.stderr)
          err, out = getstatusoutput(cmd)
          print(out, file=sys.stderr)

        for log in self.logs:
            cmd = "echo '- %s' >> copy-logs/%s" % (log, tgtFile)
            print(cmd, file=sys.stderr)
            err, out = getstatusoutput(cmd)
            print(out, file=sys.stderr)

        for log in self.logs:
            cmd = "echo '## Begin %s' >> copy-logs/%s" % (log, tgtFile)
            print(cmd, file=sys.stderr)
            err, out = getstatusoutput(cmd)
            print(out, file=sys.stderr)
            cmd = "cat %s >> copy-logs/%s" % (log, tgtFile)
            print(cmd, file=sys.stderr)
            err, out = getstatusoutput(cmd)
            print(out, file=sys.stderr)
            cmd = "echo '## End %s' >> copy-logs/%s" % (log, tgtFile)
            print(cmd, file=sys.stderr)
            err, out = getstatusoutput(cmd)
            print(out, file=sys.stderr)

    def rsync(self, dest):
        err, out = getstatusoutput("cd copy-logs && rsync -av ./ %s" % dest)
        if err:
            print("Error while copying logs to store.", file=sys.stderr)
            print(out, file=sys.stderr)

    def s3Upload(self, dest):
        m = re.compile("^s3://([^.]+)[.](.*)").match(dest)
        bucket_name = m.group(1)
        server = m.group(2)
        s3_client = boto3.client('s3',
                                 aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
                                 aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
                                 endpoint_url="https://%s" % (server))
        s3_client.list_buckets()
        transfer = S3Transfer(s3_client)
        matches = []
        for root, dirnames, filenames in os.walk('copy-logs'):
            for filename in filenames:
                matches.append(os.path.join(root, filename))
        for src in matches:
            try:
                dst = re.compile('^copy-logs/').sub('', src)
                transfer.upload_file(src, bucket_name, dst, extra_args={'ContentType': 'text/plain',
                                                                        'ContentDisposition': 'inline'})
            except Exception as e:
                print("Failed to upload %s to %s in bucket %s.%s" % (src, dst, bucket_name, server))

def handle_branch(cgh, pr, logs, args):
    # pr_id in this case is in fact a branch name
    branch = cgh.get("/repos/{repo_name}/branches/{branch}",
                     repo_name=pr.repo_name,
                     branch=pr.id)

    sha = branch["commit"]["sha"]

    message = "Error while checking %s for %s:\n" % (args.status, sha)
    if args.message:
        message += to_unicode(args.message)
    else:
        message += "```\n%s\n```\nFull log [here](%s).\n" % (to_unicode(logs.error_log), to_unicode(logs.url))
    messageSha = calculateMessageHash(message)

    ns = Namespace(commit=args.pr,
                   status=args.status + "/error",
                   message="",
                   url="")
    setGithubStatus(cgh, ns)
    sys.exit(0)


def handle_pr_id(cgh, pr, logs, args):
    commit = cgh.get("/repos/{repo_name}/commits/{ref}",
                     repo_name=pr.repo_name,
                     ref=pr.commit)
    sha = commit["sha"]

    message = "Error while checking %s for %s at %s:\n" % (args.status, sha, datetime.datetime.now().strftime("%Y-%m-%d %H:%M"))
    if args.message:
        message += args.message
    else:
        message += "```\n%s\n```\nFull log [here](%s).\n" % (to_unicode(logs.error_log), to_unicode(logs.url))

    if args.dry_run:
        # commit does not exist...
        print("Will annotate %s" % commit["sha"])
        print(message)
        sys.exit(0)

    # Set status
    ghStatus = "/success" if args.success else "/error"
    ns = Namespace(commit=args.pr, status=args.status + ghStatus, message="", url=logs.url)
    setGithubStatus(cgh, ns)

    # Comment if appropriate
    if args.noComments or args.success:
        return

    prIssueComments = cgh.get("/repos/{repo_name}/issues/{pr_id}/comments",
                              repo_name=pr.repo_name,
                              pr_id=pr.id)

    messageHash = calculateMessageHash(message)
    for comment in prIssueComments:
        if comment["body"].startswith("Error while checking %s for %s" % (args.status, sha)):
            if calculateMessageHash(comment["body"]) != messageHash:
                print("Comment was different. Updating", file=sys.stderr)
                cgh.patch(
                    "/repos/{repo_name}/issues/comments/{commentID}",
                    {"body": message},
                    repo_name=pr.repo_name,
                    commentID=comment["id"]
                )
                sys.exit(0)

            print("Found same comment for the same commit", file=sys.stderr)
            sys.exit(0)


    cgh.post(
        "repos/{repo_name}/issues/{pr_id}/comments",
        {"body": message},
        repo_name=pr.repo_name,
        pr_id=pr.id
    )


def parse_pr(pr):
    repo_name, pr_id, pr_commit = parseGithubRef(pr)
    return Namespace(repo_name=repo_name,
                     id=pr_id,
                     commit=pr_commit)


def main():
    args = parse_args()
    pr = parse_pr(args.pr)
    logs = Logs(args, is_branch=not pr.id.isdigit())
    if not args.message:
        logs.parse()

    cache = PickledCache('.cached-commits')
    with GithubCachedClient(token=github_token(), cache=cache) as cgh:
        # If the branch is not a PR, we should look for open issues
        # for the branch. This should really folded as a special case
        # of the PR case.
        func = handle_branch if not pr.id.isdigit() else handle_pr_id
        func(cgh, pr, logs, args)

    cgh.printStats()

if __name__ == "__main__":
    main()