run_sk_stress_test

#!/usr/bin/env python
# ===--- run_sk_stress_test -----------------------------------------------===
#
#  This source file is part of the Swift.org open source project
#
#  Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
#  Licensed under Apache License v2.0 with Runtime Library Exception
#
#  See https://swift.org/LICENSE.txt for license information
#  See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ===----------------------------------------------------------------------===

"""A run script to be executed as a Jenkins build step."""

from __future__ import print_function
import sys
import os
import re
import json
import subprocess
import argparse
import platform
import codecs

import common

script_dir = os.path.abspath(os.path.dirname(__file__))


def main():
    if platform.system() != 'Darwin':
        raise common.UnsupportedPlatform

    common.debug_print('** RUN SOURCEKIT STRESS TESTER **')
    os.chdir(os.path.dirname(__file__))

    args = parse_args()
    common.set_swift_branch(args.swift_branch)
    workspace = common.private_workspace('.')

    if not args.skip_tools_clone:
        common.clone_repos()
        clone_swift_syntax(workspace, args.swift_branch)
        clone_stress_tester(workspace, args.swift_branch)

    if not args.skip_tools_build:
        build_swift_toolchain(workspace, args)

    if not args.skip_runner:
        if not execute_runner(workspace, args):
            return 1

    return 0


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('swift_branch')
    parser.add_argument('--sandbox', action='store_true')
    parser.add_argument('--projects',
                        metavar='PATH',
                        help='JSON project file',
                        default='projects.json')
    parser.add_argument('--filter-by-tag',
                        metavar='TAG',
                        help='Only run project actions with the given tag',
                        default='sourcekit')
    parser.add_argument('--sourcekit-xfails',
                        metavar='PATH',
                        help='JSON file specifying expected sourcekit failures',
                        default='sourcekit-xfails.json')
    parser.add_argument('--verbose',
                        action='store_true')
    parser.add_argument('--assertions',
                        help='Build Swift with asserts',
                        action='store_true')
    parser.add_argument('--debug',
                        help='Build Swift in debug mode',
                        action='store_true')
    parser.add_argument('--swiftc',
                        metavar='PATH',
                        help='swiftc executable')
    parser.add_argument('--skip-tools-build',
                        action='store_true')
    parser.add_argument('--skip-ci-steps',
                        action='store_true')
    parser.add_argument('--skip-tools-clone',
                        action='store_true')
    parser.add_argument('--skip-runner',
                        action='store_true')
    parser.add_argument('--add-swift-flags',
                        metavar='FLAGS',
                        help='add flags to each Swift invocation (note: field '
                             'names from projects.json enclosed in {} will be '
                             'replaced with their value)',
                        default='')
    parser.add_argument('--add-xcodebuild-flags',
                        metavar='FLAGS',
                        help='add flags to each xcodebuild invocation (note: field '
                             'names from projects.json enclosed in {} will be '
                             'replaced with their value)',
                        default='')
    return parser.parse_args()

def get_swiftc_path(workspace, swiftc):
    swiftc_path = (
        swiftc if swiftc else
        os.path.join(workspace, 'build/compat_macos/install/toolchain/usr/bin/swiftc')
    )
    return swiftc_path

def get_sandbox_profile_flags():
    return [
        '--sandbox-profile-xcodebuild',
        '../../../workspace-private/swift-source-compat-suite-sandbox/sandbox_xcodebuild.sb',
        '--sandbox-profile-package',
        '../../../workspace-private/swift-source-compat-suite-sandbox/sandbox_package.sb'
    ]

def clone_stress_tester(workspace, swift_branch):
    stress_clone_cmd = [
        'git','clone', '-q', '-b', swift_branch, '--recursive',
        'https://github.com/apple/swift-stress-tester',
        '{}/swift-stress-tester'.format(workspace)
    ]
    common.check_execute(stress_clone_cmd, timeout=-1)

def clone_swift_syntax(workspace, swift_branch):
    syntax_clone_cmd = [
        'git','clone', '-q', '-b', swift_branch, '--recursive',
        'https://github.com/apple/swift-syntax',
        '{}/swift-syntax'.format(workspace)
    ]
    common.check_execute(syntax_clone_cmd, timeout=-1)


def execute_runner(workspace, args):
    swiftc_path = get_swiftc_path(workspace, args.swiftc)
    wrapper_path = os.path.join(os.path.dirname(swiftc_path), 'sk-swiftc-wrapper')
    stress_tester_path = os.path.join(os.path.dirname(swiftc_path), 'sk-stress-test')

    extra_runner_args = []
    if args.sandbox:
        extra_runner_args += get_sandbox_profile_flags()

    if args.add_swift_flags:
        extra_runner_args += ['--add-swift-flags=%s' % args.add_swift_flags]

    if args.add_xcodebuild_flags:
        extra_runner_args += ['--add-xcodebuild-flags=%s' % args.add_xcodebuild_flags]

    if args.filter_by_tag:
        extra_runner_args += ['--include-actions', '"tags" in locals() and "{}" in tags.split()'.format(args.filter_by_tag)]

    runner = StressTesterRunner(wrapper_path, stress_tester_path, swiftc_path, args.projects, args.swift_branch, os.path.abspath(args.sourcekit_xfails))
    passed = runner.run(extra_runner_args)

    return passed


def build_swift_toolchain(workspace, args):
    build_command = [
        os.path.join(workspace, 'swift/utils/build-script'),
        '--debug' if args.debug else '--release',
        '--assertions' if args.assertions else '--no-assertions',
        '--build-ninja',
        '--llbuild',
        '--swiftpm',
        '--swiftsyntax',
        '--skstresstester',
        '--ios',
        '--tvos',
        '--watchos',
        '--skip-build-benchmarks',
        '--build-subdir=compat_macos',
        '--compiler-vendor=apple',
        '--',
        '--darwin-install-extract-symbols',
        '--darwin-toolchain-alias=swift',
        '--darwin-toolchain-bundle-identifier=org.swift.compat-macos',
        '--darwin-toolchain-display-name-short=Swift Development Snapshot'
        '--darwin-toolchain-display-name=Swift Development Snapshot',
        '--darwin-toolchain-name=swift-DEVELOPMENT-SNAPSHOT',
        '--darwin-toolchain-version=3.999.999',
        '--install-llbuild',
        '--install-swift',
        '--install-swiftpm',
        '--install-swiftsyntax',
        '--install-skstresstester',
        '--install-destdir={}/build/compat_macos/install'.format(workspace),
        '--install-prefix=/toolchain/usr',
        '--install-symroot={}/build/compat_macos/symroot'.format(workspace),
        '--installable-package={}/build/compat_macos/root.tar.gz'.format(workspace),
        '--llvm-install-components=libclang;libclang-headers;dsymutil',
        '--swift-install-components=compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;license;sourcekit-xpc-service;swift-remote-mirror;swift-remote-mirror-headers',
        '--symbols-package={}/build/compat_macos/root-symbols.tar.gz'.format(workspace),
        '--verbose-build',
        '--reconfigure',
    ]
    common.check_execute(build_command, timeout=9999999)


def processed_files_contain(processed_files, file_path):
        """
        Returns `True` if a path in `processed_files` contains `file_path` as a substring, `False` otherwise
        """
        for processed_file in processed_files:
            if file_path in processed_file:
                return True
        return False


class StressTesterRunner(object):
    """sets up the Swift compatibility suite runner to use the stress tester's swiftc-wrapper, executes it, and processes its output for failures."""

    def __init__(self, wrapper, stress_tester, swiftc, projects, branch, xfails):
        self.wrapper = wrapper
        self.stress_tester = stress_tester
        self.swiftc = swiftc

        self.xfails_path = xfails
        self.projects_path = projects
        self.swift_branch = branch

        self.compat_runner_failed = False


    def run(self, extra_runner_args=[]):
        # temporary file paths
        filtered_projects = os.path.join(script_dir, 'stress-tester-projects.json')
        results = os.path.join(script_dir, 'stress-tester-results.json')

        # remove temporary files if they already exist
        self._cleanup([filtered_projects, results])

        run_env = {
            'SK_STRESS_TEST': self.stress_tester,
            'SK_STRESS_SWIFTC': self.swiftc,
            'SK_STRESS_SILENT': 'true',
            'SK_STRESS_AST_BUILD_LIMIT': '1000',
            'SK_STRESS_OUTPUT': results,
            'SK_XFAILS_PATH': self.xfails_path,
            'SK_STRESS_ACTIVE_CONFIG': self.swift_branch,
            'SK_STRESS_SUPPRESS_OUTPUT': 'true',
            'SK_STRESS_REWRITE_MODES': 'none concurrent insideout'}
        run_env.update(os.environ)
        run_cmd = ['./runner.py',
          '--projects', filtered_projects,
          '--verbose',
          '--swiftc', self.wrapper,
          '--swift-branch', self.swift_branch,
          '--default-timeout', str(-1),
          '--only-latest-versions',
          # archs override is set to arm64 for generic/iOS actions that would otherwise invoke the stress tester for both arm64 and armv7
          '--add-xcodebuild-flags', 'ARCHS={archs_override}']

        if extra_runner_args:
            run_cmd.extend(extra_runner_args)

        self._filter_projects(filtered_projects)
        try:
            common.check_execute(run_cmd, timeout=-1, env=run_env)
        except common.ExecuteCommandFailure:
            self.compat_runner_failed = True

        success = self._process_output(results, self.xfails_path)

        return success


    def _process_output(self, results_path, xfails_path):
        if not os.path.isfile(results_path):
            return True

        with open(results_path, 'rb') as results_file:
            results = json.load(results_file, encoding='utf-8')
        with open(xfails_path, 'rb') as xfails_path:
            xfails = json.load(xfails_path, encoding='utf-8')

        xfails_not_processed = []
        for xfail in xfails:
            # The XFails start with '*' to match any path prefix. Remove it so we can perform substring matching in processed_files_contain
            xfailed_file = xfail['path']
            if xfailed_file[0] == '*':
                xfailed_file = xfailed_file[1:]
            if not processed_files_contain(results['processedFiles'], xfailed_file):
                xfails_not_processed.append(xfail)

        num_failures = len(results['issues'])
        num_xfails = sum(len(value) for _, value in results['expectedIssues'].iteritems())
        num_xfail_issues = len(results['expectedIssues'])
        unmatched = results['unmatchedExpectedIssues']

        success = num_failures == 0 and len(unmatched) == 0

        if num_xfails > 0:
            print('Expected stress tester issues:')
            for url, issues in results['expectedIssueMessages'].iteritems():
                for (index, issue) in enumerate(issues):
                    self._print_issue(index, issue, url)
            print('\n========================================')

        if len(xfails_not_processed) > 0:
            print('XFails not processed:')
            print('The files that are covered by these XFails were not tested in the current Stress Tester Run.')
            print('The reason for this might be:')
            print(' * The project that contains the file was not tested in the current run')
            print(' * A different file from the same compiler invocation failed and thus this file was not tested')
            print(' * The file has been removed in the original project')
            print('')
            for xfail in xfails_not_processed:
                print('      {}'.format(json.dumps(xfail)))
            print('\n========================================')

        if num_failures > 0:
            print('Unexpected stress tester issues:')
            for (index, issue) in enumerate(results['issueMessages']):
                self._print_issue(index, issue)
            print('\n========================================')


        print('SourceKit Stress Tester summary:')

        print('  {} underlying source compatibility build'.format('failed' if self.compat_runner_failed else 'passed'))
        if self.compat_runner_failed:
            print('      > treat this as a source compatibility failure')

        print('  {} XFails not processed'.format(len(xfails_not_processed)))
        if num_failures > 0:
            print('      > see "XFails not processed" how to handle them. This is an info, not an error')

        print('  {} unexpected stress tester failures'.format(num_failures))
        if num_failures > 0:
            print('      > see "Unexpected stress tester issues" above for individual failures')

        print('  {} expected stress tester failures tracked by {} issues'.format(num_xfails, num_xfail_issues))

        if not self.compat_runner_failed and unmatched:
            print('  {} expected stress tester failures not seen'.format(len(unmatched)))
            print('      > if resolved, remove "{}" from "branches" in the following entries in {} or the entire entry if no branches remain:'.format(self.swift_branch, self.xfails_path))
            for xfail in unmatched:
                print('      {}'.format(json.dumps(xfail)))

        print('========================================')
        print('Result: {result}'.format(result=('PASS' if success else 'FAIL')))
        print('========================================')

        return success

    @staticmethod
    def _print_issue(index, issue, url = None):
        if url != None:
            print(u'\n{}. [{}] {}'.format(index + 1, url, issue))
        else:
            print(u'\n{}. {}'.format(index + 1, issue))


    def _filter_projects(self, output):
        with open(self.projects_path) as json_file:
            projects = json.load(json_file)
            for project in projects:
                for action in project['actions']:
                    # This is used in combination with --add-xcodebuild-flags
                    # to prevent running the stress tester over the same files
                    # for difference architectures. generic/iOS actions
                    # normally invoke the wrapper for both arm64 and armv7 and
                    # generic/macOS actions for arm64 and x86_64.
                    if action.get('destination') == 'generic/platform=iOS':
                        action['archs_override'] = 'arm64'
                    elif action.get('destination') == 'generic/platform=macOS':
                        action['archs_override'] = 'x86_64'
                    else:
                        action['archs_override'] = '$ARCHS'

        with open(output, 'w') as outfile:
            json.dump(projects, outfile, indent=4)
        return output

    @staticmethod
    def _cleanup(paths):
        for path in paths:
            try:
                os.remove(path)
            except OSError:
                pass

if os.isatty(sys.stdout.fileno()):
    encoding = sys.stdout.encoding
    errors = 'replace'
else:
    encoding = 'utf-8'
    errors = None
sys.stdout = codecs.getwriter(encoding)(sys.stdout, errors=errors)

if __name__ == '__main__':
    sys.exit(main())