build.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Copyright (c) 2019 The ungoogled-chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
ungoogled-chromium build script for Microsoft Windows
"""

import sys
import time

if sys.version_info.major < 3 or sys.version_info.minor < 6:
    raise RuntimeError('Python 3.6+ is required for this script. You have: {}.{}'.format(
        sys.version_info.major, sys.version_info.minor))

import argparse
import os
import re
import shutil
import subprocess
import ctypes
from pathlib import Path
from datetime import  datetime

import distutils
from distutils.dir_util import copy_tree

sys.path.insert(0, str(Path(__file__).resolve().parent / 'ungoogled-chromium' / 'utils'))
import downloads
#import domain_substitution
import replace_product_name
import prune_binaries
import patches
from _common import ENCODING, USE_REGISTRY, ExtractorEnum, get_logger
sys.path.pop(0)

_ROOT_DIR = Path(__file__).resolve().parent
_PATCH_BIN_RELPATH = Path('third_party/git/usr/bin/patch.exe')

def _backup_patched_files():
    with open('affected_files.txt', 'r') as file:
            Lines = file.readlines()
            # Strips the newline character
            for line in Lines:
                line = line.replace('/', '\\')
                line = line.replace('\n', '')
                relative_path = 'build\\patched_files\\'
                if '\\' in line:
                    file_relative_path = line[:line.rfind('\\')]
                    relative_path +=  file_relative_path

                if not os.path.exists(relative_path):
                    os.makedirs(relative_path)
                    
                try:
                    shutil.copyfile('build\\src\\' + line, 'build\\patched_files\\' + line)
                except Exception as ex:
                    print ("Error trying to backup patched file:", str(ex))
                    pass
                
def _backup_pruned_files():
    '''
        with open(_ROOT_DIR / 'ungoogled-chromium' / 'pruning.list', 'r') as file:
            Lines = file.readlines()
            # Strips the newline character
            for line in Lines:
                line = line.replace('/', '\\')
                line = line.replace('\n', '')
                shutil.copyfile('build\\src\\' + line, 'build\\prunned_binaries\\' + line)
        '''
    with open('ungoogled-chromium\\pruning.list', 'r') as file:
            Lines = file.readlines()
            # Strips the newline character
            for line in Lines:
                line = line.replace('/', '\\')
                line = line.replace('\n', '')
                relative_path = 'build\\pruned_files\\'
                if '\\' in line:
                    file_relative_path = line[:line.rfind('\\')]
                    relative_path +=  file_relative_path

                if not os.path.exists(relative_path):
                    os.makedirs(relative_path)
                    
                try:
                    shutil.copyfile('build\\src\\' + line, 'build\\pruned_files\\' + line)
                except Exception as ex:
                    print ("Error trying to backup pruned file:", str(ex))
                
def _restore_patched_files():
    copy_tree('build\\patched_files', 'build\\src')
    
    with open('patches_created_files.txt', 'r') as file:
            Lines = file.readlines()
            # Strips the newline character
            for line in Lines:
                line = line.replace('/', '\\')
                line = line.replace('\n', '')
                relative_path = 'build\\src\\' + line
                try:
                    os.remove(relative_path)
                except Exception as ex:
                    print ("Error trying to remove created file from patches:", str(ex))
                    pass
    # os.remove('build\\src\\chrome\\browser\\bromite_flag_choices.h')
    # os.remove('build\\src\\chrome\\browser\\bromite_flag_entries.h')
    # os.remove('build\\src\\chrome\\browser\\croma_flag_choices.h')
    # os.remove('build\\src\\chrome\\browser\\croma_flag_entries.h')
    # os.remove('build\\src\\chrome\\browser\\croma_platform_flag_choices.h')
    # os.remove('build\\src\\chrome\\browser\\croma_platform_flag_entries.h')
    # os.remove('build\\src\\components\\croma\\BUILD.gn')
    # os.remove('build\\src\\components\\croma\\ungoogled_switches.h')
    # os.remove('build\\src\\components\\croma\\ungoogled_switches.cc')

def _get_vcvars_path(name='64'):
    """
    Returns the path to the corresponding vcvars*.bat path

    As of VS 2017, name can be one of: 32, 64, all, amd64_x86, x86_amd64
    """
    vswhere_exe = '%ProgramFiles(x86)%\\Microsoft Visual Studio\\Installer\\vswhere.exe'
    result = subprocess.run(
        '"{}" -prerelease -latest -property installationPath'.format(vswhere_exe),
        shell=True,
        check=True,
        stdout=subprocess.PIPE,
        universal_newlines=True)
    vcvars_path = Path(result.stdout.strip(), 'VC/Auxiliary/Build/vcvars{}.bat'.format(name))
    if not vcvars_path.exists():
        raise RuntimeError(
            'Could not find vcvars batch script in expected location: {}'.format(vcvars_path))
    return vcvars_path


def _run_build_process(*args, **kwargs):
    """
    Runs the subprocess with the correct environment variables for building
    """
    # Add call to set VC variables
    cmd_input = ['call "%s" >nul' % _get_vcvars_path()]
    cmd_input.append('set DEPOT_TOOLS_WIN_TOOLCHAIN=0')
    cmd_input.append(' '.join(map('"{}"'.format, args)))
    cmd_input.append('exit\n')
    subprocess.run(('cmd.exe', '/k'),
                   input='\n'.join(cmd_input),
                   check=True,
                   encoding=ENCODING,
                   **kwargs)


def _run_build_process_timeout(*args, timeout):
    """
    Runs the subprocess with the correct environment variables for building
    """
    # Add call to set VC variables
    cmd_input = ['call "%s" >nul' % _get_vcvars_path()]
    cmd_input.append('set DEPOT_TOOLS_WIN_TOOLCHAIN=0')
    cmd_input.append(' '.join(map('"{}"'.format, args)))
    cmd_input.append('exit\n')
    with subprocess.Popen(('cmd.exe', '/k'), encoding=ENCODING, stdin=subprocess.PIPE, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) as proc:
        proc.stdin.write('\n'.join(cmd_input))
        proc.stdin.close()
        try:
            proc.wait(timeout)
            if proc.returncode != 0:
                raise RuntimeError('Build failed!')
        except subprocess.TimeoutExpired:
            print('Sending keyboard interrupt')
            for _ in range(3):
                ctypes.windll.kernel32.GenerateConsoleCtrlEvent(1, proc.pid)
                time.sleep(1)
            try:
                proc.wait(10)
            except:
                proc.kill()
            raise KeyboardInterrupt


def _make_tmp_paths():
    """Creates TMP and TEMP variable dirs so ninja won't fail"""
    tmp_path = Path(os.environ['TMP'])
    if not tmp_path.exists():
        tmp_path.mkdir()
    tmp_path = Path(os.environ['TEMP'])
    if not tmp_path.exists():
        tmp_path.mkdir()


def main():
    """CLI Entrypoint"""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        '--disable-ssl-verification',
        action='store_true',
        help='Disables SSL verification for downloading')
    parser.add_argument(
        '--7z-path',
        dest='sevenz_path',
        default=USE_REGISTRY,
        help=('Command or path to 7-Zip\'s "7z" binary. If "_use_registry" is '
              'specified, determine the path from the registry. Default: %(default)s'))
    parser.add_argument(
        '--winrar-path',
        dest='winrar_path',
        default=USE_REGISTRY,
        help=('Command or path to WinRAR\'s "winrar.exe" binary. If "_use_registry" is '
              'specified, determine the path from the registry. Default: %(default)s'))
    parser.add_argument(
        '--ci',
        action='store_true'
    )
    parser.add_argument(
        '--x86',
        action='store_true'
    )
    parser.add_argument(
        '--j',
        choices = [str(x) for x in range(1, 17)]
    )
    args = parser.parse_args()

    # Set common variables
    source_tree = _ROOT_DIR / 'build' / 'src'
    downloads_cache = _ROOT_DIR / 'build' / 'download_cache'

    if not args.ci or not (source_tree / 'BUILD.gn').exists():
        
        if not source_tree.exists():
            # Setup environment
            source_tree.mkdir(parents=True, exist_ok=True)
            downloads_cache.mkdir(parents=True, exist_ok=True)

            _make_tmp_paths()

            # Get download metadata (DownloadInfo)
            download_info = downloads.DownloadInfo([
                _ROOT_DIR / 'downloads.ini',
                _ROOT_DIR / 'ungoogled-chromium' / 'downloads.ini',
            ])

            # Retrieve downloads
            get_logger().info('Downloading required files...')
            downloads.retrieve_downloads(download_info, downloads_cache, True,
                                                args.disable_ssl_verification)
            try:
                downloads.check_downloads(download_info, downloads_cache)
            except downloads.HashMismatchError as exc:
                get_logger().error('File checksum does not match: %s', exc)
                exit(1)

            # Unpack downloads
            extractors = {
                ExtractorEnum.SEVENZIP: args.sevenz_path,
                ExtractorEnum.WINRAR: args.winrar_path,
            }
            get_logger().info('Unpacking downloads...')
            downloads.unpack_downloads(download_info, downloads_cache, source_tree, extractors)

        if not os.path.exists(_ROOT_DIR / 'build' / 'patched_files'):
            os.makedirs(_ROOT_DIR / 'build' / 'patched_files')
            _backup_patched_files()
                    
        if not os.path.exists(_ROOT_DIR / 'build' / 'pruned_files'):
            os.makedirs(_ROOT_DIR / 'build' / 'pruned_files')
            _backup_pruned_files()

        # Prune binaries
        # unremovable_files = prune_binaries.prune_dir(
        #     source_tree,
        #     (_ROOT_DIR / 'ungoogled-chromium' / 'pruning.list').read_text(encoding=ENCODING).splitlines()
        # )
        # if unremovable_files:
        #     #get_logger().error('Files could not be pruned: %s', unremovable_files)
        #     get_logger().error('Files could not be pruned')
        #     #parser.exit(1)
        

        try:
            # Apply patches
            # First, ungoogled-chromium-patches
            patches.apply_patches(
                patches.generate_patches_from_series(_ROOT_DIR / 'ungoogled-chromium' / 'patches', resolve=True),
                source_tree,
                patch_bin_path=(source_tree / _PATCH_BIN_RELPATH)
            )
            # Then Windows-specific patches
            patches.apply_patches(
                patches.generate_patches_from_series(_ROOT_DIR / 'patches', resolve=True),
                source_tree,
                patch_bin_path=(source_tree / _PATCH_BIN_RELPATH)
            )
        except:
            _restore_patched_files()
            raise
        # Substitute domains
        # domain_substitution.apply_substitution(
        #     _ROOT_DIR / 'ungoogled-chromium' / 'domain_regex.list',
        #     _ROOT_DIR / 'ungoogled-chromium' / 'domain_substitution.list',
        #     source_tree,
        #     domsubcache
        # )

        # Substitute Chromium names
        print("Start replacing Chromium name")
        replace_product_name.apply_substitution(
           source_tree
        )

        copy_tree(str(_ROOT_DIR / 'croma-build'), str(source_tree))

    if not args.ci or not (source_tree / 'out/Default').exists():
        # Output args.gn
        (source_tree / 'out/Default').mkdir(parents=True)
        gn_flags = (_ROOT_DIR / 'ungoogled-chromium' / 'flags.gn').read_text(encoding=ENCODING)
        gn_flags += '\n'
        windows_flags = (_ROOT_DIR / 'flags.windows.gn').read_text(encoding=ENCODING)
        if args.x86:
            windows_flags = windows_flags.replace('x64', 'x86')
        gn_flags += windows_flags
        (source_tree / 'out/Default/args.gn').write_text(gn_flags, encoding=ENCODING)

    # Enter source tree to run build commands
    os.chdir(source_tree)

    if not args.ci or not os.path.exists('out\\Default\\gn.exe'):
        # Run GN bootstrap
        _run_build_process(
            sys.executable, 'tools\\gn\\bootstrap\\bootstrap.py', '-o', 'out\\Default\\gn.exe',
            '--skip-generate-buildfiles')

        # Run gn gen
        _run_build_process('out\\Default\\gn.exe', 'gen', 'out\\Default', '--fail-on-unused-args')

    # Run ninja
  
    if args.ci:
        if args.j:
            print(args)
            _run_build_process_timeout('third_party\\ninja\\ninja.exe', '-j' + args.j, '-C', 'out\\Default', 'chrome',
                                    'chromedriver', 'mini_installer', timeout = 45*60*60)
        else:
            _run_build_process_timeout('third_party\\ninja\\ninja.exe', '-C', 'out\\Default', 'chrome',
                                    'chromedriver', 'mini_installer', timeout = 45*60*60)
        # package
        os.chdir(_ROOT_DIR)
        subprocess.run([sys.executable, 'package.py'])
        print(datetime.now().strftime("%m/%d/%Y, %H:%M:%S"))
    else:
        _run_build_process('third_party\\ninja\\ninja.exe', '-C', 'out\\Default', 'chrome',
                           'chromedriver', 'mini_installer')


if __name__ == '__main__':
    main()