diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index 8e73524878..a7a796f3b7 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -269,8 +269,8 @@ def PreBuildCommonSteps(Map pipelineConfig, String repositoryName, String projec unstash name: 'incremental_build_script' def pythonCmd = '' - if(env.IS_UNIX) pythonCmd = 'sudo -E python -u ' - else pythonCmd = 'python -u ' + if(env.IS_UNIX) pythonCmd = 'sudo -E python3 -u ' + else pythonCmd = 'python3 -u ' if(env.RECREATE_VOLUME?.toBoolean()) { palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume', winSlashReplacement=false) @@ -382,8 +382,8 @@ def PostBuildCommonSteps(String workspace, boolean mount = true) { if (mount) { def pythonCmd = '' - if(env.IS_UNIX) pythonCmd = 'sudo -E python -u ' - else pythonCmd = 'python -u ' + if(env.IS_UNIX) pythonCmd = 'sudo -E python3 -u ' + else pythonCmd = 'python3 -u ' try { timeout(5) { diff --git a/scripts/build/bootstrap/incremental_build_util.py b/scripts/build/bootstrap/incremental_build_util.py index 0a1dce5130..ef49a27bb1 100755 --- a/scripts/build/bootstrap/incremental_build_util.py +++ b/scripts/build/bootstrap/incremental_build_util.py @@ -7,7 +7,7 @@ import argparse import ast import boto3 import datetime -import urllib2 +import urllib.request, urllib.error, urllib.parse import os import psutil import time @@ -49,7 +49,7 @@ if os.name == 'nt': def is_dir_symlink(path): FILE_ATTRIBUTE_REPARSE_POINT = 0x0400 - return os.path.isdir(path) and (ctypes.windll.kernel32.GetFileAttributesW(unicode(path)) & FILE_ATTRIBUTE_REPARSE_POINT) + return os.path.isdir(path) and (ctypes.windll.kernel32.GetFileAttributesW(str(path)) & FILE_ATTRIBUTE_REPARSE_POINT) def get_free_space_mb(path): if sys.version_info < (3,): # Python 2? @@ -84,7 +84,7 @@ else: return st.f_bavail * st.f_frsize / 1024 / 1024 def error(message): - print message + print(message) exit(1) def parse_args(): @@ -137,19 +137,19 @@ def get_ec2_client(region): def get_ec2_instance_id(): try: - instance_id = urllib2.urlopen('http://169.254.169.254/latest/meta-data/instance-id').read() - return instance_id + instance_id = urllib.request.urlopen('http://169.254.169.254/latest/meta-data/instance-id').read() + return instance_id.decode("utf-8") except Exception as e: - print e.message + print(e.message) error('No EC2 metadata! Check if you are running this script on an EC2 instance.') def get_availability_zone(): try: - availability_zone = urllib2.urlopen('http://169.254.169.254/latest/meta-data/placement/availability-zone').read() - return availability_zone + availability_zone = urllib.request.urlopen('http://169.254.169.254/latest/meta-data/placement/availability-zone').read() + return availability_zone.decode("utf-8") except Exception as e: - print e.message + print(e.message) error('No EC2 metadata! Check if you are running this script on an EC2 instance.') @@ -158,11 +158,11 @@ def kill_processes(workspace='/dev/'): Kills all processes that have open file paths associated with the workspace. Uses PSUtil for cross-platform compatibility ''' - print 'Checking for any stuck processes...' + print('Checking for any stuck processes...') for proc in psutil.process_iter(): try: if workspace in str(proc.open_files()): - print "{} has open files in {}. Terminating".format(proc.name(), proc.open_files()) + print("{} has open files in {}. Terminating".format(proc.name(), proc.open_files())) proc.kill() time.sleep(1) # Just to make sure a parent process has time to close except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): @@ -171,7 +171,7 @@ def kill_processes(workspace='/dev/'): def delete_volume(ec2_client, volume_id): response = ec2_client.delete_volume(VolumeId=volume_id) - print 'Volume {} deleted'.format(volume_id) + print('Volume {} deleted'.format(volume_id)) def find_snapshot_id(ec2_client, repository_name, project, pipeline, platform, build_type, disk_size): mount_name = get_mount_name(repository_name, project, pipeline, 'stabilization_2106', platform, build_type) # we take snapshots out of stabilization_2106 @@ -234,29 +234,29 @@ def create_volume(ec2_client, availability_zone, repository_name, project, pipel time.sleep(1) response = ec2_client.describe_volumes(VolumeIds=[volume_id, ]) - print("Volume {} created\n\tSnapshot: {}\n\tRepository {}\n\tProject {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}" - .format(volume_id, snapshot_id, repository_name, project, pipeline, branch, platform, build_type)) + print(("Volume {} created\n\tSnapshot: {}\n\tRepository {}\n\tProject {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}" + .format(volume_id, snapshot_id, repository_name, project, pipeline, branch, platform, build_type))) return volume_id, created def mount_volume(created): - print 'Mounting volume...' + print('Mounting volume...') if os.name == 'nt': f = tempfile.NamedTemporaryFile(delete=False) f.write(""" select disk 1 online disk attribute disk clear readonly - """) # assume disk # for now + """.encode('utf-8')) # assume disk # for now if created: - print 'Creating filesystem on new volume' + print('Creating filesystem on new volume') f.write("""create partition primary select partition 1 format quick fs=ntfs assign active - """) + """.encode('utf-8')) f.close() @@ -267,7 +267,7 @@ def mount_volume(created): drives_after = win32api.GetLogicalDriveStrings() drives_after = drives_after.split('\000')[:-1] - print drives_after + print(drives_after) #drive_letter = next(item for item in drives_after if item not in drives_before) drive_letter = MOUNT_PATH @@ -284,7 +284,7 @@ def mount_volume(created): def attach_volume(volume, volume_id, instance_id, timeout=DEFAULT_TIMEOUT): - print 'Attaching volume {} to instance {}'.format(volume_id, instance_id) + print('Attaching volume {} to instance {}'.format(volume_id, instance_id)) volume.attach_to_instance(Device='xvdf', InstanceId=instance_id, VolumeId=volume_id) @@ -297,7 +297,7 @@ def attach_volume(volume, volume_id, instance_id, timeout=DEFAULT_TIMEOUT): time.sleep(1) volume.load() if (time.clock() - timeout_init) > timeout: - print 'ERROR: Timeout reached trying to mount EBS' + print('ERROR: Timeout reached trying to mount EBS') exit(1) volume.create_tags( Tags=[ @@ -307,11 +307,11 @@ def attach_volume(volume, volume_id, instance_id, timeout=DEFAULT_TIMEOUT): }, ] ) - print 'Volume {} has been attached to instance {}'.format(volume_id, instance_id) + print('Volume {} has been attached to instance {}'.format(volume_id, instance_id)) def unmount_volume(): - print 'Umounting volume...' + print('Umounting volume...') if os.name == 'nt': kill_processes(MOUNT_PATH + 'workspace') f = tempfile.NamedTemporaryFile(delete=False) @@ -328,7 +328,7 @@ def unmount_volume(): def detach_volume(volume, ec2_instance_id, force, timeout=DEFAULT_TIMEOUT): - print 'Detaching volume {} from instance {}'.format(volume.volume_id, ec2_instance_id) + print('Detaching volume {} from instance {}'.format(volume.volume_id, ec2_instance_id)) volume.detach_from_instance(Device='xvdf', Force=force, InstanceId=ec2_instance_id, @@ -338,16 +338,16 @@ def detach_volume(volume, ec2_instance_id, force, timeout=DEFAULT_TIMEOUT): time.sleep(1) volume.load() if (time.clock() - timeout_init) > timeout: - print 'ERROR: Timeout reached trying to unmount EBS.' + print('ERROR: Timeout reached trying to unmount EBS.') volume.detach_from_instance(Device='xvdf',Force=True,InstanceId=ec2_instance_id,VolumeId=volume.volume_id) exit(1) - print 'Volume {} has been detached from instance {}'.format(volume.volume_id, ec2_instance_id) + print('Volume {} has been detached from instance {}'.format(volume.volume_id, ec2_instance_id)) volume.load() if len(volume.attachments): - print 'Volume still has attachments' + print('Volume still has attachments') for attachment in volume.attachments: - print 'Volume {} {} to instance {}'.format(attachment['VolumeId'], attachment['State'], attachment['InstanceId']) + print('Volume {} {} to instance {}'.format(attachment['VolumeId'], attachment['State'], attachment['InstanceId'])) def attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created): @@ -379,10 +379,10 @@ def mount_ebs(repository_name, project, pipeline, branch, platform, build_type, for volume in ec2_instance.volumes.all(): for attachment in volume.attachments: - print 'attachment device: {}'.format(attachment['Device']) + print('attachment device: {}'.format(attachment['Device'])) if 'xvdf' in attachment['Device'] and attachment['State'] != 'detached': - print 'A device is already attached to xvdf. This likely means a previous build failed to detach its ' \ - 'build volume. This volume is considered orphaned and will be detached from this instance.' + print('A device is already attached to xvdf. This likely means a previous build failed to detach its ' \ + 'build volume. This volume is considered orphaned and will be detached from this instance.') unmount_volume() detach_volume(volume, ec2_instance_id, False) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount @@ -393,21 +393,21 @@ def mount_ebs(repository_name, project, pipeline, branch, platform, build_type, created = False if 'Volumes' in response and not len(response['Volumes']): - print 'Volume for {} doesn\'t exist creating it...'.format(mount_name) + print('Volume for {} doesn\'t exist creating it...'.format(mount_name)) # volume doesn't exist, create it volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type) else: volume = response['Volumes'][0] volume_id = volume['VolumeId'] - print 'Current volume {} is a {} GB {}'.format(volume_id, volume['Size'], volume['VolumeType']) + print('Current volume {} is a {} GB {}'.format(volume_id, volume['Size'], volume['VolumeType'])) if (volume['Size'] != disk_size or volume['VolumeType'] != disk_type): - print 'Override disk attributes does not match the existing volume, deleting {} and replacing the volume'.format(volume_id) + print('Override disk attributes does not match the existing volume, deleting {} and replacing the volume'.format(volume_id)) delete_volume(ec2_client, volume_id) volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type) if len(volume['Attachments']): # this is bad we shouldn't be attached, we should have detached at the end of a build attachment = volume['Attachments'][0] - print ('Volume already has attachment {}, detaching...'.format(attachment)) + print(('Volume already has attachment {}, detaching...'.format(attachment))) detach_volume(ec2_resource.Volume(volume_id), attachment['InstanceId'], True) volume = ec2_resource.Volume(volume_id) @@ -416,23 +416,23 @@ def mount_ebs(repository_name, project, pipeline, branch, platform, build_type, drives_before = win32api.GetLogicalDriveStrings() drives_before = drives_before.split('\000')[:-1] - print drives_before + print(drives_before) attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created) free_space_mb = get_free_space_mb(MOUNT_PATH) - print 'Free disk space {}MB'.format(free_space_mb) + print('Free disk space {}MB'.format(free_space_mb)) if free_space_mb < LOW_EBS_DISK_SPACE_LIMIT: - print 'Volume is running below EBS free disk space treshhold {}MB. Recreating volume and running clean build.'.format(LOW_EBS_DISK_SPACE_LIMIT) + print('Volume is running below EBS free disk space treshhold {}MB. Recreating volume and running clean build.'.format(LOW_EBS_DISK_SPACE_LIMIT)) unmount_volume() detach_volume(volume, ec2_instance_id, False) delete_volume(ec2_client, volume_id) new_disk_size = int(volume.size * 1.25) if new_disk_size > MAX_EBS_DISK_SIZE: - print 'Error: EBS disk size reached to the allowed maximum disk size {}MB, please contact ly-infra@ and ly-build@ to investigate.'.format(MAX_EBS_DISK_SIZE) + print('Error: EBS disk size reached to the allowed maximum disk size {}MB, please contact ly-infra@ and ly-build@ to investigate.'.format(MAX_EBS_DISK_SIZE)) exit(1) - print 'Recreating the EBS with disk size {}'.format(new_disk_size) + print('Recreating the EBS with disk size {}'.format(new_disk_size)) volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, branch, platform, build_type, new_disk_size, disk_type) volume = ec2_resource.Volume(volume_id) attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created) @@ -454,13 +454,13 @@ def unmount_ebs(): for attached_volume in ec2_instance.volumes.all(): for attachment in attached_volume.attachments: - print 'attachment device: {}'.format(attachment['Device']) + print('attachment device: {}'.format(attachment['Device'])) if attachment['Device'] == 'xvdf': volume = attached_volume if not volume: # volume is not mounted - print 'Volume is not mounted' + print('Volume is not mounted') else: unmount_volume() detach_volume(volume, ec2_instance_id, False)