@ -4,18 +4,18 @@
# #
import argparse import argparse
import ast
import boto3 import boto3
import datetime import datetime
import urllib . request , urllib . error , urllib . parse import urllib . request , urllib . error , urllib . parse
import os import os
import psutil import psutil
import time import time
import requests
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import traceback from contextlib import contextmanager
import threading
import _thread
DEFAULT_REGION = ' us-west-2 ' DEFAULT_REGION = ' us-west-2 '
DEFAULT_DISK_SIZE = 300 DEFAULT_DISK_SIZE = 300
@ -42,14 +42,18 @@ if os.name == 'nt':
kernel32 = ctypes . WinDLL ( ' kernel32 ' , use_last_error = True )
kernel32 = ctypes . WinDLL ( ' kernel32 ' , use_last_error = True )
kernel32 . GetDiskFreeSpaceExW . argtypes = ( ctypes . c_wchar_p , ) + ( PULARGE_INTEGER , ) * 3
kernel32 . GetDiskFreeSpaceExW . argtypes = ( ctypes . c_wchar_p , ) + ( PULARGE_INTEGER , ) * 3
class UsageTuple ( collections . namedtuple ( ' UsageTuple ' , ' total, used, free ' ) ) :
class UsageTuple ( collections . namedtuple ( ' UsageTuple ' , ' total, used, free ' ) ) :
def __str__ ( self ) :
def __str__ ( self ) :
# Add thousands separator to numbers displayed
# Add thousands separator to numbers displayed
return self . __class__ . __name__ + ' (total= {:n} , used= {:n} , free= {:n} ) ' . format ( * self )
return self . __class__ . __name__ + ' (total= {:n} , used= {:n} , free= {:n} ) ' . format ( * self )
def is_dir_symlink ( path ) :
def is_dir_symlink ( path ) :
FILE_ATTRIBUTE_REPARSE_POINT = 0x0400
FILE_ATTRIBUTE_REPARSE_POINT = 0x0400
return os . path . isdir ( path ) and ( ctypes . windll . kernel32 . GetFileAttributesW ( str ( path ) ) & FILE_ATTRIBUTE_REPARSE_POINT )
return os . path . isdir ( path ) and (
ctypes . windll . kernel32 . GetFileAttributesW ( str ( path ) ) & FILE_ATTRIBUTE_REPARSE_POINT )
def get_free_space_mb ( path ) :
def get_free_space_mb ( path ) :
if sys . version_info < ( 3 , ) : # Python 2?
if sys . version_info < ( 3 , ) : # Python 2?
@ -77,16 +81,39 @@ if os.name == 'nt':
used = total . value - free . value
used = total . value - free . value
return free . value / 1024 / 1024 # for now
return free . value / 1024 / 1024 # for now
else : else :
def get_free_space_mb ( dirname ) :
def get_free_space_mb ( dirname ) :
st = os . statvfs ( dirname )
st = os . statvfs ( dirname )
return st . f_bavail * st . f_frsize / 1024 / 1024
return st . f_bavail * st . f_frsize / 1024 / 1024
def error ( message ) : def error ( message ) :
print ( message )
print ( message )
exit ( 1 )
exit ( 1 )
@contextmanager
def timeout ( duration , timeout_message ) :
timer = threading . Timer ( duration , lambda : _thread . interrupt_main ( ) )
timer . start ( )
try :
yield
except KeyboardInterrupt :
print ( timeout_message )
raise TimeoutError
finally :
# If the action ends in specified time, timer is canceled
timer . cancel ( )
def print_drives ( ) :
if os . name == ' nt ' :
drives_before = win32api . GetLogicalDriveStrings ( )
drives_before = drives_before . split ( ' \000 ' ) [ : - 1 ]
print ( drives_before )
def parse_args ( ) : def parse_args ( ) :
parser = argparse . ArgumentParser ( )
parser = argparse . ArgumentParser ( )
parser . add_argument ( ' -a ' , ' --action ' , dest = " action " , help = " Action (mount|unmount|delete) " )
parser . add_argument ( ' -a ' , ' --action ' , dest = " action " , help = " Action (mount|unmount|delete) " )
@ -96,8 +123,10 @@ def parse_args():
parser . add_argument ( ' -b ' , ' --branch ' , dest = " branch " , help = " Branch " )
parser . add_argument ( ' -b ' , ' --branch ' , dest = " branch " , help = " Branch " )
parser . add_argument ( ' -plat ' , ' --platform ' , dest = " platform " , help = " Platform " )
parser . add_argument ( ' -plat ' , ' --platform ' , dest = " platform " , help = " Platform " )
parser . add_argument ( ' -c ' , ' --build_type ' , dest = " build_type " , help = " Build type " )
parser . add_argument ( ' -c ' , ' --build_type ' , dest = " build_type " , help = " Build type " )
parser . add_argument ( ' -ds ' , ' --disk_size ' , dest = " disk_size " , help = " Disk size in Gigabytes (defaults to {} ) " . format ( DEFAULT_DISK_SIZE ) , default = DEFAULT_DISK_SIZE )
parser . add_argument ( ' -ds ' , ' --disk_size ' , dest = " disk_size " ,
parser . add_argument ( ' -dt ' , ' --disk_type ' , dest = " disk_type " , help = " Disk type (defaults to {} ) " . format ( DEFAULT_DISK_TYPE ) , default = DEFAULT_DISK_TYPE )
help = f " Disk size in Gigabytes (defaults to { DEFAULT_DISK_SIZE } ) " , default = DEFAULT_DISK_SIZE )
parser . add_argument ( ' -dt ' , ' --disk_type ' , dest = " disk_type " , help = f " Disk type (defaults to { DEFAULT_DISK_TYPE } ) " ,
default = DEFAULT_DISK_TYPE )
args = parser . parse_args ( )
args = parser . parse_args ( )
# Input validation
# Input validation
@ -117,19 +146,30 @@ def parse_args():
error ( ' No platform specified ' )
error ( ' No platform specified ' )
if args . build_type is None :
if args . build_type is None :
error ( ' No build_type specified ' )
error ( ' No build_type specified ' )
return args
return args
def get_mount_name ( repository_name , project , pipeline , branch , platform , build_type ) : def get_mount_name ( repository_name , project , pipeline , branch , platform , build_type ) :
mount_name = " {} _ {} _ {} _ {} _ {} _ {} " . format ( repository_name , project , pipeline , branch , platform , build_type )
mount_name = f" { repository_name } _ { project } _ { pipeline } _ { branch } _ { platform } _ { build_type } "
mount_name = mount_name . replace ( ' / ' , ' _ ' ) . replace ( ' \\ ' , ' _ ' )
mount_name = mount_name . replace ( ' / ' , ' _ ' ) . replace ( ' \\ ' , ' _ ' )
return mount_name
return mount_name
def get_pipeline_and_branch ( pipeline , branch ) : def get_pipeline_and_branch ( pipeline , branch ) :
pipeline_and_branch = " {} _ {} " . format ( pipeline , branch )
pipeline_and_branch = f" { pipeline } _ { branch } "
pipeline_and_branch = pipeline_and_branch . replace ( ' / ' , ' _ ' ) . replace ( ' \\ ' , ' _ ' )
pipeline_and_branch = pipeline_and_branch . replace ( ' / ' , ' _ ' ) . replace ( ' \\ ' , ' _ ' )
return pipeline_and_branch
return pipeline_and_branch
def get_region_name ( ) :
session = boto3 . session . Session ( )
region = session . region_name
if region is None :
region = DEFAULT_REGION
return region
def get_ec2_client ( region ) : def get_ec2_client ( region ) :
client = boto3 . client ( ' ec2 ' , region_name = region )
client = boto3 . client ( ' ec2 ' , region_name = region )
return client
return client
@ -140,48 +180,51 @@ def get_ec2_instance_id():
instance_id = urllib . request . urlopen ( ' http://169.254.169.254/latest/meta-data/instance-id ' ) . read ( )
instance_id = urllib . request . urlopen ( ' http://169.254.169.254/latest/meta-data/instance-id ' ) . read ( )
return instance_id . decode ( " utf-8 " )
return instance_id . decode ( " utf-8 " )
except Exception as e :
except Exception as e :
print ( e . message )
print ( e )
error ( ' No EC2 metadata! Check if you are running this script on an EC2 instance. ' )
error ( ' No EC2 metadata! Check if you are running this script on an EC2 instance. ' )
def get_availability_zone ( ) : def get_availability_zone ( ) :
try :
try :
availability_zone = urllib . request . urlopen ( ' http://169.254.169.254/latest/meta-data/placement/availability-zone ' ) . read ( )
availability_zone = urllib . request . urlopen (
' http://169.254.169.254/latest/meta-data/placement/availability-zone ' ) . read ( )
return availability_zone . decode ( " utf-8 " )
return availability_zone . decode ( " utf-8 " )
except Exception as e :
except Exception as e :
print ( e . message )
print ( e )
error ( ' No EC2 metadata! Check if you are running this script on an EC2 instance. ' )
error ( ' No EC2 metadata! Check if you are running this script on an EC2 instance. ' )
def kill_processes ( workspace = ' /dev/ ' ) : def kill_processes ( workspace = ' /dev/ ' ) :
'''
"""
Kills all processes that have open file paths associated with the workspace .
Kills all processes that have open file paths associated with the workspace .
Uses PSUtil for cross - platform compatibility
Uses PSUtil for cross - platform compatibility
'''
"""
print ( ' Checking for any stuck processes... ' )
print ( ' Checking for any stuck processes... ' )
for proc in psutil . process_iter ( ) :
for proc in psutil . process_iter ( ) :
try :
try :
if workspace in str ( proc . open_files ( ) ) :
if workspace in str ( proc . open_files ( ) ) :
print ( " { } has open files in { }. Terminating " . format( proc . name ( ) , proc. open_files ( ) ) )
print ( f " { proc . name ( ) has open files in { . open_files ( ) } . Terminating " )
proc . kill ( )
proc . kill ( )
time . sleep ( 1 ) # Just to make sure a parent process has time to close
time . sleep ( 1 ) # Just to make sure a parent process has time to close
except ( psutil . NoSuchProcess , psutil . AccessDenied , psutil . ZombieProcess ) :
except ( psutil . NoSuchProcess , psutil . AccessDenied , psutil . ZombieProcess ) :
continue
continue
def delete_volume ( ec2_client , volume_id ) : def delete_volume ( ec2_client , volume_id ) :
response = ec2_client . delete_volume ( VolumeId = volume_id )
response = ec2_client . delete_volume ( VolumeId = volume_id )
print ( ' Volume {} deleted ' . format ( volume_id ) )
print ( f ' Volume { volume_id } deleted ' )
def find_snapshot_id ( ec2_client , repository_name , project , pipeline , platform , build_type , disk_size ) : def find_snapshot_id ( ec2_client , repository_name , project , pipeline , platform , build_type , disk_size ) :
mount_name = get_mount_name ( repository_name , project , pipeline , ' stabilization_2106 ' , platform , build_type ) # we take snapshots out of stabilization_2106
mount_name = get_mount_name ( repository_name , project , pipeline , ' stabilization_2106 ' , platform ,
response = ec2_client . describe_snapshots ( Filters = [ {
build_type ) # we take snapshots out of stabilization_2106
response = ec2_client . describe_snapshots ( Filters = [ {
' Name ' : ' tag:Name ' , ' Values ' : [ mount_name ]
' Name ' : ' tag:Name ' , ' Values ' : [ mount_name ]
} ] )
} ] )
snapshot_id = None
snapshot_id = None
if ' Snapshots ' in response and len ( response [ ' Snapshots ' ] ) > 0 :
if ' Snapshots ' in response and len ( response [ ' Snapshots ' ] ) > 0 :
snapshot_start_time_max = None # find the latest snapshot
snapshot_start_time_max = None # find the latest snapshot
for snapshot in response [ ' Snapshots ' ] :
for snapshot in response [ ' Snapshots ' ] :
if snapshot [ ' State ' ] == ' completed ' and snapshot [ ' VolumeSize ' ] == disk_size :
if snapshot [ ' State ' ] == ' completed ' and snapshot [ ' VolumeSize ' ] == disk_size :
snapshot_start_time = snapshot [ ' StartTime ' ]
snapshot_start_time = snapshot [ ' StartTime ' ]
@ -190,28 +233,33 @@ def find_snapshot_id(ec2_client, repository_name, project, pipeline, platform, b
snapshot_id = snapshot [ ' SnapshotId ' ]
snapshot_id = snapshot [ ' SnapshotId ' ]
return snapshot_id
return snapshot_id
def create_volume ( ec2_client , availability_zone , repository_name , project , pipeline , branch , platform , build_type , disk_size , disk_type ) :
# The actual EBS default calculation for IOps is a floating point number, the closest approxmiation is 4x of the disk size for simplicity
def create_volume ( ec2_client , availability_zone , repository_name , project , pipeline , branch , platform , build_type ,
disk_size , disk_type ) :
mount_name = get_mount_name ( repository_name , project , pipeline , branch , platform , build_type )
mount_name = get_mount_name ( repository_name , project , pipeline , branch , platform , build_type )
pipeline_and_branch = get_pipeline_and_branch ( pipeline , branch )
pipeline_and_branch = get_pipeline_and_branch ( pipeline , branch )
parameters = dict (
parameters = dict (
AvailabilityZone = availability_zone ,
AvailabilityZone = availability_zone ,
VolumeType = disk_type ,
VolumeType = disk_type ,
TagSpecifications = [ {
TagSpecifications = [ {
' ResourceType ' : ' volume ' ,
' ResourceType ' : ' volume ' ,
' Tags ' : [
' Tags ' : [
{ ' Key ' : ' Name ' , ' Value ' : mount_name } ,
{ ' Key ' : ' Name ' , ' Value ' : mount_name } ,
{ ' Key ' : ' RepositoryName ' , ' Value ' : repository_name } ,
{ ' Key ' : ' RepositoryName ' , ' Value ' : repository_name } ,
{ ' Key ' : ' Project ' , ' Value ' : project } ,
{ ' Key ' : ' Project ' , ' Value ' : project } ,
{ ' Key ' : ' Pipeline ' , ' Value ' : pipeline } ,
{ ' Key ' : ' Pipeline ' , ' Value ' : pipeline } ,
{ ' Key ' : ' BranchName ' , ' Value ' : branch } ,
{ ' Key ' : ' BranchName ' , ' Value ' : branch } ,
{ ' Key ' : ' Platform ' , ' Value ' : platform } ,
{ ' Key ' : ' Platform ' , ' Value ' : platform } ,
{ ' Key ' : ' BuildType ' , ' Value ' : build_type } ,
{ ' Key ' : ' BuildType ' , ' Value ' : build_type } ,
{ ' Key ' : ' PipelineAndBranch ' , ' Value ' : pipeline_and_branch } , # used so the snapshoting easily identifies which volumes to snapshot
# Used so the snapshoting easily identifies which volumes to snapshot
{ ' Key ' : ' PipelineAndBranch ' , ' Value ' : pipeline_and_branch } ,
]
]
} ]
} ]
)
)
if ' io1 ' in disk_type . lower ( ) :
# The actual EBS default calculation for IOps is a floating point number,
# the closest approxmiation is 4x of the disk size for simplicity
if ' io1 ' in disk_type . lower ( ) :
parameters [ ' Iops ' ] = ( 4 * disk_size )
parameters [ ' Iops ' ] = ( 4 * disk_size )
snapshot_id = find_snapshot_id ( ec2_client , repository_name , project , pipeline , platform , build_type , disk_size )
snapshot_id = find_snapshot_id ( ec2_client , repository_name , project , pipeline , platform , build_type , disk_size )
@ -230,16 +278,17 @@ def create_volume(ec2_client, availability_zone, repository_name, project, pipel
time . sleep ( 1 )
time . sleep ( 1 )
response = ec2_client . describe_volumes ( VolumeIds = [ volume_id , ] )
response = ec2_client . describe_volumes ( VolumeIds = [ volume_id , ] )
while ( response [ ' Volumes ' ] [ 0 ] [ ' State ' ] != ' available ' ) :
with timeout ( DEFAULT_TIMEOUT , ' ERROR: Timeout reached trying to create EBS. ' ) :
time . sleep ( 1 )
while response [ ' Volumes ' ] [ 0 ] [ ' State ' ] != ' available ' :
response = ec2_client . describe_volumes ( VolumeIds = [ volume_id , ] )
time . sleep ( 1 )
response = ec2_client . describe_volumes ( VolumeIds = [ volume_id , ] )
print ( ( " Volume { } created \n \t Snapshot: { }\n \t Repository {} \n \t Project {} \n \t Pipeline {} \n \t Branch {} \n \t Platform: {} \n \t Build type: {} "
print ( f " Volume { volume_id created \n \t Snapshot: { snapshot_id } \n \t Repository { repository_name } \n \t "
. format ( volume_id , snapshot_id , repository_name , project , pipeline , branch , platform , build_type ) ) )
f " Project { project } \n \t Pipeline { pipeline } \n \t Branch { branch } \n \t Platform: { platform } \n \t Build type: { build_type } " )
return volume_id , created
return volume_id , created
def mount_volume ( created ) : def mount_volume _to_device ( created ) :
print ( ' Mounting volume... ' )
print ( ' Mounting volume... ' )
if os . name == ' nt ' :
if os . name == ' nt ' :
f = tempfile . NamedTemporaryFile ( delete = False )
f = tempfile . NamedTemporaryFile ( delete = False )
@ -247,7 +296,7 @@ def mount_volume(created):
select disk 1
select disk 1
online disk
online disk
attribute disk clear readonly
attribute disk clear readonly
""" .encode( ' utf-8 ' )) # assume disk # for now
""" .encode( ' utf-8 ' )) # assume disk # for now
if created :
if created :
print ( ' Creating filesystem on new volume ' )
print ( ' Creating filesystem on new volume ' )
@ -259,18 +308,12 @@ def mount_volume(created):
""" .encode( ' utf-8 ' ))
""" .encode( ' utf-8 ' ))
f . close ( )
f . close ( )
subprocess . call ( [ ' diskpart ' , ' /s ' , f . name ] )
subprocess . call ( [ ' diskpart ' , ' /s ' , f . name ] )
time . sleep ( 5 )
time . sleep ( 5 )
drives_after = win32api . GetLogicalDriveStrings ( )
print_drives ( )
drives_after = drives_after . split ( ' \000 ' ) [ : - 1 ]
print ( drives_after )
#drive_letter = next(item for item in drives_after if item not in drives_before)
drive_letter = MOUNT_PATH
os . unlink ( f . name )
os . unlink ( f . name )
@ -283,8 +326,8 @@ def mount_volume(created):
subprocess . call ( [ ' mount ' , ' /dev/xvdf ' , MOUNT_PATH ] )
subprocess . call ( [ ' mount ' , ' /dev/xvdf ' , MOUNT_PATH ] )
def attach_volume ( volume , volume_id , instance_id , timeout = DEFAULT_TIMEOUT ) : def attach_volume _to_ec2_instance ( volume , volume_id , instance_id , timeout _duration = DEFAULT_TIMEOUT ) :
print ( ' Attaching volume {} to instance {} ' . format ( volume_id , instance_id ) )
print ( f' Attaching volume { volume_id } to instance { instance_id } ' )
volume . attach_to_instance ( Device = ' xvdf ' ,
volume . attach_to_instance ( Device = ' xvdf ' ,
InstanceId = instance_id ,
InstanceId = instance_id ,
VolumeId = volume_id )
VolumeId = volume_id )
@ -292,13 +335,10 @@ def attach_volume(volume, volume_id, instance_id, timeout=DEFAULT_TIMEOUT):
time . sleep ( 2 )
time . sleep ( 2 )
# reload the volume just in case
# reload the volume just in case
volume . load ( )
volume . load ( )
timeout_init = time . clock ( )
with timeout ( timeout_duration , ' ERROR: Timeout reached trying to mount EBS. ' ) :
while ( len ( volume . attachments ) and volume . attachments [ 0 ] [ ' State ' ] != ' attached ' ) :
while len ( volume . attachments ) and volume . attachments [ 0 ] [ ' State ' ] != ' attached ' :
time . sleep ( 1 )
time . sleep ( 1 )
volume . load ( )
volume . load ( )
if ( time . clock ( ) - timeout_init ) > timeout :
print ( ' ERROR: Timeout reached trying to mount EBS ' )
exit ( 1 )
volume . create_tags (
volume . create_tags (
Tags = [
Tags = [
{
{
@ -307,11 +347,11 @@ def attach_volume(volume, volume_id, instance_id, timeout=DEFAULT_TIMEOUT):
} ,
} ,
]
]
)
)
print ( ' Volume {} has been attached to instance {} ' . format ( volume_id , instance_id ) )
print ( f' Volume { volume_id } has been attached to instance { instance_id } ' )
def unmount_volume ( ) : def unmount_volume _from_device ( ) :
print ( ' U mounting volume...' )
print ( ' U n mounting EBS volume from devic e...' )
if os . name == ' nt ' :
if os . name == ' nt ' :
kill_processes ( MOUNT_PATH + ' workspace ' )
kill_processes ( MOUNT_PATH + ' workspace ' )
f = tempfile . NamedTemporaryFile ( delete = False )
f = tempfile . NamedTemporaryFile ( delete = False )
@ -327,50 +367,31 @@ def unmount_volume():
subprocess . call ( [ ' umount ' , ' -f ' , MOUNT_PATH ] )
subprocess . call ( [ ' umount ' , ' -f ' , MOUNT_PATH ] )
def detach_volume ( volume , ec2_instance_id , force , timeout = DEFAULT_TIMEOUT ) : def detach_volume _from_ec2_instance ( volume , ec2_instance_id , force , timeout _duration = DEFAULT_TIMEOUT ) :
print ( ' Detaching volume {} from instance {} ' . format ( volume . volume_id , ec2_instance_id ) )
print ( f' Detaching volume { volume . volume_id } from instance { ec2_instance_id } ' )
volume . detach_from_instance ( Device = ' xvdf ' ,
volume . detach_from_instance ( Device = ' xvdf ' ,
Force = force ,
Force = force ,
InstanceId = ec2_instance_id ,
InstanceId = ec2_instance_id ,
VolumeId = volume . volume_id )
VolumeId = volume . volume_id )
timeout_init = time . clock ( )
try :
while len ( volume . attachments ) and volume . attachments [ 0 ] [ ' State ' ] != ' detached ' :
with timeout ( timeout_duration , ' ERROR: Timeout reached trying to unmount EBS. ' ) :
time . sleep ( 1 )
while len ( volume . attachments ) and volume . attachments [ 0 ] [ ' State ' ] != ' detached ' :
volume . load ( )
time . sleep ( 1 )
if ( time . clock ( ) - timeout_init ) > timeout :
volume . load ( )
print ( ' ERROR: Timeout reached trying to unmount EBS. ' )
except TimeoutError :
volume . detach_from_instance ( Device = ' xvdf ' , Force = True , InstanceId = ec2_instance_id , VolumeId = volume . volume_id )
print ( ' Force detaching EBS. ' )
exit ( 1 )
volume . detach_from_instance ( Device = ' xvdf ' , Force = True , InstanceId = ec2_instance_id , VolumeId = volume . volume_id )
print ( ' Volume { } has been detached from instance { }' . format ( volume . volume_id , ec2_instance_id ) )
print ( f ' Volume { volume . volume_id has been detached from instance { ec2_instance_id } ' )
volume . load ( )
volume . load ( )
if len ( volume . attachments ) :
if len ( volume . attachments ) :
print ( ' Volume still has attachments ' )
print ( ' Volume still has attachments ' )
for attachment in volume . attachments :
for attachment in volume . attachments :
print ( ' Volume {} {} to instance {} ' . format ( attachment [ ' VolumeId ' ] , attachment [ ' State ' ] , attachment [ ' InstanceId ' ] ) )
print ( f " Volume { attachment [ ' VolumeId ' ] } { attachment [ ' State ' ] } to instance { attachment [ ' InstanceId ' ] } " )
def attach_ebs_and_create_partition_with_retry ( volume , volume_id , ec2_instance_id , created ) :
attach_volume ( volume , volume_id , ec2_instance_id )
mount_volume ( created )
attempt = 1
while attempt < = MAX_EBS_MOUNTING_ATTEMPT :
if os . name == ' nt ' :
drives_after = win32api . GetLogicalDriveStrings ( )
drives_after = drives_after . split ( ' \000 ' ) [ : - 1 ]
if MOUNT_PATH not in drives_after :
print ( ' Disk partitioning failed, retrying... ' )
unmount_volume ( )
detach_volume ( volume , ec2_instance_id , False )
attach_volume ( volume , volume_id , ec2_instance_id )
mount_volume ( created )
attempt + = 1
def mount_ebs ( repository_name , project , pipeline , branch , platform , build_type , disk_size , disk_type ) : def mount_ebs ( repository_name , project , pipeline , branch , platform , build_type , disk_size , disk_type ) :
session = boto3 . session . Session ( )
region = get_region_name ( )
region = session . region_name
if region is None :
region = DEFAULT_REGION
ec2_client = get_ec2_client ( region )
ec2_client = get_ec2_client ( region )
ec2_instance_id = get_ec2_instance_id ( )
ec2_instance_id = get_ec2_instance_id ( )
ec2_availability_zone = get_availability_zone ( )
ec2_availability_zone = get_availability_zone ( )
@ -379,70 +400,70 @@ def mount_ebs(repository_name, project, pipeline, branch, platform, build_type,
for volume in ec2_instance . volumes . all ( ) :
for volume in ec2_instance . volumes . all ( ) :
for attachment in volume . attachments :
for attachment in volume . attachments :
print ( ' attachment device: {} ' . format ( attachment [ ' Device ' ] ) )
print ( f" attachment device: { attachment [ ' Device ' ] } " )
if ' xvdf ' in attachment [ ' Device ' ] and attachment [ ' State ' ] != ' detached ' :
if ' xvdf ' in attachment [ ' Device ' ] and attachment [ ' State ' ] != ' detached ' :
print ( ' A device is already attached to xvdf. This likely means a previous build failed to detach its ' \
print ( ' A device is already attached to xvdf. This likely means a previous build failed to detach its '
' build volume. This volume is considered orphaned and will be detached from this instance. ' )
' build volume. This volume is considered orphaned and will be detached from this instance. ' )
unmount_volume ( )
unmount_volume_from_device ( )
detach_volume ( volume , ec2_instance_id , False ) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount
detach_volume_from_ec2_instance ( volume , ec2_instance_id ,
False ) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount
mount_name = get_mount_name ( repository_name , project , pipeline , branch , platform , build_type )
mount_name = get_mount_name ( repository_name , project , pipeline , branch , platform , build_type )
response = ec2_client . describe_volumes ( Filters = [ {
response = ec2_client . describe_volumes ( Filters = [ {
' Name ' : ' tag:Name ' , ' Values ' : [ mount_name ]
' Name ' : ' tag:Name ' , ' Values ' : [ mount_name ]
} ] ) } ] )
created = False
created = False
if ' Volumes ' in response and not len ( response [ ' Volumes ' ] ) :
if ' Volumes ' in response and not len ( response [ ' Volumes ' ] ) :
print ( ' Volume for { } doesn \' t exist creating it... ' . format ( mount_name ) )
print ( f ' Volume for { mount_name doesn \' t exist creating it... ' )
# volume doesn't exist, create it
# volume doesn't exist, create it
volume_id , created = create_volume ( ec2_client , ec2_availability_zone , repository_name , project , pipeline , branch , platform , build_type , disk_size , disk_type )
volume_id , created = create_volume ( ec2_client , ec2_availability_zone , repository_name , project , pipeline ,
branch , platform , build_type , disk_size , disk_type )
else :
else :
volume = response [ ' Volumes ' ] [ 0 ]
volume = response [ ' Volumes ' ] [ 0 ]
volume_id = volume [ ' VolumeId ' ]
volume_id = volume [ ' VolumeId ' ]
print ( ' Current volume {} is a {} GB {} ' . format ( volume_id , volume [ ' Size ' ] , volume [ ' VolumeType ' ] ) )
print ( f " Current volume { volume_id } is a { volume [ ' Size ' ] } GB { volume [ ' VolumeType ' ] } " )
if ( volume [ ' Size ' ] != disk_size or volume [ ' VolumeType ' ] != disk_type ) :
if volume [ ' Size ' ] != disk_size or volume [ ' VolumeType ' ] != disk_type :
print ( ' Override disk attributes does not match the existing volume, deleting {} and replacing the volume ' . format ( volume_id ) )
print (
f ' Override disk attributes does not match the existing volume, deleting { volume_id } and replacing the volume ' )
delete_volume ( ec2_client , volume_id )
delete_volume ( ec2_client , volume_id )
volume_id , created = create_volume ( ec2_client , ec2_availability_zone , repository_name , project , pipeline , branch , platform , build_type , disk_size , disk_type )
volume_id , created = create_volume ( ec2_client , ec2_availability_zone , repository_name , project , pipeline ,
branch , platform , build_type , disk_size , disk_type )
if len ( volume [ ' Attachments ' ] ) :
if len ( volume [ ' Attachments ' ] ) :
# this is bad we shouldn't be attached, we should have detached at the end of a build
# this is bad we shouldn't be attached, we should have detached at the end of a build
attachment = volume [ ' Attachments ' ] [ 0 ]
attachment = volume [ ' Attachments ' ] [ 0 ]
print ( ( ' Volume already has attachment { }, detaching... ' . format ( attachment ) ) )
print ( f ' Volume already has attachment { attachment , detaching... ' )
detach_volume ( ec2_resource . Volume ( volume_id ) , attachment [ ' InstanceId ' ] , True )
detach_volume _from_ec2_instance ( ec2_resource . Volume ( volume_id ) , attachment [ ' InstanceId ' ] , True )
volume = ec2_resource . Volume ( volume_id )
volume = ec2_resource . Volume ( volume_id )
if os . name == ' nt ' :
print_drives ( )
drives_before = win32api . GetLogicalDriveStrings ( )
attach_volume_to_ec2_instance ( volume , volume_id , ec2_instance_id )
drives_before = drives_before . split ( ' \000 ' ) [ : - 1 ]
mount_volume_to_device ( created )
print_drives ( )
print ( drives_before )
attach_ebs_and_create_partition_with_retry ( volume , volume_id , ec2_instance_id , created )
free_space_mb = get_free_space_mb ( MOUNT_PATH )
free_space_mb = get_free_space_mb ( MOUNT_PATH )
print ( ' Free disk space {} MB ' . format ( free_space_mb ) )
print ( f ' Free disk space { free_space_mb } MB ' )
if free_space_mb < LOW_EBS_DISK_SPACE_LIMIT :
if free_space_mb < LOW_EBS_DISK_SPACE_LIMIT :
print ( ' Volume is running below EBS free disk space treshhold { }MB. Recreating volume and running clean build. ' . format ( LOW_EBS_DISK_SPACE_LIMIT ) )
print ( f ' Volume is running below EBS free disk space treshhold { LOW_EBS_DISK_SPACE_LIMIT MB. Recreating volume and running clean build. ' )
unmount_volume ( )
unmount_volume _from_device ( )
detach_volume ( volume , ec2_instance_id , False )
detach_volume _from_ec2_instance ( volume , ec2_instance_id , False )
delete_volume ( ec2_client , volume_id )
delete_volume ( ec2_client , volume_id )
new_disk_size = int ( volume . size * 1.25 )
new_disk_size = int ( volume . size * 1.25 )
if new_disk_size > MAX_EBS_DISK_SIZE :
if new_disk_size > MAX_EBS_DISK_SIZE :
print ( ' Error: EBS disk size reached to the allowed maximum disk size { }MB, please contact ly-infra@ and ly-build@ to investigate. ' . format ( MAX_EBS_DISK_SIZE ) )
print ( f ' Error: EBS disk size reached to the allowed maximum disk size { MAX_EBS_DISK_SIZE MB, please contact ly-infra@ and ly-build@ to investigate. ' )
exit ( 1 )
exit ( 1 )
print ( ' Recreating the EBS with disk size {} ' . format ( new_disk_size ) )
print ( f ' Recreating the EBS with disk size { new_disk_size } ' )
volume_id , created = create_volume ( ec2_client , ec2_availability_zone , repository_name , project , pipeline , branch , platform , build_type , new_disk_size , disk_type )
volume_id , created = create_volume ( ec2_client , ec2_availability_zone , repository_name , project , pipeline ,
branch , platform , build_type , new_disk_size , disk_type )
volume = ec2_resource . Volume ( volume_id )
volume = ec2_resource . Volume ( volume_id )
attach_ebs_and_create_partition_with_retry ( volume , volume_id , ec2_instance_id , created )
attach_volume_to_ec2_instance ( volume , volume_id , ec2_instance_id )
mount_volume_to_device ( created )
def unmount_ebs ( ) : def unmount_ebs ( ) :
session = boto3 . session . Session ( )
region = get_region_name ( )
region = session . region_name
if region is None :
region = DEFAULT_REGION
ec2_client = get_ec2_client ( region )
ec2_instance_id = get_ec2_instance_id ( )
ec2_instance_id = get_ec2_instance_id ( )
ec2_resource = boto3 . resource ( ' ec2 ' , region_name = region )
ec2_resource = boto3 . resource ( ' ec2 ' , region_name = region )
ec2_instance = ec2_resource . Instance ( ec2_instance_id )
ec2_instance = ec2_resource . Instance ( ec2_instance_id )
@ -454,7 +475,7 @@ def unmount_ebs():
for attached_volume in ec2_instance . volumes . all ( ) :
for attached_volume in ec2_instance . volumes . all ( ) :
for attachment in attached_volume . attachments :
for attachment in attached_volume . attachments :
print ( ' attachment device: {} ' . format ( attachment [ ' Device ' ] ) )
print ( f" attachment device: { attachment [ ' Device ' ] } " )
if attachment [ ' Device ' ] == ' xvdf ' :
if attachment [ ' Device ' ] == ' xvdf ' :
volume = attached_volume
volume = attached_volume
@ -462,24 +483,18 @@ def unmount_ebs():
# volume is not mounted
# volume is not mounted
print ( ' Volume is not mounted ' )
print ( ' Volume is not mounted ' )
else :
else :
unmount_volume ( )
unmount_volume_from_device ( )
detach_volume ( volume , ec2_instance_id , False )
detach_volume_from_ec2_instance ( volume , ec2_instance_id , False )
def delete_ebs ( repository_name , project , pipeline , branch , platform , build_type ) : def delete_ebs ( repository_name , project , pipeline , branch , platform , build_type ) :
unmount_ebs ( )
unmount_ebs ( )
region = get_region_name ( )
session = boto3 . session . Session ( )
region = session . region_name
if region is None :
region = DEFAULT_REGION
ec2_client = get_ec2_client ( region )
ec2_client = get_ec2_client ( region )
ec2_instance_id = get_ec2_instance_id ( )
ec2_resource = boto3 . resource ( ' ec2 ' , region_name = region )
ec2_instance = ec2_resource . Instance ( ec2_instance_id )
mount_name = get_mount_name ( repository_name , project , pipeline , branch , platform , build_type )
mount_name = get_mount_name ( repository_name , project , pipeline , branch , platform , build_type )
response = ec2_client . describe_volumes ( Filters = [
response = ec2_client . describe_volumes ( Filters = [
{ ' Name ' : ' tag:Name ' , ' Values ' : [ mount_name ] }
{ ' Name ' : ' tag:Name ' , ' Values ' : [ mount_name ] }
] )
] )
if ' Volumes ' in response and len ( response [ ' Volumes ' ] ) :
if ' Volumes ' in response and len ( response [ ' Volumes ' ] ) :
@ -496,7 +511,8 @@ def main(action, repository_name, project, pipeline, branch, platform, build_typ
elif action == ' delete ' :
elif action == ' delete ' :
delete_ebs ( repository_name , project , pipeline , branch , platform , build_type )
delete_ebs ( repository_name , project , pipeline , branch , platform , build_type )
if __name__ == " __main__ " : if __name__ == " __main__ " :
args = parse_args ( )
args = parse_args ( )
ret = main( args . action , args . repository_name , args . project , args . pipeline , args . branch , args . platform , args . build_type , args . disk_size , args . disk_type )
( args . action , args . repository_name , args . project , args . pipeline , args . branch , args . platform ,
sys . exit ( ret )
args . build_type , args . disk_size , args . disk_type )
Chris Galvan
4 years ago