Mercurial > repos > shellac > guppy_basecaller
diff env/lib/python3.7/site-packages/boto/manage/volume.py @ 5:9b1c78e6ba9c draft default tip
"planemo upload commit 6c0a8142489327ece472c84e558c47da711a9142"
author | shellac |
---|---|
date | Mon, 01 Jun 2020 08:59:25 -0400 |
parents | 79f47841a781 |
children |
line wrap: on
line diff
--- a/env/lib/python3.7/site-packages/boto/manage/volume.py Thu May 14 16:47:39 2020 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,420 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -from __future__ import print_function - -from boto.sdb.db.model import Model -from boto.sdb.db.property import StringProperty, IntegerProperty, ListProperty, ReferenceProperty, CalculatedProperty -from boto.manage.server import Server -from boto.manage import propget -import boto.utils -import boto.ec2 -import time -import traceback -from contextlib import closing -import datetime - - -class CommandLineGetter(object): - - def get_region(self, params): - if not params.get('region', None): - prop = self.cls.find_property('region_name') - params['region'] = propget.get(prop, choices=boto.ec2.regions) - - def get_zone(self, params): - if not params.get('zone', None): - prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', - choices=self.ec2.get_all_zones) - params['zone'] = propget.get(prop) - - def get_name(self, params): - if not params.get('name', None): - prop = self.cls.find_property('name') - params['name'] = propget.get(prop) - - def get_size(self, params): - if not params.get('size', None): - prop = IntegerProperty(name='size', verbose_name='Size (GB)') - params['size'] = propget.get(prop) - - def get_mount_point(self, params): - if not params.get('mount_point', None): - prop = self.cls.find_property('mount_point') - params['mount_point'] = propget.get(prop) - - def get_device(self, params): - if not params.get('device', None): - prop = self.cls.find_property('device') - params['device'] = propget.get(prop) - - def get(self, cls, params): - self.cls = cls - self.get_region(params) - self.ec2 = params['region'].connect() - self.get_zone(params) - self.get_name(params) - self.get_size(params) - self.get_mount_point(params) - self.get_device(params) - -class Volume(Model): - - name = StringProperty(required=True, unique=True, verbose_name='Name') - region_name = StringProperty(required=True, verbose_name='EC2 Region') - zone_name = StringProperty(required=True, verbose_name='EC2 Zone') - mount_point = StringProperty(verbose_name='Mount Point') - device = StringProperty(verbose_name="Device Name", default='/dev/sdp') - volume_id = StringProperty(required=True) - past_volume_ids = ListProperty(item_type=str) - server = ReferenceProperty(Server, collection_name='volumes', - verbose_name='Server Attached To') - volume_state = CalculatedProperty(verbose_name="Volume State", - calculated_type=str, use_method=True) - attachment_state = CalculatedProperty(verbose_name="Attachment State", - calculated_type=str, use_method=True) - size = CalculatedProperty(verbose_name="Size (GB)", - calculated_type=int, use_method=True) - - @classmethod - def create(cls, **params): - getter = CommandLineGetter() - getter.get(cls, params) - region = params.get('region') - ec2 = region.connect() - zone = params.get('zone') - size = params.get('size') - ebs_volume = ec2.create_volume(size, zone.name) - v = cls() - v.ec2 = ec2 - v.volume_id = ebs_volume.id - v.name = params.get('name') - v.mount_point = params.get('mount_point') - v.device = params.get('device') - v.region_name = region.name - v.zone_name = zone.name - v.put() - return v - - @classmethod - def create_from_volume_id(cls, region_name, volume_id, name): - vol = None - ec2 = boto.ec2.connect_to_region(region_name) - rs = ec2.get_all_volumes([volume_id]) - if len(rs) == 1: - v = rs[0] - vol = cls() - vol.volume_id = v.id - vol.name = name - vol.region_name = v.region.name - vol.zone_name = v.zone - vol.put() - return vol - - def create_from_latest_snapshot(self, name, size=None): - snapshot = self.get_snapshots()[-1] - return self.create_from_snapshot(name, snapshot, size) - - def create_from_snapshot(self, name, snapshot, size=None): - if size < self.size: - size = self.size - ec2 = self.get_ec2_connection() - if self.zone_name is None or self.zone_name == '': - # deal with the migration case where the zone is not set in the logical volume: - current_volume = ec2.get_all_volumes([self.volume_id])[0] - self.zone_name = current_volume.zone - ebs_volume = ec2.create_volume(size, self.zone_name, snapshot) - v = Volume() - v.ec2 = self.ec2 - v.volume_id = ebs_volume.id - v.name = name - v.mount_point = self.mount_point - v.device = self.device - v.region_name = self.region_name - v.zone_name = self.zone_name - v.put() - return v - - def get_ec2_connection(self): - if self.server: - return self.server.ec2 - if not hasattr(self, 'ec2') or self.ec2 is None: - self.ec2 = boto.ec2.connect_to_region(self.region_name) - return self.ec2 - - def _volume_state(self): - ec2 = self.get_ec2_connection() - rs = ec2.get_all_volumes([self.volume_id]) - return rs[0].volume_state() - - def _attachment_state(self): - ec2 = self.get_ec2_connection() - rs = ec2.get_all_volumes([self.volume_id]) - return rs[0].attachment_state() - - def _size(self): - if not hasattr(self, '__size'): - ec2 = self.get_ec2_connection() - rs = ec2.get_all_volumes([self.volume_id]) - self.__size = rs[0].size - return self.__size - - def install_xfs(self): - if self.server: - self.server.install('xfsprogs xfsdump') - - def get_snapshots(self): - """ - Returns a list of all completed snapshots for this volume ID. - """ - ec2 = self.get_ec2_connection() - rs = ec2.get_all_snapshots() - all_vols = [self.volume_id] + self.past_volume_ids - snaps = [] - for snapshot in rs: - if snapshot.volume_id in all_vols: - if snapshot.progress == '100%': - snapshot.date = boto.utils.parse_ts(snapshot.start_time) - snapshot.keep = True - snaps.append(snapshot) - snaps.sort(cmp=lambda x, y: cmp(x.date, y.date)) - return snaps - - def attach(self, server=None): - if self.attachment_state == 'attached': - print('already attached') - return None - if server: - self.server = server - self.put() - ec2 = self.get_ec2_connection() - ec2.attach_volume(self.volume_id, self.server.instance_id, self.device) - - def detach(self, force=False): - state = self.attachment_state - if state == 'available' or state is None or state == 'detaching': - print('already detached') - return None - ec2 = self.get_ec2_connection() - ec2.detach_volume(self.volume_id, self.server.instance_id, self.device, force) - self.server = None - self.put() - - def checkfs(self, use_cmd=None): - if self.server is None: - raise ValueError('server attribute must be set to run this command') - # detemine state of file system on volume, only works if attached - if use_cmd: - cmd = use_cmd - else: - cmd = self.server.get_cmdshell() - status = cmd.run('xfs_check %s' % self.device) - if not use_cmd: - cmd.close() - if status[1].startswith('bad superblock magic number 0'): - return False - return True - - def wait(self): - if self.server is None: - raise ValueError('server attribute must be set to run this command') - with closing(self.server.get_cmdshell()) as cmd: - # wait for the volume device to appear - cmd = self.server.get_cmdshell() - while not cmd.exists(self.device): - boto.log.info('%s still does not exist, waiting 10 seconds' % self.device) - time.sleep(10) - - def format(self): - if self.server is None: - raise ValueError('server attribute must be set to run this command') - status = None - with closing(self.server.get_cmdshell()) as cmd: - if not self.checkfs(cmd): - boto.log.info('make_fs...') - status = cmd.run('mkfs -t xfs %s' % self.device) - return status - - def mount(self): - if self.server is None: - raise ValueError('server attribute must be set to run this command') - boto.log.info('handle_mount_point') - with closing(self.server.get_cmdshell()) as cmd: - cmd = self.server.get_cmdshell() - if not cmd.isdir(self.mount_point): - boto.log.info('making directory') - # mount directory doesn't exist so create it - cmd.run("mkdir %s" % self.mount_point) - else: - boto.log.info('directory exists already') - status = cmd.run('mount -l') - lines = status[1].split('\n') - for line in lines: - t = line.split() - if t and t[2] == self.mount_point: - # something is already mounted at the mount point - # unmount that and mount it as /tmp - if t[0] != self.device: - cmd.run('umount %s' % self.mount_point) - cmd.run('mount %s /tmp' % t[0]) - cmd.run('chmod 777 /tmp') - break - # Mount up our new EBS volume onto mount_point - cmd.run("mount %s %s" % (self.device, self.mount_point)) - cmd.run('xfs_growfs %s' % self.mount_point) - - def make_ready(self, server): - self.server = server - self.put() - self.install_xfs() - self.attach() - self.wait() - self.format() - self.mount() - - def freeze(self): - if self.server: - return self.server.run("/usr/sbin/xfs_freeze -f %s" % self.mount_point) - - def unfreeze(self): - if self.server: - return self.server.run("/usr/sbin/xfs_freeze -u %s" % self.mount_point) - - def snapshot(self): - # if this volume is attached to a server - # we need to freeze the XFS file system - try: - self.freeze() - if self.server is None: - snapshot = self.get_ec2_connection().create_snapshot(self.volume_id) - else: - snapshot = self.server.ec2.create_snapshot(self.volume_id) - boto.log.info('Snapshot of Volume %s created: %s' % (self.name, snapshot)) - except Exception: - boto.log.info('Snapshot error') - boto.log.info(traceback.format_exc()) - finally: - status = self.unfreeze() - return status - - def get_snapshot_range(self, snaps, start_date=None, end_date=None): - l = [] - for snap in snaps: - if start_date and end_date: - if snap.date >= start_date and snap.date <= end_date: - l.append(snap) - elif start_date: - if snap.date >= start_date: - l.append(snap) - elif end_date: - if snap.date <= end_date: - l.append(snap) - else: - l.append(snap) - return l - - def trim_snapshots(self, delete=False): - """ - Trim the number of snapshots for this volume. This method always - keeps the oldest snapshot. It then uses the parameters passed in - to determine how many others should be kept. - - The algorithm is to keep all snapshots from the current day. Then - it will keep the first snapshot of the day for the previous seven days. - Then, it will keep the first snapshot of the week for the previous - four weeks. After than, it will keep the first snapshot of the month - for as many months as there are. - - """ - snaps = self.get_snapshots() - # Always keep the oldest and the newest - if len(snaps) <= 2: - return snaps - snaps = snaps[1:-1] - now = datetime.datetime.now(snaps[0].date.tzinfo) - midnight = datetime.datetime(year=now.year, month=now.month, - day=now.day, tzinfo=now.tzinfo) - # Keep the first snapshot from each day of the previous week - one_week = datetime.timedelta(days=7, seconds=60*60) - print(midnight-one_week, midnight) - previous_week = self.get_snapshot_range(snaps, midnight-one_week, midnight) - print(previous_week) - if not previous_week: - return snaps - current_day = None - for snap in previous_week: - if current_day and current_day == snap.date.day: - snap.keep = False - else: - current_day = snap.date.day - # Get ourselves onto the next full week boundary - if previous_week: - week_boundary = previous_week[0].date - if week_boundary.weekday() != 0: - delta = datetime.timedelta(days=week_boundary.weekday()) - week_boundary = week_boundary - delta - # Keep one within this partial week - partial_week = self.get_snapshot_range(snaps, week_boundary, previous_week[0].date) - if len(partial_week) > 1: - for snap in partial_week[1:]: - snap.keep = False - # Keep the first snapshot of each week for the previous 4 weeks - for i in range(0, 4): - weeks_worth = self.get_snapshot_range(snaps, week_boundary-one_week, week_boundary) - if len(weeks_worth) > 1: - for snap in weeks_worth[1:]: - snap.keep = False - week_boundary = week_boundary - one_week - # Now look through all remaining snaps and keep one per month - remainder = self.get_snapshot_range(snaps, end_date=week_boundary) - current_month = None - for snap in remainder: - if current_month and current_month == snap.date.month: - snap.keep = False - else: - current_month = snap.date.month - if delete: - for snap in snaps: - if not snap.keep: - boto.log.info('Deleting %s(%s) for %s' % (snap, snap.date, self.name)) - snap.delete() - return snaps - - def grow(self, size): - pass - - def copy(self, snapshot): - pass - - def get_snapshot_from_date(self, date): - pass - - def delete(self, delete_ebs_volume=False): - if delete_ebs_volume: - self.detach() - ec2 = self.get_ec2_connection() - ec2.delete_volume(self.volume_id) - super(Volume, self).delete() - - def archive(self): - # snapshot volume, trim snaps, delete volume-id - pass - -