diff --git a/oss2/models.py b/oss2/models.py index ba120c1b..8754d2ca 100644 --- a/oss2/models.py +++ b/oss2/models.py @@ -776,13 +776,19 @@ class LifecycleExpiration(object): """过期删除操作。 :param days: 表示在文件修改后过了这么多天，就会匹配规则，从而被删除 - :param date: 表示在该日期之后，规则就一直生效。即每天都会对符合前缀的文件执行删除操作（如，删除），而不管文件是什么时候生成的。 - *不建议使用* - :param created_before_date: delete files if their last modified time earlier than created_before_date + :type days: int + :param date: 表示在该日期之后，规则就一直生效。即每天都会对符合前缀的文件执行删除操作（如，删除），而不管文件是什么时候生成的。*不建议使用* :type date: `datetime.date` + + :param created_before_date: delete files if their last modified time earlier than created_before_date + :type created_before_date: `datetime.date` + + :param expired_detete_marker: 真实文件删除之后是否自动移除删除标记，适用于多版本场景。 + :param expired_detete_marker: bool + """ - def __init__(self, days=None, date=None, created_before_date=None): + def __init__(self, days=None, date=None, created_before_date=None, expired_detete_marker=None): not_none_fields = 0 if days is not None: not_none_fields += 1 @@ -790,13 +796,16 @@ def __init__(self, days=None, date=None, created_before_date=None): not_none_fields += 1 if created_before_date is not None: not_none_fields += 1 + if expired_detete_marker is not None: + not_none_fields += 1 if not_none_fields > 1: - raise ClientError('More than one field(days, date and created_before_date) has been specified') + raise ClientError('More than one field(days, date and created_before_date, expired_detete_marker) has been specified') self.days = days self.date = date self.created_before_date = created_before_date + self.expired_detete_marker = expired_detete_marker class AbortMultipartUpload(object): @@ -830,18 +839,52 @@ def __init__(self, days=None, created_before_date=None, storage_class=None): self.storage_class = storage_class +class NoncurrentVersionExpiration(object): + """OSS何时将非当前版本的object删除 + + :param noncurrent_days: 指定多少天之后删除 + :type noncurrent_days: int + """ + def __init__(self, noncurrent_days): + self.noncurrent_days = noncurrent_days + + +class NoncurrentVersionStorageTransition(object): + """生命周期内，OSS何时将指定Object的非当前版本转储为IA或者Archive存储类型。 + + :param noncurrent_days: 多少天之后转存储 + :type noncurrent_days: int + """ + def __init__(self, noncurrent_days, storage_class): + self.noncurrent_days = noncurrent_days + self.storage_class = storage_class + + class LifecycleRule(object): """生命周期规则。 :param id: 规则名 + :type id: str + :param prefix: 只有文件名匹配该前缀的文件才适用本规则 + :type prefix: str + :param expiration: 过期删除操作。 :type expiration: :class:`LifecycleExpiration` + :param status: 启用还是禁止该规则。可选值为 `LifecycleRule.ENABLED` 或 `LifecycleRule.DISABLED` + :param storage_transitions: 存储类型转换规则 - :type storage_transitions: :class:`StorageTransition` + :type storage_transitions: list of class:`StorageTransition ` + :param tagging: object tagging 规则 - :type tagging: :class:`Tagging` + :type tagging: :class:`Tagging ` + + :param noncurrent_version_expiration: 指定Object非当前版本生命周期规则的过期属性。适用于多版本场景。 + :type noncurrent_version_expiration class:`NoncurrentVersionExpiration ` + + :param noncurrent_version_sotrage_transitions: 在有效生命周期中，OSS何时将指定Object的非当前版本转储为IA或者Archive存储类型，适用于多版本场景。 + :type noncurrent_version_sotrage_transitions: list of class:`NoncurrentVersionStorageTransition ` """ ENABLED = 'Enabled' @@ -850,7 +893,9 @@ class LifecycleRule(object): def __init__(self, id, prefix, status=ENABLED, expiration=None, abort_multipart_upload=None, - storage_transitions=None, tagging=None): + storage_transitions=None, tagging=None, + noncurrent_version_expiration=None, + noncurrent_version_sotrage_transitions=None): self.id = id self.prefix = prefix self.status = status @@ -858,13 +903,15 @@ def __init__(self, id, prefix, self.abort_multipart_upload = abort_multipart_upload self.storage_transitions = storage_transitions self.tagging = tagging + self.noncurrent_version_expiration = noncurrent_version_expiration + self.noncurrent_version_sotrage_transitions = noncurrent_version_sotrage_transitions class BucketLifecycle(object): """Bucket的生命周期配置。 :param rules: 规则列表， - :type rules: list of :class:`LifecycleRule` + :type rules: list of :class:`LifecycleRule ` """ def __init__(self, rules=None): self.rules = rules or [] diff --git a/oss2/resumable.py b/oss2/resumable.py index 9871b7bb..3acb4112 100644 --- a/oss2/resumable.py +++ b/oss2/resumable.py @@ -240,7 +240,7 @@ def _populate_valid_headers(headers=None, valid_keys=None): class _ResumableOperation(object): def __init__(self, bucket, key, filename, size, store, - progress_callback=None): + progress_callback=None, versionid=None): self.bucket = bucket self.key = to_string(key) self.filename = filename @@ -249,7 +249,12 @@ def __init__(self, bucket, key, filename, size, store, self._abspath = os.path.abspath(filename) self.__store = store - self.__record_key = self.__store.make_store_key(bucket.bucket_name, self.key, self._abspath) + + if versionid is None: + self.__record_key = self.__store.make_store_key(bucket.bucket_name, self.key, self._abspath) + else: + self.__record_key = self.__store.make_store_key(bucket.bucket_name, self.key, self._abspath, versionid) + logger.debug("Init _ResumableOperation, record_key: {0}".format(self.__record_key)) # protect self.__progress_callback @@ -295,9 +300,13 @@ def __init__(self, bucket, key, filename, objectInfo, num_threads=None, params=None, headers=None): + versionid = None + if params is not None and params.get('versionId') is not None: + versionid = params.get('versionId') super(_ResumableDownloader, self).__init__(bucket, key, filename, objectInfo.size, store or ResumableDownloadStore(), - progress_callback=progress_callback) + progress_callback=progress_callback, + versionid=versionid) self.objectInfo = objectInfo self.__part_size = defaults.get(part_size, defaults.multiget_part_size) @@ -718,10 +727,15 @@ def __init__(self, root=None, dir=None): super(ResumableDownloadStore, self).__init__(root or os.path.expanduser('~'), dir or _DOWNLOAD_TEMP_DIR) @staticmethod - def make_store_key(bucket_name, key, filename): + def make_store_key(bucket_name, key, filename, versionid=None): filepath = _normalize_path(filename) + oss_pathname = None - oss_pathname = 'oss://{0}/{1}'.format(bucket_name, key) + if versionid is None: + oss_pathname = 'oss://{0}/{1}'.format(bucket_name, key) + else: + oss_pathname = 'oss://{0}/{1}?versionid={2}'.format(bucket_name, key, versionid) + return utils.md5_string(oss_pathname) + '-' + utils.md5_string(filepath) + '-download' diff --git a/oss2/xml_utils.py b/oss2/xml_utils.py index 6fb6670c..5d7b7530 100644 --- a/oss2/xml_utils.py +++ b/oss2/xml_utils.py @@ -47,7 +47,9 @@ REDIRECT_TYPE_MIRROR, REDIRECT_TYPE_EXTERNAL, REDIRECT_TYPE_INTERNAL, - REDIRECT_TYPE_ALICDN) + REDIRECT_TYPE_ALICDN, + NoncurrentVersionStorageTransition, + NoncurrentVersionExpiration) from .select_params import (SelectJsonTypes, SelectParameters) @@ -583,6 +585,10 @@ def parse_lifecycle_expiration(expiration_node): expiration.days = _find_int(expiration_node, 'Days') elif expiration_node.find('Date') is not None: expiration.date = iso8601_to_date(_find_tag(expiration_node, 'Date')) + elif expiration_node.find('CreatedBeforeDate') is not None: + expiration.created_before_date = iso8601_to_date(_find_tag(expiration_node, 'CreatedBeforeDate')) + elif expiration_node.find('ExpiredObjectDeleteMarker') is not None: + expiration.expired_detete_marker = _find_bool(expiration_node, 'ExpiredObjectDeleteMarker') return expiration @@ -629,6 +635,25 @@ def parse_lifecycle_object_taggings(lifecycle_tagging_nodes): return Tagging(tagging_rule) +def parse_lifecycle_version_expiration(version_expiration_node): + if version_expiration_node is None: + return None + + noncurrent_days = _find_int(version_expiration_node, 'NoncurrentDays') + expiration = NoncurrentVersionExpiration(noncurrent_days) + + return expiration + +def parse_lifecycle_verison_storage_transitions(version_storage_transition_nodes): + version_storage_transitions = [] + for transition_node in version_storage_transition_nodes: + storage_class = _find_tag(transition_node, 'StorageClass') + non_crurrent_days = _find_int(transition_node, 'NoncurrentDays') + version_storage_transition = NoncurrentVersionStorageTransition(non_crurrent_days, storage_class) + version_storage_transitions.append(version_storage_transition) + + return version_storage_transitions + def parse_get_bucket_lifecycle(result, body): root = ElementTree.fromstring(body) @@ -639,6 +664,9 @@ def parse_get_bucket_lifecycle(result, body): abort_multipart_upload = parse_lifecycle_abort_multipart_upload(rule_node.find('AbortMultipartUpload')) storage_transitions = parse_lifecycle_storage_transitions(rule_node.findall('Transition')) tagging = parse_lifecycle_object_taggings(rule_node.findall('Tag')) + noncurrent_version_expiration = parse_lifecycle_version_expiration(rule_node.find('NoncurrentVersionExpiration')) + noncurrent_version_sotrage_transitions = parse_lifecycle_verison_storage_transitions(rule_node.findall('NoncurrentVersionTransition')) + rule = LifecycleRule( _find_tag(rule_node, 'ID'), _find_tag(rule_node, 'Prefix'), @@ -646,7 +674,9 @@ def parse_get_bucket_lifecycle(result, body): expiration=expiration, abort_multipart_upload=abort_multipart_upload, storage_transitions=storage_transitions, - tagging=tagging + tagging=tagging, + noncurrent_version_expiration = noncurrent_version_expiration, + noncurrent_version_sotrage_transitions = noncurrent_version_sotrage_transitions ) result.rules.append(rule) @@ -852,6 +882,8 @@ def to_put_bucket_lifecycle(bucket_lifecycle): _add_text_child(expiration_node, 'Date', date_to_iso8601(expiration.date)) elif expiration.created_before_date is not None: _add_text_child(expiration_node, 'CreatedBeforeDate', date_to_iso8601(expiration.created_before_date)) + elif expiration.expired_detete_marker is not None: + _add_text_child(expiration_node, 'ExpiredObjectDeleteMarker', str(expiration.expired_detete_marker)) abort_multipart_upload = rule.abort_multipart_upload if abort_multipart_upload: @@ -880,6 +912,19 @@ def to_put_bucket_lifecycle(bucket_lifecycle): tag_node = ElementTree.SubElement(rule_node, 'Tag') _add_text_child(tag_node, 'Key', key) _add_text_child(tag_node, 'Value', tagging_rule[key]) + + noncurrent_version_expiration = rule.noncurrent_version_expiration + if noncurrent_version_expiration is not None: + version_expiration_node = ElementTree.SubElement(rule_node, 'NoncurrentVersionExpiration') + _add_text_child(version_expiration_node, 'NoncurrentDays', str(noncurrent_version_expiration.noncurrent_days)) + + noncurrent_version_sotrage_transitions = rule.noncurrent_version_sotrage_transitions + if noncurrent_version_sotrage_transitions is not None: + for noncurrent_version_sotrage_transition in noncurrent_version_sotrage_transitions: + version_transition_node = ElementTree.SubElement(rule_node, 'NoncurrentVersionTransition') + _add_text_child(version_transition_node, 'NoncurrentDays', str(noncurrent_version_sotrage_transition.noncurrent_days)) + _add_text_child(version_transition_node, 'StorageClass', str(noncurrent_version_sotrage_transition.storage_class)) + return _node_to_string(root) diff --git a/tests/test_lifecycle_versioning.py b/tests/test_lifecycle_versioning.py new file mode 100644 index 00000000..091e6657 --- /dev/null +++ b/tests/test_lifecycle_versioning.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- + + +import datetime +import json + +from .common import * +from oss2 import to_string +from oss2.models import (BucketVersioningConfig, + TaggingRule, + Tagging, + LifecycleRule, + AbortMultipartUpload, + BucketLifecycle, + LifecycleExpiration, + StorageTransition, + NoncurrentVersionStorageTransition, + NoncurrentVersionExpiration) + + +class TestLifecycleVersioning(OssTestCase): + def setUp(self): + OssTestCase.setUp(self) + self.endpoint = "http://oss-ap-south-1.aliyuncs.com" + + def test_lifecycle_without_versioning(self): + auth = oss2.Auth(OSS_ID, OSS_SECRET) + bucket_name = OSS_BUCKET + "-test-lifecycle-without-versioning" + bucket = oss2.Bucket(auth, self.endpoint, bucket_name) + bucket.create_bucket(oss2.BUCKET_ACL_PRIVATE) + wait_meta_sync() + + rule1 = LifecycleRule('rule1', 'tests/', + status=LifecycleRule.ENABLED, + expiration=LifecycleExpiration(days=3)) + + rule2 = LifecycleRule('rule2', 'logging-', + status=LifecycleRule.ENABLED, + expiration=LifecycleExpiration(created_before_date=datetime.date(2020, 12, 12))) + + rule3 = LifecycleRule('rule3', 'tests1/', + status=LifecycleRule.ENABLED, + abort_multipart_upload=AbortMultipartUpload(days=3)) + + rule4 = LifecycleRule('rule4', 'logging1-', + status=LifecycleRule.ENABLED, + abort_multipart_upload=AbortMultipartUpload(created_before_date=datetime.date(2020, 12, 12))) + + tagging_rule = TaggingRule() + tagging_rule.add('test_key1', 'test_value1') + tagging_rule.add('test_key2', 'test_value2') + tagging = Tagging(tagging_rule) + rule5 = LifecycleRule('rule5', 'logging2-', + status=LifecycleRule.ENABLED, + storage_transitions= + [StorageTransition(days=100, storage_class=oss2.BUCKET_STORAGE_CLASS_IA), + StorageTransition(days=356, storage_class=oss2.BUCKET_STORAGE_CLASS_ARCHIVE)], + tagging = tagging) + + lifecycle = BucketLifecycle([rule1, rule2, rule3, rule4, rule5]) + + bucket.put_bucket_lifecycle(lifecycle) + + lifecycle = bucket.get_bucket_lifecycle() + self.assertEquals(5, len(lifecycle.rules)) + + bucket.delete_bucket() + + def test_lifecycle_versioning(self): + auth = oss2.Auth(OSS_ID, OSS_SECRET) + bucket_name = OSS_BUCKET + "-test-lifecycle-versioning" + bucket = oss2.Bucket(auth, self.endpoint, bucket_name) + bucket.create_bucket(oss2.BUCKET_ACL_PRIVATE) + wait_meta_sync() + + rule = LifecycleRule('rule1', 'test-prefix', + status=LifecycleRule.ENABLED, + expiration=LifecycleExpiration(expired_detete_marker=True), + noncurrent_version_expiration = NoncurrentVersionExpiration(30), + noncurrent_version_sotrage_transitions = + [NoncurrentVersionStorageTransition(12, oss2.BUCKET_STORAGE_CLASS_IA), + NoncurrentVersionStorageTransition(20, oss2.BUCKET_STORAGE_CLASS_ARCHIVE)]) + + lifecycle = BucketLifecycle([rule]) + + bucket.put_bucket_lifecycle(lifecycle) + + lifecycle = bucket.get_bucket_lifecycle() + self.assertEquals(1, len(lifecycle.rules)) + self.assertEqual('rule1', lifecycle.rules[0].id) + self.assertEqual('test-prefix', lifecycle.rules[0].prefix) + self.assertEquals(LifecycleRule.ENABLED, lifecycle.rules[0].status) + self.assertEquals(True, lifecycle.rules[0].expiration.expired_detete_marker) + self.assertEquals(30, lifecycle.rules[0].noncurrent_version_expiration.noncurrent_days) + self.assertEquals(2, len(lifecycle.rules[0].noncurrent_version_sotrage_transitions)) + bucket.delete_bucket() + + def test_lifecycle_veriong_wrong(self): + auth = oss2.Auth(OSS_ID, OSS_SECRET) + bucket_name = OSS_BUCKET + "-test-lifecycle-versioning-wrong" + bucket = oss2.Bucket(auth, self.endpoint, bucket_name) + bucket.create_bucket(oss2.BUCKET_ACL_PRIVATE) + wait_meta_sync() + # days and expired_detete_marker cannot both exsit. + self.assertRaises(oss2.exceptions.ClientError, LifecycleExpiration, days=10, expired_detete_marker=True) + + rule = LifecycleRule('rule1', 'test-prefix', + status=LifecycleRule.ENABLED, + expiration=LifecycleExpiration(expired_detete_marker=True), + noncurrent_version_expiration = NoncurrentVersionExpiration(30), + noncurrent_version_sotrage_transitions = + [NoncurrentVersionStorageTransition(20, oss2.BUCKET_STORAGE_CLASS_IA), + NoncurrentVersionStorageTransition(12, oss2.BUCKET_STORAGE_CLASS_ARCHIVE)]) + + lifecycle = BucketLifecycle([rule]) + + # Archive transition days < IA transition days + self.assertRaises(oss2.exceptions.InvalidArgument, bucket.put_bucket_lifecycle, lifecycle) + + bucket.delete_bucket() + + +if __name__ == '__main__': + unittest.main()