# -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Gsutil API for interacting with cloud storage providers.""" from __future__ import absolute_import class CloudApi(object): """Abstract base class for interacting with cloud storage providers. Implementations of the gsutil Cloud API are not guaranteed to be thread-safe. Behavior when calling a gsutil Cloud API instance simultaneously across threads is undefined and doing so will likely cause errors. Therefore, a separate instance of the gsutil Cloud API should be instantiated per-thread. """ def __init__(self, bucket_storage_uri_class, logger, provider=None, debug=0, trace_token=None, perf_trace_token=None): """Performs necessary setup for interacting with the cloud storage provider. Args: bucket_storage_uri_class: boto storage_uri class, used by APIs that provide boto translation or mocking. logger: logging.logger for outputting log messages. provider: Default provider prefix describing cloud storage provider to connect to. debug: Debug level for the API implementation (0..3). trace_token: Google internal trace token to pass to the API implementation (string). perf_trace_token: Performance trace token to use when making API calls. """ self.bucket_storage_uri_class = bucket_storage_uri_class self.logger = logger self.provider = provider self.debug = debug self.trace_token = trace_token self.perf_trace_token = perf_trace_token def GetBucket(self, bucket_name, provider=None, fields=None): """Gets Bucket metadata. Args: bucket_name: Name of the bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Bucket metadata fields, for example, ['logging', 'defaultObjectAcl'] Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Bucket object. """ raise NotImplementedError('GetBucket must be overloaded') def ListBuckets(self, project_id=None, provider=None, fields=None): """Lists bucket metadata for the given project. Args: project_id: Project owning the buckets, default from config if None. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these metadata fields for the listing, for example: ['items/logging', 'items/defaultObjectAcl']. Note that the WildcardIterator class should be used to list buckets instead of calling this function directly. It amends the fields definition from get-like syntax such as ['logging', 'defaultObjectAcl'] so that the caller does not need to prepend 'items/' or specify fields necessary for listing (like nextPageToken). Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Iterator over Bucket objects. """ raise NotImplementedError('ListBuckets must be overloaded') def PatchBucket(self, bucket_name, metadata, canned_acl=None, canned_def_acl=None, preconditions=None, provider=None, fields=None): """Updates bucket metadata for the bucket with patch semantics. Args: bucket_name: Name of bucket to update. metadata: Bucket object defining metadata to be updated. canned_acl: Canned ACL to apply to the bucket. canned_def_acl: Canned default object ACL to apply to the bucket. preconditions: Preconditions for the request. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Bucket metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Bucket object describing new bucket metadata. """ raise NotImplementedError('PatchBucket must be overloaded') def CreateBucket(self, bucket_name, project_id=None, metadata=None, provider=None, fields=None): """Creates a new bucket with the specified metadata. Args: bucket_name: Name of the new bucket. project_id: Project owner of the new bucket, default from config if None. metadata: Bucket object defining new bucket metadata. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Bucket metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Bucket object describing new bucket metadata. """ raise NotImplementedError('CreateBucket must be overloaded') def DeleteBucket(self, bucket_name, preconditions=None, provider=None): """Deletes a bucket. Args: bucket_name: Name of the bucket to delete. preconditions: Preconditions for the request. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ raise NotImplementedError('DeleteBucket must be overloaded') class CsObjectOrPrefixType(object): """Enum class for describing CsObjectOrPrefix types.""" OBJECT = 'object' # Cloud object PREFIX = 'prefix' # Cloud bucket subdirectory class CsObjectOrPrefix(object): """Container class for ListObjects results.""" def __init__(self, data, datatype): """Stores a ListObjects result. Args: data: Root object, either an apitools Object or a string Prefix. datatype: CsObjectOrPrefixType of data. """ self.data = data self.datatype = datatype def ListObjects(self, bucket_name, prefix=None, delimiter=None, all_versions=None, provider=None, fields=None): """Lists objects (with metadata) and prefixes in a bucket. Args: bucket_name: Bucket containing the objects. prefix: Prefix for directory-like behavior. delimiter: Delimiter for directory-like behavior. all_versions: If true, list all object versions. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these metadata fields for the listing, for example: ['items/acl', 'items/updated', 'prefixes']. Note that the WildcardIterator class should be used to list objects instead of calling this function directly. It amends the fields definition from get-like syntax such as ['acl', 'updated'] so that the caller does not need to prepend 'items/' or specify any fields necessary for listing (such as prefixes or nextPageToken). Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Iterator over CsObjectOrPrefix wrapper class. """ raise NotImplementedError('ListObjects must be overloaded') def GetObjectMetadata(self, bucket_name, object_name, generation=None, provider=None, fields=None): """Gets object metadata. If decryption is supported by the implementing class, this function will read decryption keys from configuration and appropriately retry requests to encrypted objects with the correct key. Args: bucket_name: Bucket containing the object. object_name: Object name. generation: Generation of the object to retrieve. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields, for example, ['acl', 'updated']. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object. """ raise NotImplementedError('GetObjectMetadata must be overloaded') def PatchObjectMetadata(self, bucket_name, object_name, metadata, canned_acl=None, generation=None, preconditions=None, provider=None, fields=None): """Updates object metadata with patch semantics. Args: bucket_name: Bucket containing the object. object_name: Object name for object. metadata: Object object defining metadata to be updated. canned_acl: Canned ACL to be set on the object. generation: Generation (or version) of the object to update. preconditions: Preconditions for the request. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Updated object metadata. """ raise NotImplementedError('PatchObjectMetadata must be overloaded') class DownloadStrategy(object): """Enum class for specifying download strategy.""" ONE_SHOT = 'oneshot' RESUMABLE = 'resumable' def GetObjectMedia(self, bucket_name, object_name, download_stream, provider=None, generation=None, object_size=None, compressed_encoding=False, download_strategy=DownloadStrategy.ONE_SHOT, start_byte=0, end_byte=None, progress_callback=None, serialization_data=None, digesters=None, decryption_tuple=None): """Gets object data. Args: bucket_name: Bucket containing the object. object_name: Object name. download_stream: Stream to send the object data to. provider: Cloud storage provider to connect to. If not present, class-wide default is used. generation: Generation of the object to retrieve. object_size: Total size of the object being downloaded. compressed_encoding: If true, object is stored with a compressed encoding. download_strategy: Cloud API download strategy to use for download. start_byte: Starting point for download (for resumable downloads and range requests). Can be set to negative to request a range of bytes (python equivalent of [:-3]) end_byte: Ending byte number, inclusive, for download (for range requests). If None, download the rest of the object. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). serialization_data: Implementation-specific JSON string of a dict containing serialization information for the download. digesters: Dict of {string : digester}, where string is a name of a hash algorithm, and digester is a validation digester that supports update(bytes) and digest() using that algorithm. Implementation can set the digester value to None to indicate bytes were not successfully digested on-the-fly. decryption_tuple: Optional CryptoTuple for decrypting an encrypted object. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Content-encoding string if it was detected that the server sent an encoded object during transfer, None otherwise. """ raise NotImplementedError('GetObjectMedia must be overloaded') def UploadObject(self, upload_stream, object_metadata, canned_acl=None, size=None, preconditions=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None): """Uploads object data and metadata. Args: upload_stream: Seekable stream of object data. object_metadata: Object metadata for new object. Must include bucket and object name. canned_acl: Optional canned ACL to apply to object. Overrides ACL set in object_metadata. size: Optional object size. preconditions: Preconditions for the request. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). encryption_tuple: Optional CryptoTuple for encrypting the uploaded object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. """ raise NotImplementedError('UploadObject must be overloaded') def UploadObjectStreaming(self, upload_stream, object_metadata, canned_acl=None, preconditions=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None): """Uploads object data and metadata. Args: upload_stream: Stream of object data. May not be seekable. object_metadata: Object metadata for new object. Must include bucket and object name. canned_acl: Optional canned ACL to apply to object. Overrides ACL set in object_metadata. preconditions: Preconditions for the request. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size), but fills in only bytes_transferred. encryption_tuple: Optional CryptoTuple for encrypting the uploaded object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. """ raise NotImplementedError('UploadObjectStreaming must be overloaded') def UploadObjectResumable( self, upload_stream, object_metadata, canned_acl=None, size=None, preconditions=None, serialization_data=None, tracker_callback=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None): """Uploads object data and metadata using a resumable upload strategy. Args: upload_stream: Seekable stream of object data. object_metadata: Object metadata for new object. Must include bucket and object name. canned_acl: Optional canned ACL to apply to object. Overrides ACL set in object_metadata. size: Total size of the object. preconditions: Preconditions for the request. serialization_data: Dict of {'url' : UploadURL} allowing for uploads to be resumed. tracker_callback: Callback function taking a upload URL string. Guaranteed to be called when the implementation gets an upload URL, allowing the caller to resume the upload across process breaks by saving the upload URL in a tracker file. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). encryption_tuple: Optional CryptoTuple for encrypting the uploaded object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields when the upload is complete. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. """ raise NotImplementedError('UploadObjectResumable must be overloaded') def CopyObject(self, src_obj_metadata, dst_obj_metadata, src_generation=None, canned_acl=None, preconditions=None, progress_callback=None, max_bytes_per_call=None, encryption_tuple=None, decryption_tuple=None, provider=None, fields=None): """Copies an object in the cloud. Args: src_obj_metadata: Object metadata for source object. Must include bucket name, object name, and etag. dst_obj_metadata: Object metadata for new object. Must include bucket and object name. src_generation: Generation of the source object to copy. canned_acl: Optional canned ACL to apply to destination object. Overrides ACL set in dst_obj_metadata. preconditions: Destination object preconditions for the request. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). max_bytes_per_call: Integer describing maximum number of bytes to rewrite per service call. encryption_tuple: Optional CryptoTuple for encrypting the destination object. decryption_tuple: Optional CryptoTuple for decrypting the source object. If supplied without encryption_tuple, destination object will be written without customer-supplied encryption. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. """ raise NotImplementedError('CopyObject must be overloaded') def ComposeObject(self, src_objs_metadata, dst_obj_metadata, preconditions=None, encryption_tuple=None, provider=None, fields=None): """Composes an object in the cloud. Args: src_objs_metadata: List of ComposeRequest.SourceObjectsValueListEntries specifying the objects to compose. dst_obj_metadata: Metadata for the destination object including bucket and object name. preconditions: Destination object preconditions for the request. encryption_tuple: Optional CryptoTuple for decrypting source objects and encrypting the destination object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Composed object metadata. """ raise NotImplementedError('ComposeObject must be overloaded') def DeleteObject(self, bucket_name, object_name, preconditions=None, generation=None, provider=None): """Deletes an object. Args: bucket_name: Name of the containing bucket. object_name: Name of the object to delete. preconditions: Preconditions for the request. generation: Generation (or version) of the object to delete; if None, deletes the live object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ raise NotImplementedError('DeleteObject must be overloaded') def WatchBucket(self, bucket_name, address, channel_id, token=None, provider=None, fields=None): """Creates a notification subscription for changes to objects in a bucket. Args: bucket_name: Bucket containing the objects. address: Address to which to send notifications. channel_id: Unique ID string for the channel. token: If present, token string is delivered with each notification. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Channel metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Channel object describing the notification subscription. """ raise NotImplementedError('WatchBucket must be overloaded') def StopChannel(self, channel_id, resource_id, provider=None): """Stops a notification channel. Args: channel_id: Unique ID string for the channel. resource_id: Version-agnostic ID string for the channel. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ raise NotImplementedError('StopChannel must be overloaded') class CryptoTuple(object): """Class describing an encryption/decryption key for cloud API requests.""" def __init__(self, crypto_key): """Initialize the CryptoTuple. Args: crypto_key: Base64-encoded string of encryption key. """ self.crypto_key = crypto_key self.crypto_alg = 'AES256' # Only currently supported encryption algorithm. class Preconditions(object): """Preconditions class for specifying preconditions to cloud API requests.""" def __init__(self, gen_match=None, meta_gen_match=None): """Instantiates a Preconditions object. Args: gen_match: Perform request only if generation of target object matches the given integer. Ignored for bucket requests. meta_gen_match: Perform request only if metageneration of target object/bucket matches the given integer. """ self.gen_match = gen_match self.meta_gen_match = meta_gen_match class EncryptionException(Exception): """Exception raised when an encrypted resource cannot be decrypted.""" class ArgumentException(Exception): """Exception raised when arguments to a Cloud API method are invalid. This exception is never raised as a result of a failed call to a cloud storage provider. """ def __init__(self, reason): Exception.__init__(self) self.reason = reason def __repr__(self): return str(self) def __str__(self): return '%s: %s' % (self.__class__.__name__, self.reason) class ProjectIdException(ArgumentException): """Exception raised when a Project ID argument is required but not present.""" class ServiceException(Exception): """Exception raised when a cloud storage provider request fails. This exception is raised only as a result of a failed remote call. """ def __init__(self, reason, status=None, body=None): Exception.__init__(self) self.reason = reason self.status = status self.body = body def __repr__(self): return str(self) def __str__(self): message = '%s:' % self.__class__.__name__ if self.status: message += ' %s' % self.status message += ' %s' % self.reason if self.body: message += '\n%s' % self.body return message class RetryableServiceException(ServiceException): """Exception class for retryable exceptions.""" class ResumableDownloadException(RetryableServiceException): """Exception raised for res. downloads that can be retried later.""" class ResumableUploadException(RetryableServiceException): """Exception raised for res. uploads that can be retried w/ same upload ID.""" class ResumableUploadStartOverException(RetryableServiceException): """Exception raised for res. uploads that can be retried w/ new upload ID.""" class ResumableUploadAbortException(ServiceException): """Exception raised for resumable uploads that cannot be retried later.""" class AuthenticationException(ServiceException): """Exception raised for errors during the authentication process.""" class PreconditionException(ServiceException): """Exception raised for precondition failures.""" class NotFoundException(ServiceException): """Exception raised when a resource is not found (404).""" class BucketNotFoundException(NotFoundException): """Exception raised when a bucket resource is not found (404).""" def __init__(self, reason, bucket_name, status=None, body=None): super(BucketNotFoundException, self).__init__(reason, status=status, body=body) self.bucket_name = bucket_name class NotEmptyException(ServiceException): """Exception raised when trying to delete a bucket is not empty.""" class BadRequestException(ServiceException): """Exception raised for malformed requests. Where it is possible to detect invalid arguments prior to sending them to the server, an ArgumentException should be raised instead. """ class AccessDeniedException(ServiceException): """Exception raised when authenticated user has insufficient access rights. This is raised when the authentication process succeeded but the authenticated user does not have access rights to the requested resource. """