OXIESEC PANEL
- Current Dir:
/
/
opt
/
gsutil
/
gslib
/
utils
/
__pycache__
Server IP: 2a02:4780:11:1594:0:ef5:22d7:a
Upload:
Create Dir:
Name
Size
Modified
Perms
📁
..
-
02/11/2025 08:19:48 AM
rwxr-xr-x
📄
__init__.cpython-39.pyc
126 bytes
02/11/2025 08:19:48 AM
rw-r--r--
📄
acl_helper.cpython-39.pyc
9.38 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
arg_helper.cpython-39.pyc
1.01 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
boto_util.cpython-39.pyc
16.46 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
cat_helper.cpython-39.pyc
4.21 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
cloud_api_helper.cpython-39.pyc
3.29 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
constants.cpython-39.pyc
2.88 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
copy_helper.cpython-39.pyc
101.64 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
encryption_helper.cpython-39.pyc
5.84 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
execution_util.cpython-39.pyc
1.06 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
hashing_helper.cpython-39.pyc
15.71 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
iam_helper.cpython-39.pyc
8.61 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
ls_helper.cpython-39.pyc
12.4 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
metadata_util.cpython-39.pyc
3.1 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
parallelism_framework_util.cpython-39.pyc
11.79 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
posix_util.cpython-39.pyc
12.46 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
retention_util.cpython-39.pyc
8.77 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
retry_util.cpython-39.pyc
2.5 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
rsync_util.cpython-39.pyc
1.6 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
shim_util.cpython-39.pyc
19.8 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
signurl_helper.cpython-39.pyc
4.21 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
stet_util.cpython-39.pyc
3.35 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
system_util.cpython-39.pyc
7.9 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
temporary_file_util.cpython-39.pyc
915 bytes
02/11/2025 08:19:49 AM
rw-r--r--
📄
text_util.cpython-39.pyc
12.62 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
translation_helper.cpython-39.pyc
25.67 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
unit_util.cpython-39.pyc
7.01 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
update_util.cpython-39.pyc
3.39 KB
02/11/2025 08:19:49 AM
rw-r--r--
📄
user_agent_helper.cpython-39.pyc
1.49 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
version_check.cpython-39.pyc
1.39 KB
02/11/2025 08:19:48 AM
rw-r--r--
📄
wrapped_credentials.cpython-39.pyc
5.81 KB
02/11/2025 08:19:49 AM
rw-r--r--
Editing: copy_helper.cpython-39.pyc
Close
a +(Wg�� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZ ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddlZdd l m Z dd l m Z dd l!m"Z" dd l#m$Z$ ddl%Z%ddl&Z&dd l'm(Z( dd l'm)Z) ddl'm*Z* ddl'm+Z+ ddl'm,Z, ddl'm-Z- ddl'm.Z. ddl'm/Z/ ddl'm0Z0 ddl'm1Z1 ddl'm2Z2 ddl'm3Z3 ddl4m5Z5 ddl6m7Z7 ddl6m8Z8 ddl6m9Z9 dd l6m:Z: dd l6m;Z; dd l6m<Z< dd l=m>Z> dd l?m@Z@ dd!lAmBZB dd"lAmCZC dd#lAmDZD dd$lEmFZF dd%lGmHZH dd&lGmIZI dd'lGmJZJ dd(lGmKZK dd)lGmLZL dd*lMmNZN dd+lMmOZO dd,lPmQZQ dd-l&mRZR dd.lSmTZT dd/lSmUZU dd0lSmVZV dd1lSmWZW dd2lXmYZZ dd3l[m\Z\ dd4l[m]Z] dd5l^m_Z_ dd6l^m`Z` dd7l^maZa dd8l^mbZb dd9l^mcZc dd:l^mdZd dd;l^meZe dd<l^mfZf dd=l^mgZg dd>l^mhZh dd?l^miZi dd@l^mjZj ddAlkmlZl ddBlkmmZm ddClkmnZn ddDlkmoZo ddElpmqZq ddFlpmrZr ddGlpmsZs ddHlpmtZt ddIlpmuZu ddJlvmwZw ddKlvmxZx ddLlymzZz ddMlym{Z{ ddNlym|Z| ddOl}m~Z~ ddPl}mZ ddQl}m�Z� ddRl}m�Z� ddSl�m�Z� ddTl�m�Z� ddUl�m�Z� ddVl�m�Z� ddWl�m�Z� ddXl�m�Z� ddYl�m�Z� ddZl�m�Z� dd[l�m�Z� dd\l�m�Z� dd]l�m�Z� dd^l�m�Z� dd_l�m�Z� dd`l�m�Z� ddal�m�Z� ddbl�m�Z� ddcl�m�Z� dddl�m�Z� ddel�m�Z� ddfl�m�Z� ddgl�m�Z� ddhl�m�Z� ddil�m�Z� ddjl�m�Z� ddkl�m�Z� ddll�m�Z� ddml�m�Z� ddnl�m�Z� ddol�m�Z� ddpl�m�Z� ddql�m�Z� ddrl�m�Z� ddsl�m�Z� ddtl�m�Z� ddul�m�Z� ddvl�m�Z� ddwl�m�Z� ej��rXe�Z�e��rfddl�Z�e�e�� j��rxelj�nddx�a�el��� a�dyZ�dzZ�d{Z�d|Z�g d}�Z�ed~d�Z�ed�d��Z�ed�d��Z�d�Z�d�Z�e�Z�d�Z�d�Z�e�e�� j��r�elj�nddx�a�el��� a�d�d�d��Z�G d�d�� d�eȃZ�d�d�� Z�d�d�� Zːdd�d��Z�ed�g d���Z͐dd�d��Z�d�d�� Z�d�d�� Z�d�d�� Z�d�d�� Z�d�d�� Z�d�d�� ZԐdd�d��Z�d�d�� Z�d�d�� Z�d�d�� Zؐdd�d��Z�d�d�� Z�d�d�� Zېdd�d��Z�d�d�� Zݐdd�d��Zސdd�d��Zߐdd�d��Z�d�d�� Z�d�d�� Z�d�d�� Z�d�d Z�dd�dĄZ�d�dƄ Z�dd�dȄZ�dd�dʄZ�dd�d̄Z�d�d΄ Z�d�dЄ Z�d d�dӄZ�d�dՄ Z�d�dׄ Z�d d�dلZ�d�dۄ Z�G d�d݄ d�e�Z�d d�d߄Z�d d�d�Z�d d�d�Z��d!d�d�Z��d"d�d�Z��d#d�d�Z��d$d�d�Z�d�d� Z��d%d�d�Z��d&d�d�Z�d�e� d�e� d�e� d��Z�d�d�� Z�d�d�� Z�d�d�� �Z �d'd�d���ZG d�d�� d�e��ZG �d �d� �deȃ�ZG �d�d� �deȃ�ZG �d�d� �d�e��Z�d�d� �Z�d�d � �Z�d �d � �Z�d(�d �d ��Z �d�d� �Z dS () z(Helper functions for copy functionality.� )�absolute_import)�print_function)�division)�unicode_literalsN)� namedtuple)� attrgetter)�range)� protojson)�config)�AccessDeniedException)�ArgumentException)�CloudApi)�EncryptionException)�NotFoundException)�PreconditionException)� Preconditions)�ResumableDownloadException)� ResumableUploadAbortException)�ResumableUploadException)�!ResumableUploadStartOverException)�ServiceException)�MAX_COMPOSE_ARITY)�0DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE)�+DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD)�-DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE)�-DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS)�(DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD)� DEFAULT_GZIP_COMPRESSION_LEVEL)� ApiSelector)�DaisyChainWrapper)�CommandException)�HashMismatchException)�InvalidUrlError)�FilePart)� GenerateComponentObjectPrefix)� ReadParallelUploadTrackerFile)�$ValidateParallelCompositeTrackerData)�)WriteComponentToParallelUploadTrackerFile)� WriteParallelUploadTrackerFile)�FileProgressCallbackHandler)�ProgressCallbackWithTimeout)�#ResumableStreamingJsonUploadWrapper)� storage_url)�ContainsWildcard)�GenerationFromUrlAndString)�IsCloudSubdirPlaceholder)�StorageUrlFromString)�storage_v1_messages)� FileMessage)�RetryableErrorMessage)�DeleteDownloadTrackerFiles)�DeleteTrackerFile)� ENCRYPTION_UPLOAD_TRACKER_ENTRY)�GetDownloadStartByte)�GetTrackerFilePath)�GetUploadTrackerData)�#RaiseUnwritableTrackerFileException)� ReadOrCreateDownloadTrackerFile)�"SERIALIZATION_UPLOAD_TRACKER_ENTRY)�TrackerFileType)�!WriteDownloadComponentTrackerFile)�WriteJsonDataToTrackerFile)�parallelism_framework_util)� stet_util)�temporary_file_util)� text_util)�GetJsonResumableChunkSize)�GetMaxRetryDelay)� GetNumRetries)�ResumableThreshold)�UsingCrcmodExtension)�GetCloudApiInstance)� GetDownloadSerializationData)�DEFAULT_FILE_BUFFER_SIZE)�MIN_SIZE_COMPUTE_LOGGING)�UTF8)� CryptoKeyType)�CryptoKeyWrapperFromKey)� FindMatchingCSEKInBotoConfig)�GetEncryptionKeyWrapper)�Base64EncodeHash)�"CalculateB64EncodedMd5FromContents)�CalculateHashesFromContents)� CHECK_HASH_IF_FAST_ELSE_FAIL)�CHECK_HASH_NEVER)� ConcatCrc32c)�GetDownloadHashAlgs)�GetMd5)�GetUploadHashAlgs)�HashingFileUploadWrapper)�ObjectIsGzipEncoded)� AtomicDict)�$CheckMultiprocessingAvailableAndInit)�PutToQueueWithTimeout)� ATIME_ATTR)�ConvertDatetimeToPOSIX)�GID_ATTR)� MODE_ATTR)� MTIME_ATTR)�ParseAndSetPOSIXAttributes)�UID_ATTR)�CheckFreeSpace)� GetFileSize)�GetStreamFromFileUrl)� IS_WINDOWS)� AddS3MarkerAclToObjectMetadata)�CopyObjectMetadata)�DEFAULT_CONTENT_TYPE)�ObjectMetadataFromHeaders)�PreconditionsFromHeaders)� S3MarkerAclFromObjectMetadata)� DivideAndCeil)�HumanReadableToBytes)�MakeHumanReadable)�SECONDS_PER_DAY)�TEN_MIB)�CreateWildcardIterator)�managerFZ_corruptzF/gsutil/tmp/parallel_composite_uploads/for_details_see/gsutil_help_cp/ad PARALLEL_UPLOAD_SALT_TO_PREVENT_COLLISIONS. The theory is that no user will have prepended this to the front of one of their object names and then done an MD5 hash of the name, and then prepended PARALLEL_UPLOAD_TEMP_NAMESPACE to the front of their object name. Note that there will be no problems with object name length since we hash the original name. )�crc32c�customerEncryption�etag� generation�md5Hash�size�%PerformParallelUploadFileToObjectArgsz�filename file_start file_length src_url dst_url canned_acl content_type storage_class tracker_file tracker_file_lock encryption_key_sha256 gzip_encoded�%PerformSlicedDownloadObjectToFileArgszicomponent_num src_url src_obj_metadata_json dst_url download_file_name start_byte end_byte decryption_key�!PerformSlicedDownloadReturnValueszKcomponent_num crc32c bytes_transferred component_total_size server_encodingi �GZIP_ALL_FILESi ` l z text/markdownzapplication/gzip)�mdZtgzc @ s e Zd ZdZdS )�FileConcurrencySkipErrorz@Raised when skipping a file due to a concurrent, duplicate copy.N��__name__� __module__� __qualname__�__doc__� r� r� �&/opt/gsutil/gslib/utils/copy_helper.pyr� + s r� c C s | j �t|�� dS )�9Simple exception handler to allow post-completion status.N)�logger�error�str��cls�er� r� r� �_RmExceptionHandler/ s r� c C s4 | j �t|�� | jd7 _| j �dt�� � dS )r� � z* Encountered exception while copying: %s N)r� r� r� �op_failure_count�debug� traceback� format_excr� r� r� r� � _ParallelCopyExceptionHandler4 s �r� c C s� t |j|j|j�}t| |d�}|�� d}tj|jj|jj |j |j d�}|j }zLt jr\tj|_ t|j||j|j|||| j| tddd|jd�}W t jr�||_ nt jr�||_ 0 W d � n1 s�0 Y |d } t|j|j| | j|jd� |S ) aO Function argument to Apply for performing parallel composite uploads. Args: cls: Calling Command class. args: PerformParallelUploadFileToObjectArgs tuple describing the target. thread_state: gsutil Cloud API instance to use for the operation. Returns: StorageUrl representing a successfully uploaded component. �� thread_stateN)�name�bucket� contentType� storageClassFT)� gzip_exts�allow_splitting� is_component� gzip_encoded� ��encryption_key_sha256)r# �filename� file_start� file_lengthrI �apitools_messages�Object�dst_url� object_name� bucket_name� content_type� storage_class� prefer_api�global_copy_helper_opts� canned_aclr �XML�_UploadFileToObject�src_urlr� r� r� r' � tracker_file�tracker_file_lockr� ) r� �argsr� �fp� gsutil_api� preconditionsZdst_object_metadata�orig_prefer_api�ret� componentr� r� r� �"_PerformParallelUploadFileToObject< sT ���&�r� �CopyHelperOpts� � perform_mv� no_clobber� daisy_chain�read_args_from_stdin� print_ver� use_manifest� preserve_aclr� �skip_unsupported_objects�test_callback_file�dest_storage_classc C s" t | ||||||||| | d� atS )z9Creates CopyHelperOpts for passing options to CopyHelper.r� )r� r� r� r� r� r� �CreateCopyHelperOpts� s � r� c C s t S )z.Returns namedtuple holding CopyHelper options.)r� r� r� r� r� �GetCopyHelperOpts� s r� c C sh d}| � � rP| jtjkr d}z t�| j�j}t�|�r:d}W n tyN Y n0 |r\tj j S tj j S dS )z�Get download strategy based on the destination object. Args: dst_url: Destination StorageUrl. Returns: gsutil Cloud API DownloadStrategy. FTN) � IsFileUrlr� �os�devnull�stat�st_mode�S_ISCHR�OSErrorr �DownloadStrategy�ONE_SHOT� RESUMABLE)r� Zdst_is_special�moder� r� r� �_SelectDownloadStrategy� s r� c C s4 | � � r| �� r$| �� r0| �� r0|s0td| ��dS )a� Ensures the destination URL names a container. Acceptable containers include directory, bucket, bucket subdir, and non-existent bucket subdir. Args: exp_dst_url: Wildcard-expanded destination StorageUrl. have_existing_dst_container: bool indicator of whether exp_dst_url names a container (directory, bucket, or existing bucket subdir). command_name: Name of command making call. May not be the same as the calling class's self.command_name in the case of commands implemented atop other commands (like mv command). Raises: CommandException: if the URL being checked does not name a container. zuDestination URL must name a directory, bucket, or bucket subdirectory for the multiple source form of the %s command.N)r� � IsDirectory� IsCloudUrl�IsBucketr )� exp_dst_url�have_existing_dst_container� command_namer� r� r� �InsistDstUrlNamesContainer� s ����r� c C s |rdS |� � r | p|o|S dS )a" Checks whether dst_url should be treated as a bucket "sub-directory". The decision about whether something constitutes a bucket "sub-directory" depends on whether there are multiple sources in this request and whether there is an existing bucket subdirectory. For example, when running the command: gsutil cp file gs://bucket/abc if there's no existing gs://bucket/abc bucket subdirectory we should copy file to the object gs://bucket/abc. In contrast, if there's an existing gs://bucket/abc bucket subdirectory we should copy file to gs://bucket/abc/file. And regardless of whether gs://bucket/abc exists, when running the command: gsutil cp file1 file2 gs://bucket/abc we should copy file1 to gs://bucket/abc/file1 (and similarly for file2). Finally, for recursive copies, if the source is a container then we should copy to a container as the target. For example, when running the command: gsutil cp -r dir1 gs://bucket/dir2 we should copy the subtree of dir1 to gs://bucket/dir2. Note that we don't disallow naming a bucket "sub-directory" where there's already an object at that URL. For example it's legitimate (albeit confusing) to have an object called gs://bucket/dir and then run the command gsutil cp file1 file2 gs://bucket/dir Doing so will end up with objects gs://bucket/dir, gs://bucket/dir/file1, and gs://bucket/dir/file2. Args: have_multiple_srcs: Bool indicator of whether this is a multi-source operation. dst_url: StorageUrl to check. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. src_url_names_container: bool indicator of whether the source URL is a container. recursion_requested: True if a recursive operation has been requested. Returns: bool indicator. TN)r� )�have_multiple_srcsr� �have_existing_dest_subdir�src_url_names_container�recursion_requestedr� r� r� � _ShouldTreatDstUrlAsBucketSubDir� s ,�r� c C s6 |r | r dS |� � r |�� S | o0| o0|�� S dS )av Checks that dst_url names a single file/object after wildcard expansion. It is possible that an object path might name a bucket sub-directory. Args: src_url_names_container: Bool indicator of whether the source for the operation is a container (bucket, bucket subdir, or directory). have_multiple_srcs: Bool indicator of whether this is a multi-source operation. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. dst_url: StorageUrl to check. recursion_requested: True if a recursive operation has been requested. Returns: bool indicator. FN)r� r� �IsObject)r� r� r� r� r� r� r� r� � _ShouldTreatDstUrlAsSingleton s �r� c sN � � � s dS � j�� j�}|�� j�\}}}|tjvoL|� fdd�tjD �vS )ac Returns True if not FileUrl ending in relative path symbols. A URL is invalid if it is a FileUrl and the parent directory of the file is a relative path symbol. Unix will not allow a file itself to be named with a relative path symbol, but one can be the parent. Notably, "../obj" can lead to unexpected behavior at the copy destination. We examine the pre-recursion "url", which might point to "..", to see if the parent is valid. If the user does a recursive copy from a URL, it may not end up the final parent of the copied object. For example, see: "dir/nested_dir/obj". If you ran "cp -r dir gs://bucket" from the parent of "dir", then the "url" arg would be "dir", but "nested_dir" would be the parent of "obj". This actually doesn't matter since recursion won't add relative path symbols to the path. However, we still return if "url" is valid because there are cases where we need to copy every parent directory up to "dir" to prevent file name conflicts. Args: url: StorageUrl before recursion. Returns: Boolean indicating if the "url" is valid as a parent directory. Tc s g | ]}� j d | �qS )z://)�scheme)�.0�symbol��urlr� r� � <listcomp>U s �z(_IsUrlValidParentDir.<locals>.<listcomp>)r� �versionless_url_string�rstrip�delim� rpartitionr, �RELATIVE_PATH_SYMBOLS)r� Z#url_string_minus_trailing_delimiter�_Zafter_last_delimiterr� r� r� �_IsUrlValidParentDir3 s �� ��r� c C s |� � r |�� r |r td��t|||||�r0|S |� � rl|�� sH|�� rl|rh|�� rXdnd} td| ��|S |s�|s�|j�| j�d } td|j � |j�|j| f �S |s�|s�|� � r�|� � r�|j � |j�s�td|j |jf �}t | �} | s�|r�td�| ���d | jv�o| �o|�p|} | �s8|�rt|�� �s8|� � �rtt| |�} |jt| �d � �| j�}| �s�|�| j�d }n|j�| j�d }|� � �s�t|||||��r�|j�r�|j� |j��r�d|j� |j�|j|f }n(|j�r�|jnd }d|j�p�d ||f }|�� }|�| j|j�|_|S ) aQ Constructs the destination URL for a given exp_src_url/exp_dst_url pair. Uses context-dependent naming rules that mimic Linux cp and mv behavior. Args: src_url: Source StorageUrl to be copied. exp_src_url: Single StorageUrl from wildcard expansion of src_url. src_url_names_container: True if src_url names a container (including the case of a wildcard-named bucket subdir (like gs://bucket/abc, where gs://bucket/abc/* matched some objects). have_multiple_srcs: True if this is a multi-source request. This can be true if src_url wildcard-expanded to multiple URLs or if there were multiple source URLs in the request. has_multiple_top_level_srcs: Same as have_multiple_srcs but measured before recursion. exp_dst_url: the expanded StorageUrl requested for the cp destination. Final written path is constructed from this plus a context-dependent variant of src_url. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. recursion_requested: True if a recursive operation has been requested. preserve_posix: True if preservation of posix attributes has been requested. Returns: StorageUrl to use for copy. Raises: CommandException if destination object name not specified for source and source is a stream. z/Cannot preserve POSIX attributes with a stream.�stream� named pipez2Destination object name needed when source is a %s���z%s%s%sz%s%szmPresence of multiple top-level sources and invalid expanded URL make file name conflicts possible for URL: {}z**N� )r� �IsStreamr r� �IsFifor� r� r� r0 � url_stringr� r� �endswithr� r" �formatr� r� �GetPathBeforeFinalDir�len�lstrip� partitionr� �Clone�replace)r� � exp_src_urlr� r� Zhas_multiple_top_level_srcsr� r� r� �preserve_posixZ type_textZsrc_final_compZsrc_url_is_valid_parentZpreserve_src_top_level_dirs�src_url_path_sans_final_dirZ dst_key_namer� Znew_exp_dst_urlr� r� r� �ConstructDstUrl[ s� '��� ���� ����) �� �� �� ���� ��r c C s8 i }| r4| D ]&}t �| | �� ��d��d�||< q |S )N� �ascii)�base64� b64encode�digestr� �decode)� digesters�digests�algr� r� r� �_CreateDigestsFromDigesters s ��� r c C s� i }d|v rt � |d< d|v r.tj�d�|d< t|d��2}t||t|jt| |dd�j �d� W d � n1 sp0 Y i }t � |�D ]\}} t | � � �||< q�|S ) a� Creates a base64 CRC32C and/or MD5 digest from file_name. Args: status_queue: Queue for posting progress messages for UI/Analytics. algs: List of algorithms to compute. file_name: File to digest. src_url: StorageUrl for local object. Used to track progress. src_obj_metadata: Metadata of source object. Returns: Dict of algorithm name : base 64 encoded digest �md5rx zcrc-32c�rb�Hashing)r� �operation_name)�callback_processorN)rY �crcmod� predefined�Crc�openrT r* r} r) �call�six� iteritemsrR � hexdigest) � status_queueZalgs� file_namer� �src_obj_metadata� hash_dictr� r �alg_namer r� r� r� �_CreateDigestsFromLocalFile s* ���$r$ c C s t �ddt�}|tkrdS d}i }i }|jr@t�|j�} | |d< |jrZt�|j�} | |d< |jrtt�|j�} | |d< |jr�t�|j�} | |d< t�|�D ]f\} }| |vr�q�|| }tj r�t |t �r�|� d�}| � d| |||� ||kr�td | |||f ��d }q�|�s| �d ||� dS ) a� Validates integrity of two cloud objects copied via daisy-chain. Args: logger: for outputting log messages. src_url: CloudUrl for source cloud object. dst_url: CloudUrl for destination cloud object. src_obj_metadata: Cloud Object metadata for object being downloaded from. dst_obj_metadata: Cloud Object metadata for object being uploaded to. Raises: CommandException: if cloud digests don't match local digests. �GSUtil� check_hashesNFr rx r z;Comparing source vs destination %s-checksum for %s. (%s/%s)zn%s signature for source object (%s) doesn't match destination object digest (%s). Object (%s) will be deleted.Tz~WARNING: Found no hashes to validate object downloaded from %s and uploaded to %s. Integrity cannot be assured without hashes.)r �getrU rV r| r � ensure_binaryrx r �PY3� isinstancer� �encoder� r! �warn)r� r� r� r! �dst_obj_metadata�check_hashes_config� checked_oneZdownload_hashesZ upload_hashesZ src_md5hashZsrc_crc32c_hashZ dst_md5hashZdst_crc32c_hashr Zupload_b64_digestZdownload_b64_digestr� r� r� �_CheckCloudHashes. sT � � ���r0 c C s� |}i }|j r(t�|j �}|�d�|d< |jrHt�|j�} | �d�|d< d} |D ]h} | |vr^qPt�|| �} || } | �d| || | � | | kr�td| | | |r�dnd|r�|n|f ��d } qP| s�|r�| �d |� n | �d |� d S ) a Validates integrity by comparing cloud digest to local digest. Args: logger: for outputting log messages. obj_url: CloudUrl for cloud object. obj_metadata: Cloud Object being downloaded from or uploaded to. file_name: Local file name on disk being downloaded to or uploaded from (used only for logging). digests: Computed Digests for the object. is_upload: If true, comparing for an uploaded object (controls logging). Raises: CommandException: if cloud digests don't match local digests. r r rx Fz4Comparing local vs cloud %s-checksum for %s. (%s/%s)zl%s signature computed for local file (%s) doesn't match cloud-supplied digest (%s). %s (%s) will be deleted.z Cloud objectz Local fileTzgWARNING: Found no hashes to validate object uploaded to %s. Integrity cannot be assured without hashes.ziWARNING: Found no hashes to validate object downloaded to %s. Integrity cannot be assured without hashes.N)r| r r( r� rx r� r! r, )r� Zobj_url� obj_metadatar r � is_upload� local_hashes� cloud_hashesZmd5_b64_digestZcrc32c_b64_hashr/ r Zlocal_b64_digestZcloud_b64_digestr� r� r� � _CheckHashesn sL �� �����r5 c C s t | t�p t | t�o d| jv S )aR Checks to see if the server attempted to clobber a file. In this case we specified via a precondition that we didn't want the file clobbered. Args: e: The Exception that was generated by a failed copy operation Returns: bool indicator - True indicates that the server did attempt to clobber an existing file. Z412)r* r r �message)r� r� r� r� �IsNoClobberServerException� s �r7 c C s^ |� � r dS |j}tj�|�}tj�|�r<td| j|f ��tj�|�rZtd| j|f ��dS )a� Checks whether copying exp_src_url into dst_url is not possible. This happens if a directory exists in local file system where a file needs to go or vice versa. In that case we print an error message and exits. Example: if the file "./x" exists and you try to do: gsutil cp gs://mybucket/x/y . the request can't succeed because it requires a directory where the file x exists. Note that we don't enforce any corresponding restrictions for buckets, because the flat namespace semantics for buckets doesn't prohibit such cases the way hierarchical file systems do. For example, if a bucket contains an object called gs://bucket/dir and then you run the command: gsutil cp file1 file2 gs://bucket/dir you'll end up with objects gs://bucket/dir, gs://bucket/dir/file1, and gs://bucket/dir/file2. Args: exp_src_url: Expanded source StorageUrl. dst_url: Destination StorageUrl. Raises: CommandException: if errors encountered. NzTCannot retrieve %s because a file exists where a directory needs to be created (%s).zVCannot retrieve %s because a directory exists (%s) where the file needs to be created.) r� r� r� �path�dirname�isfiler r� �isdir)r r� Zdst_pathZ final_dirr� r� r� �CheckForDirFileConflict� s � �r< c C s� t t�ddt��} t|t| �\} }i }g }t| �D ]�}t�t |j �}t � }|� |� |� � }|t | d t|� }|�� }||_|| d k r�|}n|| d | }|| }t|j ||||| |||| | | � }|�|� |||< q2|S )a� Partitions a file into FilePart objects to be uploaded and later composed. These objects, when composed, will match the original file. This entails splitting the file into parts, naming and forming a destination URL for each part, and also providing the PerformParallelUploadFileToObjectArgs corresponding to each part. Args: canned_acl: The user-provided canned_acl, if applicable. content_type: content type for the component and final objects. dst_bucket_url: CloudUrl for the destination bucket. file_size: The size of fp, in bytes. fp: The file object to be partitioned. random_prefix: The randomly-generated prefix used to prevent collisions among the temporary component names. src_url: Source FileUrl from the original command. storage_class: storage class for the component and final objects. tracker_file: The path to the parallel composite upload tracker file. tracker_file_lock: The lock protecting access to the tracker file. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: dst_args: The destination URIs for the temporary component objects. r% �(parallel_composite_upload_component_sizer� r� )rr r r' r �_GetPartitionInfor r r r( �PARALLEL_UPLOAD_STATIC_SALTr� rY �updater � PARALLEL_UPLOAD_TEMP_NAMESPACEr� r r� r~ �append)r� r� �dst_bucket_url� file_sizer� � random_prefixr� r� r� r� r� r� r= �num_components�component_size�dst_argsZ file_names�i� encoded_name� content_md5r Ztemp_file_nameZ tmp_dst_urlZfile_part_length�offsetZ func_argsr� r� r� �_PartitionFile� sD %��� �� � rM c C s t | j| j�d�d d� �S )z�Gets component number from component CloudUrl. Used during parallel composite upload. Args: component: CloudUrl representing component. Returns: component number r� r� N)�intr� �rfind)r� r� r� r� �_GetComponentNumber7 s rP c ' C s t � � } t|j�} |j|jd�}tt�}|r2|jnd}t|t j ||�}t || �\}}}t |||||| || t t� \}}|dur�|�d�nd}|dur�|nt|d�}t||||d� t�� }i }t||j| || |||j|||| d� }t||| |�\}}}|D ]H}tj|jf||jj < t |j t|j |jt � � |jdt |j�tjd�� q�|D ]V}|d j!} tj"|d f|| < t |j t||d t � � d|d t |d �tj"d �� �qF|D ]&}t |j t||t � � dtj#d �� �q�|j$t%|| d t&j'j(|j)j*d d�} g } | D ]} | �+| d � �q�| dd� |D � } t,| �t,|�k�r�t-| t d�} g }!| D ]6}"t.j/j0|"j1d�}#|"�2� �rjt3|"j4�|#_4|!�+|#� �q@|j5|!|||jg d�|d�}$�z*z�| | }%|j$t |%tt&j'j(|j)j*d� | D ]p}|j!} z:t |j t||t � � d t |�|| d || d d�� W n( t |j t||t � � d d�� Y n0 �q�|D ]&}t |j t||t � � d tj#d �� �q8W n, t6�y� | �7dd�8|�9� � � Y n0 W |� t:|� W d � n1 �s�0 Y n0|� t:|� W d � n1 �s�0 Y 0 nt;d��t � � | }&|&|$fS ) a Uploads a local file to a cloud object using parallel composite upload. The file is partitioned into parts, and then the parts are uploaded in parallel, composed to form the original destination object, and deleted. Args: fp: The file object to be uploaded. src_url: FileUrl representing the local file. dst_url: CloudUrl representing the destination file. dst_obj_metadata: apitools Object describing the destination object. canned_acl: The canned acl to apply to the object, if any. file_size: The size of the source file in bytes. preconditions: Cloud API Preconditions for the final object. gsutil_api: gsutil Cloud API instance to use. command_obj: Command object (for calling Apply). copy_exception_handler: Copy exception handler (for use in Apply). logger: logging.Logger for outputting log messages. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, crc32c, and size fields populated. ��providerNr r� )r� r� F�r} �finished� component_num� message_typer r� )rT r} rU rV )rT rV )r� �total_bytes_transferredT�� arg_checker� parallel_operations_override�should_return_resultsr� c S s g | ] }|d �qS )r r� )r� rI r� r� r� r� � � z._DoParallelCompositeUpload.<locals>.<listcomp>��key)r� )rx r{ r} )r� rR �fields�encryption_tuple)rY rZ )rT rU r} rV )rT z:Failed to delete some of the following temporary objects: � zSSome temporary components were not uploaded successfully. Please retry this upload.)<�timer0 �bucket_url_string�GetApiSelectorr� rQ r �crypto_key_sha256r8 r= �PARALLEL_UPLOADr% r& � _DeleteTempComponentObjectFnr� r r$ r( r@ � CreateLockrM r� r� �FilterExistingComponentsr2 �COMPONENT_TO_UPLOADr� r� r� r_ r r� rP r� �EXISTING_COMPONENT�EXISTING_OBJECT_TO_DELETE�Applyr� �gslib�command�DummyArgChecker�ParallelOverrideReason�SLICErB r� �sortedr� �ComposeRequest�SourceObjectsValueListEntryr� � HasGeneration�longr{ � ComposeObject� Exceptionr, �join�keysr5 r )'r� r� r� r- r� rD r� r� � command_obj�copy_exception_handlerr� r� � start_timerC � api_selector�encryption_keywrapperr� �tracker_file_nameZexisting_enc_key_sha256Zexisting_prefix�existing_componentsrE r� Zcomponents_inforH �components_to_upload�existing_objects_to_deleter� Z component_str� cp_results�uploaded_components� cp_result� componentsZrequest_componentsZ component_urlr! Zcomposed_objectZobjects_to_delete� elapsed_timer� r� r� �_DoParallelCompositeUploadE sh # ������� � ������� �� � �� �� � �� �� �� ��� �� *�,� r� c C s� t t�ddt��}|o8|�� o8|�� o8|jdko8| }|r�|dkr�|tkr�t�< t �d�s�| � d� t � d��d � dt d< W d � n1 s�0 Y |o�|dko�||kS ) a� Determines whether parallel composite upload strategy should be used. Args: logger: for outputting log messages. allow_splitting: If false, then this function returns false. src_url: FileUrl corresponding to a local file. dst_url: CloudUrl corresponding to destination cloud object. file_size: The size of the source file, in bytes. gsutil_api: CloudApi that may be used to check if the destination bucket has any metadata attributes set that would discourage us from using parallel composite uploads. canned_acl: Canned ACL to apply to destination object, if any. Returns: True iff a parallel upload should be performed on the source file. r% �#parallel_composite_upload_threshold�gsr � suggestedra a� ==> NOTE: You are uploading one or more large file(s), which would run significantly faster if you enable parallel composite uploads. This feature can be enabled by editing the "parallel_composite_upload_threshold" value in your .boto configuration file. However, note that if you do this large files will be uploaded as `composite objects <https://cloud.google.com/storage/docs/composite-objects>`_,which means that any user who downloads such objects will need to have a compiled crcmod installed (see "gsutil help crcmod"). This is because without a compiled crcmod, computing checksums on composite objects is so slow that gsutil disables downloads of composite objects.TN)rr r r' r r� r� r� �'PARALLEL_COMPOSITE_SUGGESTION_THRESHOLD� suggested_sliced_transfers_lock�suggested_sliced_transfers�inforz �textwrap�wrap) r� r� r� r� rD r� r� r� Zall_factors_but_sizer� r� r� � _ShouldDoParallelCompositeUpload( s< ������ � �� �& �r� c C sR |p t �� }t| �r\tt| |||d��}t|�dkr@td| ��|d }t|j�|� � fS t| �}|� � rx||� � fS |� � r�|dfS t |�r�|dfS |j�d�}|j|j|d|jddgd �} | D ]�} | jtjjkr�| j|d kr�|df S | jtjjk�r | jj|jd k�r |df S | jtjjkr�| jj|jkr�|d f S q�||fS ) a- Expands wildcard if present in url_str. Args: url_str: String representation of requested url. gsutil_api: gsutil Cloud API instance to use. project_id: project ID to use (for iterators). treat_nonexistent_object_as_subdir: indicates if should treat a non-existent object as a subdir. logger: logging.Logger instance to use for output. If None, the root Logger will be used. Returns: (exp_url, have_existing_dst_container) where exp_url is a StorageUrl and have_existing_dst_container is a bool indicating whether exp_url names an existing directory, bucket, or bucket subdirectory. In the case where we match a subdirectory AND an object, the object is returned. Raises: CommandException: if url_str matched more than 1 URL. )� project_idr� r� z)Destination (%s) must match exactly 1 URLr T�/�prefixesz items/name)�prefix� delimiterrR r_ z _$folder$F)�logging�Loggerr- �listrv r� r r0 r� r� r� r� r� r/ r� r� � ListObjectsr� r� �datatyper �CsObjectOrPrefixType�PREFIX�data�OBJECTr� ) �url_strr� r� Z"treat_nonexistent_object_as_subdirr� Z blr_expansion�blrr, r� � list_iterator� obj_or_prefixr� r� r� �ExpandUrlToSingleBlrh sV �� � � � � �r� c C s� t �ddd�sdS | �� s dS t| j�r,dS |dkr8dS z|j| jdg| jd� W n4 ty� } z t |t j j �rr� W Y d}~n d}~0 0 dS )al Makes a request to the destination API provider to trigger reauth. Addresses https://github.com/GoogleCloudPlatform/gsutil/issues/1639. If an API call occurs in a child process, the library that handles reauth will fail. We need to make at least one API call in the main process to allow a user to reauthorize. For cloud source URLs this already happens because the plurality of the source name expansion iterator is checked in the main thread. For cloud destination URLs, only some situations result in a similar API call. In these situations, this function exits without performing an API call. In others, this function performs an API call to trigger reauth. Args: destination_url (StorageUrl): The destination of the transfer. gsutil_api (CloudApiDelegator): API to use for the GetBucket call. worker_count (int): If greater than 1, assume that parallel execution is used. Technically, reauth challenges can be answered in the main process, but they may be triggered multiple times if multithreading is used. Returns: None, but performs an API call if necessary. r% Z0trigger_reauth_challenge_for_parallel_operationsFNr� �location)r_ rR ) r �getboolr� r- r� � GetBucketr� r� ry r* �pyu2f�errors� PluginError)�destination_urlr� Z worker_countr� r� r� r� �.TriggerReauthForDestinationProviderIfNecessary� s& � � r� c C s6 | � � r2| jdkr2|�� r2t�dd|j�}t|�}|S )a Translates Windows pathnames to cloud pathnames. Rewrites the destination URL built by ConstructDstUrl(). Args: src_url: Source StorageUrl to be copied. dst_url: The destination StorageUrl built by ConstructDstUrl(). Returns: StorageUrl to use for copy. �\z\\r� )r� r� r� �re�subr� r0 )r� r� Z trans_url_strr� r� r� �FixWindowsNaming s r� c C sP | � � r4|� � r4tj�| j�}tj�|j�}||kS | j|jkoJ| j|jkS dS )z�Checks if src_url and dst_url represent the same object or file. We don't handle anything about hard or symbolic links. Args: src_url: Source StorageUrl. dst_url: Destination StorageUrl. Returns: Bool indicator. N)r� r� r8 �normpathr� r� r{ )r� r� Z new_src_pathZ new_dst_pathr� r� r� � SrcDstSame s �r� c C sn |� � r |r |jr d|j }nd}|�� rZ|�� s:|�� rZ|�� rFdnd}| �d||� n| �d|j|� dS )a* Logs copy operation, including Content-Type if appropriate. Args: logger: logger instance to use for output. src_url: Source StorageUrl. dst_url: Destination StorageUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. z [Content-Type=%s]r� z<STDIN>r� zCopying from %s%s...zCopying %s%s...N)r� r� r� r� r� r� r� )r� r� r� r- Zcontent_type_msgZsrc_textr� r� r� �_LogCopyOperation5 s � r� c C s� t |�}tt�}t�� } t|j| |dd�j} tjrnt tjd�� } t � | � � �j} W d � n1 sd0 Y |j ||| jtj|| |jt||d� } t�� } |�� }t|| j�|_t|jt| || tj|jdd�� | | |j|| jfS )a� Performs copy-in-the cloud from specified src to dest object. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata for source object; must include etag and size. dst_url: Destination CloudUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API instance to use for the copy. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. ZCopying�r� r� r r N)�src_generationr� r� �progress_callbackrR r_ �decryption_tupler` T�rV r} rT )rO rQ r rb r) r r r� r� r �pickle�loads�read� CopyObjectr{ r� r� �UPLOAD_RETURN_FIELDSr r. r_ r2 �FILE_CLOUD_COPYr} r| )r� r! r� r- r� r� �decryption_key�decryption_keywrapperr� r~ r� �test_fpZdst_obj�end_time� result_urlr� r� r� �_CopyObjToObjInTheCloudL sP �.� ��� �r� c C sT |j du �rP| �� �rP| �� �sP| �� �sP| j}d}|dk�r@tj�|�}t� ddd��rt �sztt j ddd|gt j t j d �}|�� \}}|j�� |j�� |jd ks�|r�td ||j|f ��|�� }t�|�}W n6 t�y } z td ||f ��W Y d}~n d}~0 0 n2|�d �\} } } | tv �r2t| }nt�|�\}} |�sJt }||_ dS )z�Detects and sets Content-Type if src_url names a local file. Args: src_url: Source StorageUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. N�-r% � use_magicfileF�filez-bz--mime)�stdout�stderrr zAEncountered error running "file -b --mime %s" (returncode=%d). %sz2Encountered OSError running "file -b --mime %s" %s�.) r� r� r� r� r� r� r8 �realpathr r� rj � subprocess�Popen�PIPE� communicater� �closer� � returncoder r� r � ensure_strr� r� �COMMON_EXTENSION_RULES� mimetypes� guess_typerm ) r� r- r� r� Zreal_file_path�p�outputr� r� r� � extensionr� r� r� �_SetContentTypeFromFile� sN �� � ���� r� c C s� t |j| |dd�j}tjrVttjd�� } t�| �� �j}W d � n1 sL0 Y t � � } t t �} | � � sv| � � r�|j||tj||| |jt|d� } n"|j||tj|||| |jt|d� } t � � } | | }|| fS )a� Uploads the file using a non-resumable strategy. This function does not support component transfers. Args: src_url: Source StorageUrl to upload. src_obj_filestream: File pointer to uploadable bytes. src_obj_size (int or None): Size of the source object. dst_url: Destination StorageUrl for the upload. dst_obj_metadata: Metadata for the target object. preconditions: Preconditions for the upload, if any. gsutil_api: gsutil Cloud API instance to use for the upload. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, md5, and size fields populated. � Uploadingr� r N)�object_metadatar� r� r� r` rR r_ r� ) r� r� r} r� r� r` rR r_ r� )r) r r r� r� r r� r� r� rb rQ r r� r� �UploadObjectStreamingr� r� r� � UploadObject)r� �src_obj_filestream� src_obj_sizer� r- r� r� r� r� r� r~ r� �uploaded_objectr� r� r� r� r� � _UploadFileToObjectNonResumable� sL �.� � r� c s� t |tj|j|jd���tt�} | r6| jr6| j�d�nd� � �fdd�} t �|� d�} | rh|� d| j � d} |rxt |�nd}t |j| ||d d �j}tjr�ttjd �� }t�|�� �j}W d � n1 s�0 Y t�� }d }| �r��z�z0|j||tj||j|| | t| || d � }d} W �n~ t�y� } �zJ|� d| j � |d7 }|t� k } | �sX� zpz&|� d|j � |j |j |jd� W n( t �y� Y n t �y� � Y n0 W t �� |� d�| j f � n t �� |� d�| j f � 0 |� d| j ||f � d} |�!d � |�rt |�nd}t |j| ||d d �j}t"|jt#|t�� |d�� t�$t%t&�&� d| t'� �� W Y d}~n"d}~0 t(�y� d} � Y n0 W | �s�t �� q�| �s�t �� 0 q�t�� }|| }||fS )a� Uploads the file using a resumable strategy. Args: src_url: Source FileUrl to upload. Must not be a stream. src_obj_filestream: File pointer to uploadable bytes. src_obj_size (int or None): Size of the source object. dst_url: Destination StorageUrl for the upload. dst_obj_metadata: Metadata for the target object. preconditions: Preconditions for the upload, if any. gsutil_api: gsutil Cloud API instance to use for the upload. logger: for outputting log messages. is_component: indicates whether this is a single component or whole file. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, md5, and size fields populated. rQ r Nc s t � tt| �i}t�|� dS )a Creates a new tracker file for starting an upload from scratch. This function is called by the gsutil Cloud API implementation and the the serialization data is implementation-specific. Args: serialization_data: Serialization data used in resuming the upload. N)r6 r< r� r? )�serialization_datar� �r� r� r� r� �_UploadTrackerCallback/ s �z<_UploadFileToObjectResumable.<locals>._UploadTrackerCallbackr� zResuming upload for %sTr� )r� rU r� r r r ) r� r� r� rR r} r� r` r_ �tracker_callbackr� r� Fz:Caught ResumableUploadStartOverException for upload of %s.r� z8Checking that bucket %s exists before retrying upload...zCDeleted tracker file %s for resumable upload of %s before retrying.z�Restarting upload of %s from scratch (retry #%d) after exception indicating we need to start over with a new resumable upload ID: %s)� num_retriesr� ))r8 r= �UPLOADrd r� rQ r re r r9 r� r� rP r) r r r� r� r r� r� r� rb �UploadObjectResumabler� r� r rF r� r� r r r5 �seekr_ r3 �sleep�min�randomrE r )r� r� r� r� r- r� r� r� r� r� r� r� � tracker_data� retryablerU r� r� r~ Znum_startover_attemptsr� r� r� r� r� r� r� � _UploadFileToObjectResumable s� ������.� � � ��� �� ����� � r� c C sV d}d}| � d�}|r |r d}n.|tksD|rNt|�dkrN|d |v rN| }|}||fS )a� Selects how an upload should be compressed. This is a helper function for _UploadFileToObject. Args: object_name: The object name of the source FileUrl. is_component: indicates whether this is a single component or whole file. gzip_exts: List of file extensions to gzip prior to upload, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts for selecting which files will be encoded. Streaming files compressed is only supported on the JSON GCS API. Returns: A tuple: (If the file should be gzipped locally, if the file should be gzip transport encoded). Fr� Tr� r� )�splitr� r� )r� r� r� r� � zipped_file�gzip_encoded_fileZ fname_partsr� r� r� � _SelectUploadCompressionStrategy� s � � �r� c C s |dur |t kr |�d| � t�� \}}d}z�| �� s>| �� rFtd��|durnt|�dt|� k rntd| ��t � ddt �}t j |d|d �}|�t�}|r�|�|� |�t�}q�W |r�|�� t�|� |�� n |r�|�� t�|� |�� 0 tj�|�} t |d �} t|�| | fS ) a� Compresses a to-be-uploaded local file to save bandwidth. This is a helper function for _UploadFileToObject. Args: src_url: Source FileUrl. src_obj_filestream: Read stream of the source file - will be consumed and closed. src_obj_size (int or None): Size of the source file. logger: for outputting log messages. Returns: StorageUrl path to compressed file, read stream of the compressed file, compressed file size. NzCompressing %s (to tmp)...z�gzip compression is not currently supported on streaming uploads. Remove the compression flag or save the streamed output temporarily to a file before uploading.r� z|Inadequate temp space available to compress %s. See the CHANGING TEMP DIRECTORIES section of "gsutil help cp" for more info.r% Zgzip_compression_level�wb)� compresslevelr )rL r� �tempfile�mkstempr� r� r rg rN r �getintr �gzipr r� �GZIP_CHUNK_SIZE�writer� r� r8 �getsizer0 ) r� r� r� r� Zgzip_fhZ gzip_path�gzip_fpZcompression_levelr� Z gzip_sizeZcompressed_filestreamr� r� r� � _ApplyZippedUploadCompression� sD � ��� � r c C s� d}d}z�|rF|sFt jj� | � \}}W d � qP1 s:0 Y n | � \}}W |�� |r�zt�|j� W q� ty� |�d|j� Y q�0 n@|�� |r�zt�|j� W n ty� |�d|j� Y n0 0 ||fS )a� Handles setup and tear down logic for uploads. This is a helper function for _UploadFileToObject. Args: upload_delegate: Function that handles uploading the file. upload_url: StorageURL path to the file. upload_stream: Read stream of the file being uploaded. This will be closed after the upload. zipped_file: Flag for if the file is locally compressed prior to calling this function. If true, the local temporary file is deleted after the upload. gzip_encoded_file: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. parallel_composite_upload: Set to true if this upload represents a top-level parallel composite upload (not an upload of a component). If true, resource locking is skipped. logger: For outputting log messages. Returns: The elapsed upload time, the uploaded object. NzZCould not delete %s. This can occur in Windows because the temporary file is still locked.) rn ro �!concurrent_compressed_upload_lockr� r� �unlinkr� ry �warning) Zupload_delegate� upload_url� upload_streamr� r� �parallel_composite_uploadr� r� r� r� r� r� �_DelegateUploadFileToObject s4 * � � � r Tc s �r �j s t�dd�}|r |�_ | � |� |� t| j�| | �\}��rV�sV��d| � nH|r�t| ||��\� � � d�_�js�d�_n d�j� � vr�� jd7 _�s�t �j t � �t � � t j� dd�� d }d }t� �t�fd d ��p�i D ��}t�| � �|�tjd �}� d u �rd n� t� k �p.| �� �p.| �� }| �� �sD| �� �rj�j�jd�tjk�rj� }t|t � �� |�s�t ���r�t � |�� ��� n� � � ������� � � � f dd�}����� � � � fdd�}������� � � � f dd�}|�r�|}n|�r |}n|}t |� � |�|��\}}|�s�z"t |�}t!��|| j|dd� W nj t"�y� t#�r�t$j%�j&�j'|j(d�}�jt) �_&�j*|��jd� �j+�j,�j|j-�jd� � Y n0 ��.� } |j-| _-t/| |j-�| _-��s�t �j t � �t � � t j� dd�� ||j0| |j1fS )a| Uploads a local file to an object. Args: src_url: Source FileUrl. src_obj_filestream: Read stream of the source file to be read and closed. src_obj_size (int or None): Size of the source file. dst_url: Destination CloudUrl. dst_obj_metadata: Metadata to be applied to the destination object. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API to use for the copy. logger: for outputting log messages. command_obj: command object for use in Apply in parallel composite uploads. copy_exception_handler: For handling copy exceptions during Apply. gzip_exts: List of file extensions to gzip prior to upload, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. allow_splitting: Whether to allow the file to be split into component pieces for an parallel composite upload. is_component: indicates whether this is a single component or whole file. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts for selecting which files will be encoded. Streaming files compressed is only supported on the JSON GCS API. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. r% �content_languagez+Using compressed transport encoding for %s.r� z no-transformz ,no-transformFr� Nc 3 s | ]}|� | � fV qd S �Nr� �r� r �� hash_algsr� r� � <genexpr>� r\ z&_UploadFileToObject.<locals>.<genexpr>)r� r rQ c s" t � � ��tj���� ���d� S �N)r� )r� r� r� r� ) r| r} r- r� r� r� r� r� � upload_sizer r r� r� �CallParallelCompositeUpload� s �z8_UploadFileToObject.<locals>.CallParallelCompositeUploadc s t ����� ���d�S r )r� r� )r- r� r� r� r� r r �wrapped_filestreamr� r� �CallNonResumableUpload� s �z3_UploadFileToObject.<locals>.CallNonResumableUploadc s t �� ��� �����d� S )N)r� r� )r� r� ) r- r� r� r� r� r� r� r r r r� r� �CallResumableUpload� s �z0_UploadFileToObject.<locals>.CallResumableUploadT)r2 �r� r� rz �r{ rR )2�contentLanguager � get_valuer� r� r� r �contentEncoding� cacheControl�lowerr_ r r2 rb � FILE_UPLOADrZ �dictr� r� r� rG r� r� rd r� r �JSONr+ rD r� r[ r r r5 r! �_RENAME_ON_HASH_MISMATCHr� r� r� r� rz � _RENAME_ON_HASH_MISMATCH_SUFFIXr� � DeleteObjectr� r{ r r. r} r| ) r� r� r� r� r- r� r� r� r| r} r� r� r� r� r r� r� r� r r Znon_resumable_uploadZ orig_streamr r r Zdelegater �corrupted_obj_metadatar� r� )r| r} r- r� r� r� r r� r� r� r r r r r� r� E s� , �� ��������� �� ������� �r� c C s� t j�| j�}|r`t j�|�s`zt �|� W n2 ty^ } z|jtjkrJ� W Y d}~n d}~0 0 d}t |�r�d}t � | �}|� d|� n t � | �}t j�| j�r�t �| j� t j�|�s�t|d�}|�� ||fS )a� Creates a new download file, and deletes the file that will be replaced. Names and creates a temporary file for this download. Also, if there is an existing file at the path where this file will be placed after the download is completed, that file will be deleted. Args: dst_url: Destination FileUrl. src_obj_metadata: Metadata from the source object. logger: for outputting log messages. Returns: (download_file_name, need_to_unzip) download_file_name: The name of the temporary file to which the object will be downloaded. need_to_unzip: If true, a temporary zip file was used and must be uncompressed as part of validation. NFTz$Downloading to temp gzip filename %s�w)r� r8 r9 r� �exists�makedirsr� �errno�EEXISTr\ rB ZGetTempZipFileNamer� �GetTempFileNamer r r� )r� r! r� �dir_namer� � need_to_unzip�download_file_namer� r� r� r� �_GetDownloadFile s& r, c C s� t t�ddt��}t�ddt�}t�ddt�}|jo8t� }|pD|t k}|or| t j j uor|dkor|or|dkor|j |k} | s�|j tkr�t� s�|t kr�t�< t�d�s�|�d�t�d ��d � d td< W d � n1 s�0 Y | S ) a| Determines whether the sliced download strategy should be used. Args: download_strategy: CloudApi download strategy. src_obj_metadata: Metadata from the source object. allow_splitting: If false, then this function returns false. logger: logging.Logger for log message output. Returns: True iff a sliced download should be performed on the source file. r% � sliced_object_download_threshold�%sliced_object_download_max_componentsr& r� r r� ra z�==> NOTE: You are downloading one or more large file(s), which would run significantly faster if you enabled sliced object downloads. This feature is enabled by default but requires that compiled crcmod be installed (see "gsutil help crcmod").TN)rr r r' r r� r rU rx rH rV r r� r� r} r� r� r� r� rz r� r� ) �download_strategyr! r� r� r- �max_componentsr. Zparallel_hashingZ hashing_okayZ use_slicer� r� r� �_ShouldDoSlicedDownloadR sV ���� �������� ���&r1 c s� t | |d�}t�tj|j�}t| j|jd�� t � fdd�� p>i D ��}t |j ||j |j || j||j|j|j|jd� \}}d}d|v r�|d j}t|j|||j|j d |�S ) a" Function argument to Apply for performing sliced downloads. Args: cls: Calling Command class. args: PerformSlicedDownloadObjectToFileArgs tuple describing the target. thread_state: gsutil Cloud API instance to use for the operation. Returns: PerformSlicedDownloadReturnValues named-tuple filled with: component_num: The component number for this download. crc32c: CRC32C hash value (integer) of the downloaded bytes. bytes_transferred: The number of bytes transferred, potentially less than the component size if the download was resumed. component_total_size: The number of bytes corresponding to the whole component size, potentially more than bytes_transferred if the download was resumed. r� )�consider_crc32cc 3 s | ]}|� | � fV qd S r r� r r r� r� r � r\ z5_PerformSlicedDownloadObjectToFile.<locals>.<genexpr>)rU � start_byte�end_byter� Nrx r� )rI r �decode_messager� r� �src_obj_metadata_jsonrX r� rx r � _DownloadObjectToFileResumabler� r� r+ rU r3 r4 r� �crcValuer� ) r� r� r� r� r! r �bytes_transferred�server_encodingZ crc32c_valr� r r� �"_PerformSlicedDownloadObjectToFile� s: ��� �r; c C s� | j s J �d}| jt� k r dS t|tj|�}d}�zz�t|d�}t|�} | | jkr�t|d�}t� |�} | d | j kr�| d | j kr�| d |kr�W W |r�|� � |r�|� � dS |� � |� d|j � W nX ttf�y } z:t| t�s�| jtjk�r |� d|t| �f � W Y d} ~ n d} ~ 0 0 W |�r2|� � |�r`|� � n |�rP|� � |�r^|� � 0 t||� zPt|d ��0}| j | j |d �} |�t�| �� W d � n1 �s�0 Y W n2 t�y� } zt|| j� W Y d} ~ n d} ~ 0 0 dS ) a� Maintains sliced download tracker files in order to permit resumability. Reads or creates a sliced download tracker file representing this object download. Upon an attempt at cross-process resumption, the contents of the sliced download tracker file are verified to make sure a resumption is possible and appropriate. In the case that a resumption should not be attempted, existing component tracker files are deleted (to prevent child processes from attempting resumption), and a new sliced download tracker file is created. Args: src_obj_metadata: Metadata from the source object. Must include etag and generation. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for the download. logger: for outputting log messages. api_selector: The Cloud API implementation used. num_components: The number of components to perform this download with. Nr �rrz r{ rF z`Sliced download tracker file doesn't match for download of %s. Restarting download from scratch.zVCouldn't read sliced download tracker file (%s): %s. Restarting download from scratch.r# )rz r{ rF )rz r} rG r8 r= �SLICED_DOWNLOADr rh �json�loadr{ r� r, r� �IOError� ValueErrorr* r& �ENOENTr� r4 r� �dumpsr: �strerror) r! r� r+ r� r rF r� r� r� �existing_file_sizeZtracker_file_datar� r� r� r� �#_MaintainSlicedDownloadTrackerFiles� sj � � ��� � � �4rF c @ sR e Zd ZdZdd� Zedd� �Zdd� Zej fdd �Z d d � Z d d � Z dd� Z dS )�SlicedDownloadFileWrappera� Wraps a file object to be used in GetObjectMedia for sliced downloads. In order to allow resumability, the file object used by each thread in a sliced object download should be wrapped using SlicedDownloadFileWrapper. Passing a SlicedDownloadFileWrapper object to GetObjectMedia will allow the download component tracker file for this component to be updated periodically, while the downloaded bytes are normally written to file. c C s( || _ || _|| _d| _|| _|| _dS )a; Initializes the SlicedDownloadFileWrapper. Args: fp: The already-open file object to be used for writing in GetObjectMedia. Data will be written to file starting at the current seek position. tracker_file_name: The name of the tracker file for this component. src_obj_metadata: Metadata from the source object. Must include etag and generation. start_byte: The first byte to be downloaded for this parallel component. end_byte: The last byte to be downloaded for this parallel component. N)�_orig_fp�_tracker_file_name�_src_obj_metadata�_last_tracker_file_byte� _start_byte� _end_byte)�selfr� r� r! r3 r4 r� r� r� �__init__ s z"SlicedDownloadFileWrapper.__init__c C s t | jdd�S )z<Returns the mode of the underlying file descriptor, or None.r� N)�getattrrH �rN r� r� r� r� ' s z SlicedDownloadFileWrapper.modec C s� | j �� }| j|kr*|t|� | jd ks.J �t�| j |� | j �� }t}| jd u sp|| j |ksp|| jd kr�t | j | j |� || _d S �Nr� ) rH �tellrL r� rM rC � write_to_fd� TRACKERFILE_UPDATE_THRESHOLDrK r> rI rJ )rN r� �current_file_pos� thresholdr� r� r� r� , s" � � ��z SlicedDownloadFileWrapper.writec C s\ |t jkr"| j�|| j d � n| j�||� | j| j�� krR| jd ksXn J �d S rR )r� �SEEK_ENDrH r� rM rL rS )rN rL �whencer� r� r� r� = s z SlicedDownloadFileWrapper.seekc C s | j �� S r )rH rS rQ r� r� r� rS D s z SlicedDownloadFileWrapper.tellc C s | j �� d S r )rH �flushrQ r� r� r� rZ G s z SlicedDownloadFileWrapper.flushc C s | j r| j �� d S r )rH r� rQ r� r� r� r� J s z SlicedDownloadFileWrapper.closeN)r� r� r� r� rO �propertyr� r� r� �SEEK_SETr� rS rZ r� r� r� r� r� rG s rG c C s� t t�ddt��}t�ddt�}t|j||�\}}g } g } t|�D ]`} | | } t | d | d |jd �} | � | | d � t � |�}| � t | | |||| | |�� qB| | fS )a� Partitions an object into components to be downloaded. Each component is a byte range of the object. The byte ranges of the returned components are mutually exclusive and collectively exhaustive. The byte ranges are inclusive at both end points. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object with non-pickleable fields removed. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for the download. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: components_to_download: A list of PerformSlicedDownloadObjectToFileArgs to be used in Apply for the sliced download. r% �%sliced_object_download_component_sizer. r� )rr r r' r r� r r> r} r r� rB r �encode_messager )r� r! r� r+ r� Z sliced_download_component_sizer0 rF rG �components_to_download�component_lengthsrI r3 r4 r6 r� r� r� �_PartitionObjectO s6 ���� ��ra c C s� d|_ t| ||||�\} } t| �} t|||||| � t|d�� } | �|j� W d � n1 s`0 Y t| �D ]Z\}}|j|j d }t ||||j ||�}||j }t | t | |t � � |d|t j|d�� qr|jt| |tjj|jjdd�}t|�| k �rtd|j ��t|td �d �}|d j}|du�rTtd| �D ] }t ||| j| | �}�q6d }t |�}|D ]h}t | t | |t � � |j d|j t jd �� ||j 7 }|j!�o�|j!�"� �#d �}|�rd|�sdtd|j ���qd||fS )a� Downloads a cloud object to a local file using sliced download. Byte ranges are decided for each thread/process, and then the parts are downloaded in parallel. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. command_obj: command object for use in Apply in parallel composite uploads. logger: for outputting log messages. copy_exception_handler: For handling copy exceptions during Apply. api_selector: The Cloud API implementation used. decryption_key: Base64-encoded decryption key for the source object, if any. status_queue: Queue for posting file messages for UI/Analytics. Returns: (bytes_transferred, crc32c) bytes_transferred: Number of bytes transferred from server this call. crc32c: a crc32c hash value (integer) for the downloaded bytes, or None if crc32c hashing wasn't performed. N�r+br� F)r} rT rU rV �bytes_already_downloadedTrX zSSome components of %s were not downloaded successfully. Please retry this download.rU r] r rS r� zTDownload of %s failed because the server sent back data with an unexpected encoding.)$ry ra r� rF r �truncater} � enumerater4 r3 r7 r_ r2 rb �COMPONENT_TO_DOWNLOADrm r; rn ro rp rq rr r r� rs r rx r rW r\ Zcomponent_total_sizerU r9 r: r r� )r� r! r� r+ r| r� r} r r� r r_ r` rF r� rI r� r} �download_start_byterc r� rx r9 Z expect_gzipr� � server_gzipr� r� r� �_DoSlicedDownload� s� $ �� *� �� ��� �� � ��ri c " C s� | du r|j d } | | d } |du} |j| jd�} d}|j}| rN|d| 7 }d}�z��z6t|d�}|�|� |j| jd�} t|�}t|||| |||�\}}||k s�|| d kr�t|� t d| ��||| k}||ko�| }|r�|� d|� n|�r|� d|� t |||j d �}|�s |�r�d }|| }t |t|j|| |d d �j�}||k �r�tt|| �}|�|�}||7 }|D ]} || �t�|�� �qt|�t|�� �qFn| �s�|�d � d }t|j|| | ||d d�j} tj�rttjd�� } t � | �� �j} W d � n1 �s0 Y | �r<|j t � k�r<t ||||| �}t |�} |�s�|�|� |j!| j"| j||| | | j#|j t$j%j&| j||| t'| �d�}W n8 t(�y� } z |�)d| j*|� � W Y d} ~ n d} ~ 0 0 W |�r�|�+� n|�r�|�+� 0 | | d }!|!|fS )a] Downloads an object to a local file using the resumable strategy. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. gsutil_api: gsutil Cloud API instance to use for the download. logger: for outputting log messages. digesters: Digesters corresponding to the hash algorithms that will be used for validation. component_num: Which component of a sliced download this call is for, or None if this is not a sliced download. start_byte: The first byte of a byte range for a sliced download. end_byte: The last byte of a byte range for a sliced download. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (bytes_transferred, server_encoding) bytes_transferred: Number of bytes transferred from server this call. server_encoding: Content-encoding string if it was detected that the server sent encoded bytes during transfer, None otherwise. Nr� rQ z component %drb z�Resumable download start point for %s is not in the correct byte range. Deleting tracker file, so if you re-try this download it will start from scratchzResuming download for %szRDownload already complete for %s, skipping download but will run integrity checks.)�progress� user_projectr r )rU r� r� r � Downloading)r3 �override_total_sizer� r� rU r r ) r3 r4 �compressed_encodingr{ � object_sizer/ rR r� r r� r� z:Caught ResumableDownloadException (%s) for download of %s.),r} rd r� r� r r� rh r; r5 r r� rJ rk r* r) r r r� rK r� r@ r r( �Progressr� rd r� r� r� r� rG rG r\ �GetObjectMediar� r{ r r� r� rO r r �reasonr� )"r� r! r� r+ r� r� r rU r3 r4 r� Z download_sizeZ is_slicedr r: � download_namer� rE r� rg �download_completeZresumingr� Zbytes_digestedZtotal_bytes_to_digestZ hash_callback� bytes_to_readr� r# r� r� rn r� r9 r� r� r� r7 � s� " � �� �� �� � � 0� � � � r7 c C s� d}z�t |d�}t|d|jd�}t|j| |dd�j} tjrvt tjd�� } t� | � � �j} W d � n1 sl0 Y |j | j | j || j|jtjj| j||| t|�d� } W |r�|�� n|r�|�� 0 |j| fS ) a^ Downloads an object to a local file using the non-resumable strategy. This function does not support component transfers. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. gsutil_api: gsutil Cloud API instance to use for the download. digesters: Digesters corresponding to the hash algorithms that will be used for validation. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (bytes_transferred, server_encoding) bytes_transferred: Number of bytes transferred from server this call. server_encoding: Content-encoding string if it was detected that the server sent encoded bytes during transfer, None otherwise. Nr# r )rk rl r� r )r{ ro r/ rR r� r r� r� )r rJ rk r) r r r� r� r� r� r� rq r� r� r{ r} r r� r� r� rO r� ) r� r! r� r+ r� r r� r� r� r� r� r: r� r� r� �!_DownloadObjectToFileNonResumable� sB ��.� � rv c sD |j �|j�r8|�d�t�d|j ��� td|j ��|j| j d�} t |�} t | |||�}t |||�\}}t �( t�|d�r�t�dt|< W d � n1 s�0 Y |jo�| }t|||jd�� t� fd d �� p�i D ��}d}|jd k}d }t�� }|�s�|�r<t| ||||||| ||jd � \}}d |v �r�||d _nf| tjj u �rft | ||||||d�\}}n<| tjj u �r�t | |||||||d�\}}nt d| |f ��t�� }|�o�|�!� �d�}t"|| |||||� || ||| | | d�}t � t�#|� W d � n1 �s 0 Y t$|jt%| ||t%j&|jdd�� || |||fS )a� Downloads an object to a local file. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. gsutil_api: gsutil Cloud API instance to use for the download. logger: for outputting log messages. command_obj: command object for use in Apply in sliced downloads. copy_exception_handler: For handling copy exceptions during Apply. allow_splitting: Whether or not to allow sliced download. decryption_key: Base64-encoded decryption key for the source object, if any. is_rsync: Whether or not the caller is the rsync command. preserve_posix: Whether or not to preserve POSIX attributes. use_stet: Decrypt downloaded file with STET binary if available on system. Returns: (elapsed_time, bytes_transferred, dst_url, md5), where time elapsed excludes initial GET. Raises: FileConcurrencySkipError: if this download is already in progress. CommandException: if other errors encountered. ra z�Skipping attempt to download to filename ending with slash (%s). This typically happens when using gsutil to download from a subdirectory created by the Cloud Console (https://cloud.google.com/console)z Invalid destination path: %srQ FTN)� consider_md5r2 c 3 s | ]}|� | � fV qd S r r� r r r� r� r # r\ z(_DownloadObjectToFile.<locals>.<genexpr>r )r� r rx �r� z.Invalid download strategy %s chosen forfile %sr� )�is_rsyncr �use_stet)� message_timerV r} rT )'r� r� r� r, rz r� r� r" rd r� r� r1 r, �open_files_lock�open_files_mapr' r� r| rX rx r r} rb ri r r8 r r� r� rv r� r7 r r � _ValidateAndCompleteDownload�deleter_ r2 � FILE_DOWNLOAD)r� r! r� r� r� r| r} r� r� ry r rz r r/ �sliced_downloadr+ r* rw r r: rt r9 r~ rx r� rh Z local_md5r� r r� �_DownloadObjectToFile� s� %����� & � �� � � ��*�� r� c C s� |j }d}|D ]}|| sd} q$q|r2t|�}nt| j||||�}d}d}z t| ||||� t|| � W n� ty� } zj|r�| �d� d}nJ| tj kr�| �d� |}d}n,t|| � t r�t � ||t � n t �|� � W Y d}~n d}~0 0 |s�|s�|}n�t�|�}| tk�r | �d|� d}z�zdt�|d�}t|d��8}|�t�}|�rd|�|� |�t�}�qFW d � n1 �sz0 Y W n> t�y� } z$d t|�v �r�|�r�|�W Y d}~n d}~0 0 W |�r�|�� n|�r�|�� 0 t �|� |�slz0t| j||||�}t| ||||� t|| � W nB t�yj t|| � t �rZt � ||t � n t �|� � Y n0 |�r�t� |||| � t � ||� t ||| | d � d |v �r�|d S dS ) a� Validates and performs necessary operations on a downloaded file. Validates the integrity of the downloaded file using hash_algs. If the file was compressed (temporarily), the file will be decompressed. Then, if the integrity of the file was successfully validated, the file will be moved from its temporary download location to its permanent location on disk. Args: logger: For outputting log messages. src_url: StorageUrl for the source object. src_obj_metadata: Metadata for the source object, potentially containing hash values. dst_url: StorageUrl describing the destination file. need_to_unzip: If true, a temporary zip file was used and must be uncompressed as part of validation. server_gzip: If true, the server gzipped the bytes (regardless of whether the object metadata claimed it was gzipped). digesters: dict of {string, hash digester} that contains up-to-date digests computed during the download. If a digester for a particular algorithm is None, an up-to-date digest is not available and the hash must be recomputed from the local file. hash_algs: dict of {string, hash algorithm} that can be used if digesters don't have up-to-date digests. temporary_file_name: Temporary file name that was used for download. api_selector: The Cloud API implementation used (used tracker file naming). bytes_transferred: Number of bytes downloaded (used for logging). gsutil_api: Cloud API to use for service and status. is_rsync: Whether or not the caller is the rsync function. Used to determine if timeCreated should be used. preserve_posix: Whether or not to preserve the posix attributes. use_stet: If True, attempt to decrypt downloaded files with the STET binary if it's present on the system. Returns: An MD5 of the local file, if one was calculated as part of the integrity check. TFNzDHash did not match but server gzipped the content, will recalculate.zMHash did not match but server may have gzipped the content, will recalculate.z/Uncompressing temporarily gzipped file to %s...r r� zNot a gzipped file�ry r r ) r� r r$ r r5 r4 r! r� r r� r r� �renamer r rB r( ru r� r� r r� r� r� r@ r� r� rA Zdecrypt_downloadre )r� r� r! r� r* rh r r Ztemporary_file_namer r9 r� ry r rz Zfinal_file_nameZdigesters_succeededr r3 Zdigest_verifiedZhash_invalid_exceptionr� Z unzipped_temporary_file_namer Zf_outr� r� r� r� r~ u s� 4 � � � � � 2 � � � � � � r~ c C s� t | �}tj�|j�}|r\zt�|� W n2 tyZ } z|jtjkrF� W Y d}~n d}~0 0 t |jd��$}t � � }t � ||� W d � n1 s�0 Y | � � s�|�� t � � } t|t| || tj|r�|jnddd�� | | tj�|j�|dfS )a Copies a local file to a local file. Args: src_url: Source FileUrl. dst_url: Destination FileUrl. status_queue: Queue for posting file messages for UI/Analytics. src_obj_metadata: An apitools Object that may contain file size, or None. Returns: (elapsed_time, bytes_transferred, dst_url, md5=None). Raises: CommandException: if errors encountered. Nr� Tr� )ri r� r8 r9 r� r% r� r& r' r rb �shutil� copyfileobjr� r� r_ r2 �FILE_LOCAL_COPYr} r� ) r� r� r r! Zsrc_fpr) r� Zdst_fpr~ r� r� r� r� �_CopyFileToFile s6 * �� �r� c C s d S r r� )r� r� r� r� �_DummyTrackerCallbackK s r� c C s� t jr| j|jkrtd��t js&g |_d}t jrltt jd�� } t�| � � �j }W d � n1 sb0 Y t |�} t t �} t�� } t| |j|| ||d�} d}|jdkr�|j| |t j||jt|j| d�}n4|j| |t j||jt|jt|j| |dd�j t| d � }t�� }zt|| |||� W nv t�y� t�rptj |j |j |j d �}|j t! |_ t"|�}|j#|||j|| d � |j$|j%|j |j&|jd � � Y n0 |�'� }t(||j&�|_&t)|jt*| ||t*j+|jd d�� || |j||j,fS )a Copies from src_url to dst_url in "daisy chain" mode. See -D OPTION documentation about what daisy chain mode is. Args: src_url: Source CloudUrl src_obj_metadata: Metadata from source object dst_url: Destination CloudUrl dst_obj_metadata: Object-specific metadata that should be overidden during the copy. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API to use for the copy. logger: For outputting log messages. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. z"Cross-provider cp -p not supportedNr )rn r� r� r )r� r� r� rR r_ r} r` r� r� ) r� r� r� rR r_ r} r� r� r` r )rR r� r` r Tr� )-r� r� r� �NotImplementedError�aclr� r r� r� r� r r\ rQ r rb r r} r� r� r� r� r) r r� r0 r! r r� r� r� r� rz r� r rO r� r! r� r{ r r. r_ r2 �FILE_DAISY_COPYr| )r� r! r� r- r� r� r� r� r� r� rn r� r~ Z upload_fpr� r� r" r� r� r� r� r� �_CopyObjToObjDaisyChainModeP s� �.� ��� �������� �r� c C s� t � }| r:|�g d�� |r(|�dg� |s�|�dg� nT|�g d�� |r^|�dt dg� |r�dt dt dt dt dt g}|�|� |r�|�ddg� |r�|�dg� t|�S )a Determines the metadata fields needed for a copy operation. This function returns the fields we will need to successfully copy any cloud objects that might be iterated. By determining this prior to iteration, the cp command can request this metadata directly from the iterator's get/list calls, avoiding the need for a separate get metadata HTTP call for each iterated result. As a trade-off, filtering objects at the leaf nodes of the iteration (based on a remaining wildcard) is more expensive. This is because more metadata will be requested when object name is all that is required for filtering. The rsync command favors fast listing and comparison, and makes the opposite trade-off, optimizing for the low-delta case by making per-object get metadata HTTP call so that listing can return minimal metadata. It uses this function to determine what is needed for get metadata HTTP calls. Args: dst_is_cloud: if true, destination is a Cloud URL. skip_unsupported_objects: if true, get metadata for skipping unsupported object types. preserve_acl: if true, get object ACL. is_rsync: if true, the calling function is rsync. Determines if metadata is needed to verify download. preserve_posix: if true, retrieves POSIX attributes into user metadata. delete_source: if true, source object will be deleted after the copy (mv command). file_size_will_change: if true, do not try to record file size. Returns: List of necessary field metadata field names. )r �componentCount�contentDispositionr r r� rx ry rz r{ r| � mediaLink�metadata�metagenerationr� � timeCreatedr� r} ) rx r r� ry rz r� r| r} r{ z metadata/%sr� r� )�setr@ rd r` rb rc rf r� ) Z dst_is_cloudr� r� ry r � delete_source�file_size_will_changeZsrc_obj_fields_setZ posix_fieldsr� r� r� � GetSourceFieldsNeededForCopy� s4 ' � � r� � �Z im )ZnearlineZcoldline�archivec C sh | j dkrd|rd|jrd|jrd|j�� }t�|d�}|rd|t|j� }t�� |k rd|�d|| j |t � dS )a� Warns when deleting a gs:// object could incur an early deletion charge. This function inspects metadata for Google Cloud Storage objects that are subject to early deletion charges (such as Nearline), and warns when performing operations like mv that would delete them. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with necessary fields (per GetSourceFieldsNeededForCopy). logger: logging.Logger for outputting warning. r� Nz�Warning: moving %s object %s may incur an early deletion charge, because the original object is less than %s days old according to the local system time.) r� r� r� r � EARLY_DELETION_MINIMUM_LIFETIMEr' ra rb r, r� rt )r� r! r� Zobject_storage_classZ early_deletion_cutoff_secondsZminimum_delete_ager� r� r� �"WarnIfMvEarlyDeletionChargeAppliesI s( �� ��� �r� c C s$ | j dkr tjr |jdkr t� �dS )a Skips unsupported object types if requested. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with storageClass field (per GetSourceFieldsNeededForCopy). Raises: SkipGlacierError: if skipping a s3 Glacier object. �s3�GLACIERN)r� r� r� r� �SkipGlacierError)r� r! r� r� r� �MaybeSkipUnsupportedObjecth s ��r� c C s4 |j r0t|j jt�}|s,td|j j| f ��|S dS )a� Ensures a matching decryption key is available for the source object. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with optional customerEncryption field. Raises: EncryptionException if the object is encrypted and no matching key is found. Returns: Base64-encoded decryption key string if the object is encrypted and a matching key is found, or None if object is not encrypted. zOMissing decryption key with SHA256 hash %s. No decryption key matches object %sN)ry rP � keySha256r r )r� r! r� r� r� r� �GetDecryptionCSEKy s � ��r� c C s |r|� � }ni }t|�}|�� r6|jdkr6t|�}nt� }d}d}d}|�� r�|�� rn|j|jkrntjsnd}tjr�t ||| � t ||� t ||�}|j }|j |_ tjr�|�� r�|jdkr�|js�td| ��|j|_|jdkr�t|�}|r�t||� n�|�r t�||| �}n|}z t|�}W nl t�y� } zRd|t|�f }|j�rl| jd7 _| �|� W Y d}~dS t|��W Y d}~n d}~0 0 |�� �s�|� � �r�d}n*|�r�|j �r�|�s�|j }nt j � |j �}tj!�r�| �"|j#d |� |jdk�r$|dk�r$|du�r$|t$k�r$td |t%t$�f ��t&�rT|�'� �rT|�� �rTt(�)t|��*� t j+� tj,�r�|j-�rnt.d ��nd |_-|�'� �r�t j �/|j ��r�t0� �nJ|�� �r�z|j1|j2|j |jd �}W n t3�y� d}Y n0 |�r�t0� �|�� �rP|j |_4|j2|_5|�� �r|j |_4|j2|_5n t6||� t7t8�}|�rP|j9t:j;k�rP|jdk�rP|j<|_=|�rdt>||dd� tj?�rttj?|_@t8�Add�tBk�r�d|_CtD| |||� |�� �rj|�'� �r�tE|jFtG||tH�H� tGjI|dd�� tJ||||| ||||| | |d� S |�r.tE|jFtG||tH�H� tGjK|dd�� tL|||||||d�S tE|jFtG||tH�H� tGjM|dd�� tN||||||| |d�S n�|�� �r�tO|||||||| ||| || d� } |�r�t �P|j4� | S tE|jFtG||tH�H� tGjQ|dd�� tR|||jF|d�} |�� �s|�� �stS|j || | d� | S dS )a� Performs copy from src_url to dst_url, handling various special cases. Args: logger: for outputting log messages. src_url: Source StorageUrl. dst_url: Destination StorageUrl. gsutil_api: gsutil Cloud API instance to use for the copy. command_obj: command object for use in Apply in parallel composite uploads and sliced object downloads. copy_exception_handler: for handling copy exceptions during Apply. src_obj_metadata: If source URL is a cloud object, source object metadata with all necessary fields (per GetSourceFieldsNeededForCopy). Required for cloud source URLs. If source URL is a file, an apitools Object that may contain file size, or None. allow_splitting: Whether to allow the file to be split into component pieces for an parallel composite upload or download. headers: optional headers to use for the copy operation. manifest: optional manifest for tracking copy operations. gzip_exts: List of file extensions to gzip, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. is_rsync: Whether or not the caller is the rsync command. preserve_posix: Whether or not to preserve posix attributes. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts. Streaming files compressed is only supported on the JSON GCS API. use_stet: If True, will perform STET encryption or decryption using the binary specified in the boto config or PATH. Returns: (elapsed_time, bytes_transferred, version-specific dst_url) excluding overhead like initial GET. Raises: ItemExistsError: if no clobber flag is specified and the destination object already exists. SkipUnsupportedObjectError: if skip_unsupported_objects flag is specified and the source is an unsupported type. CommandException: if other errors encountered. r� NFTzZNo OWNER permission found for object %s. OWNER permission is required for preserving ACLs.r� z Error opening file "%s": %s.r� r} z�"%s" exceeds the maximum gsutil-supported size for an S3 upload. S3 objects greater than %s in size require multipart uploads, which gsutil does not support.zASpecifying x-goog-if-generation-match is not supported with cp -nr rQ )�overrider% r& r� )r� r� ry r rz rx )r� r� r� )r r! r� )T�copyrn r� r� ro r r� r� r� r� r� r� r} r� r� r� r rp rk rA Zencrypt_uploadri ry r� �continue_on_errorr� r� r� r� r� r8 r� r� r� �Setr� �S3_MAX_UPLOAD_SIZErs rj r� �msvcrtZsetmode�fileno�O_BINARYr� � gen_matchr r$ �ItemExistsError�GetObjectMetadatar� r r� r� r� rQ r � crypto_typerN �CMEK� crypto_key� kmsKeyNamerl r� r� r' rV r| r� r_ r r2 rb r� r� r� r� r� r� r� r r� r� re ) r� r� r� r� r| r} r! r� �headers�manifestr� ry r r� rz Zdst_obj_headersr- r� r� r� Zcopy_in_the_cloudr� �acl_textZsource_stream_urlr� r6 Z dst_objectr� Zuploaded_metadata�resultr� r� r� � PerformCopy� s� 7 � �� �� �� � �� ��� ������ � ����r� c @ sZ e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z ddd�Z dd� Z dd� Z dS )�Manifestz2Stores the manifest items for the CpCommand class.c C s8 i | _ i | _t�� | _tj�|�| _| � � | � � d S r ) �items�manifest_filterr@ rh �lockr� r8 � expanduser� manifest_path�_ParseManifest�_CreateManifestFile)rN r8 r� r� r� rO � s zManifest.__init__c C s� z�t j�| j�r�t| jd���}d}t�|�}|D ]h}|rrz|�d�}|�d�}W n typ t d| j ��Y n0 d}|| }|| }|dv r0|| j |< q0W d � n1 s�0 Y W n t y� t d | j ��Y n0 dS ) z|Load and parse a manifest file. This information will be used to skip any files that have a skip or OK status. r< T�Source�Resultz$Missing headers in manifest file: %sF)�OK�skipNzCould not parse %s) r� r8 r$ r� r �csv�reader�indexrA r r� r@ ) rN �fZ first_rowr� �rowZ source_indexZ result_index�sourcer� r� r� r� r� � s* � . zManifest._ParseManifestc C s || j v S )z?Returns whether the specified src url was marked as successful.)r� )rN �srcr� r� r� � WasSuccessful� s zManifest.WasSuccessfulc C s� z�t j�| j�r"t �| j�jdkr�tjrrt| jddd��(}t � |�}|� g d�� W d � q�1 sf0 Y nFt| jdd��(}t � |�}|� g d�� W d � n1 s�0 Y W n t y� t d ��Y n0 dS ) z;Opens the manifest file and assigns it to the file pointer.r r# r� ��newline) r� Z DestinationZStartZEndZMd5�UploadIdz Source SizezBytes Transferredr� Z DescriptionNr� r� z Could not create manifest file.)r� r8 r$ r� r� �st_sizer r) r r� �writer�writerowr@ r )rN r� r� r� r� r� r� � s � . 0 z Manifest._CreateManifestFilec C s8 |d u r d S || j v r&|| j | |< n||i| j |< d S r �r� )rN r� r^ �valuer� r� r� r� s z Manifest.Setc C sD | � |d|� | � |d|� | � |dtjjtjjd�jd d�� d S )N� source_uri�destination_urir~ ��tz��tzinfo)r� �datetime�now�timezone�utcr )rN � source_urlr� r� r� r� � Initialize) s zManifest.Initializer� c C sf | � |d|� | � |d|� | � |d|� | � |dtjjtjjd�jd d�� | �|� | �|� d S )N�bytesr� � descriptionr� r� r� )r� r� r� r� r� r �_WriteRowToManifestFile�_RemoveItemFromManifest)rN r� r9 r� r� r� r� r� � SetResult/ s $ zManifest.SetResultc C s | j | }|d |d d|d �� d|d �� d|v rB|d ndd|v rT|d ndd |v rjt|d �ndd |v r�t|d �nd|d |d g }d d� |D �}| j�X tr�tjr�t| jdddd�}nt| jdd�}t � |�}|� |� |� � W d � n1 �s0 Y dS )zBWrites a manifest entry to the manifest file for the url argument.r� r� z%sZr~ r� r r� � upload_idr} r� r� r� c S s g | ]}t �|��qS r� )r r� )r� r� r� r� r� r� G r\ z4Manifest._WriteRowToManifestFile.<locals>.<listcomp>�ar� r� N) r� � isoformatr� r� rj r r) r r� r� r� r� r� )rN r� Zrow_itemr� r� r� r� r� r� r� 7 s( � z Manifest._WriteRowToManifestFilec C s | j |= d S r r� )rN r� r� r� r� r� T s z Manifest._RemoveItemFromManifestN)r� ) r� r� r� r� rO r� r� r� r� r� r� r� r� r� r� r� r� r� � s ' r� c @ s e Zd ZdZdS )r� zHException class for objects that are skipped because they already exist.Nr� r� r� r� r� r� [ s r� c s e Zd ZdZ� fdd�Z� ZS )�SkipUnsupportedObjectError�CException for objects skipped because they are an unsupported type.c s t t| ��� d| _d S )N�Unknown)�superr� rO �unsupported_typerQ �� __class__r� r� rO c s z#SkipUnsupportedObjectError.__init__�r� r� r� r� rO � __classcell__r� r� r� r� r� ` s r� c s e Zd ZdZ� fdd�Z� ZS )r� r� c s t t| ��� d| _d S )Nr� )r� r� rO r� rQ r� r� r� rO k s zSkipGlacierError.__init__r� r� r� r� r� r� h s r� c C s| | j }| �� rJ| jtd�d� }|�|�dkr2dS d|�|��|�d S | �� r\d| j S | j�|��|�d }t ||�S )a? Returns the path section before the final directory component of the URL. This handles cases for file system directories, bucket, and bucket subdirectories. Example: for gs://bucket/dir/ we'll return 'gs://bucket', and for file://dir we'll return file:// Args: url: StorageUrl representing a filesystem directory, cloud bucket or bucket subdir. exp_src_url: StorageUrl representing the fully expanded object to-be-copied; used for resolving cloud wildcards. Returns: String name of above-described path, sans final path separator. zfile://Nr� z file://%sr z%s://) r� r� r� r� �findr� r� r� r� �$ResolveWildcardsInPathBeforeFinalDir)r� r �sepZ past_schemeZpath_sans_final_dirr� r� r� r� p s r� c C sp t | �s | S t| �j�d�}|j�d�}tt|��D ] }t || �r4|| ||< q4d�|�}|�� }||_|jS )af Returns the path section for a bucket subdir with wildcards resolved. This handles cases for bucket subdirectories where the initial source URL contains a wildcard. In this case, src_url must be wildcard-expanded before calculating the final directory. Example: A bucket containing: gs://bucket/dir1/subdir/foo gs://bucket/dir2/subdir/foo and source URL gs://bucket/*/subdir and src_url_path_sans_final dir gs://bucket/* should yield final path gs://bucket/dir1 or gs://bucket/dir2 according to the expanded source URL. Args: src_url_path_sans_final_dir: URL string with wildcards representing a bucket subdir as computed from GetPathBeforeFinalDir. exp_src_url: CloudUrl representing the fully expanded object to-be-copied. Returns: String name of above-described path, sans final path separator. r� ) r- r0 r� r� r r� rz r r� )r r Zwildcarded_src_obj_pathZexpanded_src_obj_pathZpath_segment_indexZresolved_src_pathZfinal_path_urlr� r� r� r� � s �� � r� c C s, t | |�}tt||�d�}t | |�}||fS )a Gets info about a file partition for parallel file/object transfers. Args: file_size: The number of bytes in the file to be partitioned. max_components: The maximum number of components that can be composed. default_component_size: The size of a component, assuming that max_components is infinite. Returns: The number of components in the partitioned file, and the size of each component (except the last, which will have a different size iff file_size != 0 (mod num_components)). r� )rq �maxr� )rD r0 Zdefault_component_sizerF rG r� r� r� r> � s r> c C s@ t | |�}z |j|j|j|j|jd� W n ty: Y n0 dS )zGWrapper func to be used with command.Apply to delete temporary objects.r N)rI r! r� r� r{ r� r )r� Z url_to_deleter� r� r� r� r� rg � s � rg c C s� g }dd� |D �}| D ]}||vr|� | | � qg }g }g } |D �] } | j| �� vs`| j|v r�|�� } | j| _| j| _|� | � qB| | j } t| j| j| j�} t | �}z0| j }|j |j |j|j|j g d�d�}|j}W n ty� d}Y n0 ||k�r6|� | � |� | j� | j�rd| j �� }| j|_|� |� qB| j �� } | j| _| � | | jf� |� | j� qB| �r|t�dt| �� || |fS )a] Determines course of action for component objects. Given the list of all target objects based on partitioning the file and the list of objects that have already been uploaded successfully, this function determines which objects should be uploaded, which existing components are still valid, and which existing components should be deleted. Args: dst_args: The map of file_name -> PerformParallelUploadFileToObjectArgs calculated by partitioning the file. existing_components: A list of ObjectFromTracker objects that have been uploaded in the past. bucket_url: CloudUrl of the bucket in which the components exist. gsutil_api: gsutil Cloud API instance to use for retrieving object metadata. Returns: components_to_upload: List of components that need to be uploaded. uploaded_components: List of components that have already been uploaded and are still valid. Each element of the list contains the dst_url for the uploaded component and its size. existing_objects_to_delete: List of components that have already been uploaded, but are no longer valid and are in a versioned bucket, and therefore should be deleted. c S s g | ] }|j �qS r� )r� )r� r� r� r� r� r� s z,FilterExistingComponents.<locals>.<listcomp>)ry rz r| )r{ rR r_ Nz0Found %d existing temporary components to reuse.)rB r� r{ r r{ r# r� r� r� rS r� r� r� r� r| ry r� r� r� )rH r� � bucket_urlr� r� Zexisting_component_namesZcomponent_nameZobjects_already_chosenr� r� Ztracker_objectr� Zdst_argZ file_partrK r� Z dst_metadataZ cloud_md5Z!invalid_component_with_generationr� r� r� ri � sh � � �� �ri )N) FFFFFFFNFNN)F)F)NF)F)N)FN)N)F)FF)FFF)NTFF)N)N)NN)Nr NN)N)TNFFF)FFF)NN)N)FFFF) NTNNNFFFF)N( r� � __future__r r r r r � collectionsr r� r� r& r� r> r� r� �operatorr r� r� r� r� r� r� r r� r� r� r� rb r� � six.movesr �apitools.base.protorpcliter �botor r rn �gslib.cloud_apir r r r r r r r r r r r Zgslib.commands.composer Zgslib.commands.configr r r r r r �gslib.cs_api_mapr Zgslib.daisy_chain_wrapperr �gslib.exceptionr r! r" Zgslib.file_partr# Zgslib.parallel_tracker_filer$ r% r& r' r( �gslib.progress_callbackr) r* Z gslib.resumable_streaming_uploadr+ r, �gslib.storage_urlr- r. r/ r0 �"gslib.third_party.storage_apitoolsr1 r� �gslib.thread_messager2 r3 �gslib.tracker_filer4 r5 r6 r7 r8 r9 r: r; r< r= r> r? � gslib.utilsr@ rA rB rC Zgslib.utils.boto_utilrD rE rF rG rH � gslib.utils.cloud_api_helperrI rJ �gslib.utils.constantsrK rL rM � gslib.utils.encryption_helperrN rO rP rQ �gslib.utils.hashing_helperrR rS rT rU rV rW rX rY rZ r[ �gslib.utils.metadata_utilr\ �&gslib.utils.parallelism_framework_utilr] r^ r_ �gslib.utils.posix_utilr` ra rb rc rd re rf �gslib.utils.system_utilrg rh ri rj � gslib.utils.translation_helperrk rl rm rn ro rp �gslib.utils.unit_utilrq rr rs rt ru Zgslib.wildcard_iteratorrv r) rN rw r� � is_available�top_level_managerr} rh r| r r rA r? r� r~ r r� r� r� rU r� r� r� r� r� ry r� r� r� r� r� r� r� r� r� r� r� r� r r r$ r0 r5 r7 r< rM rP r� r� r� r� r� r� r� r� r� r� r� r� r r r� r, r1 r; rF �objectrG ra ri r7 rv r� r~ r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r> rg ri r� r� r� r� �<module> s� � �� ��� � �� < � 3 0� ) "E� C2 � M� j� C � j; � D:� N � & � '7D � T:3 0SK� = � z � 0� G � " � + , � � q� " � 3 /