OXIESEC PANEL

Name	Size	Modified	Perms
📁 ..	-	02/11/2025 08:19:48 AM	rwxr-xr-x
📄 __init__.py	808 bytes	12/09/2024 05:26:03 PM	rw-r--r--
📁 __pycache__	-	02/11/2025 08:19:49 AM	rwxr-xr-x
📄 mock_cloud_api.py	7.97 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 mock_logging_handler.py	1.28 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 rewrite_helper.py	2.8 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 signurl_signatures.py	5.7 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_Doption.py	9.72 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_acl.py	55.99 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_autoclass.py	6.85 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_boto_util.py	9.65 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_bucketconfig.py	4.98 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_bucketpolicyonly.py	3.78 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_cat.py	11.55 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_cloud_api_delegator.py	2 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_command.py	3.39 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_command_runner.py	20.66 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_compose.py	14.12 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_context_config.py	18.78 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_copy_helper_funcs.py	39.76 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_copy_objects_iterator.py	4.49 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_cors.py	12.45 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_cp.py	216 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_creds_config.py	8.64 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_daisy_chain_wrapper.py	14.13 KB	12/09/2024 05:26:03 PM	rw-r--r--
📁 test_data	-	12/09/2024 05:26:03 PM	rwxr-xr-x
📄 test_defacl.py	14.36 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_defstorageclass.py	5.43 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_du.py	10.61 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_encryption_helper.py	4.62 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_execution_util.py	3.88 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_file_part.py	3.38 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_gcs_json_api.py	2.9 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_gcs_json_credentials.py	9.85 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_gcs_json_media.py	7.44 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_gsutil.py	4.68 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_hash.py	9.58 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_hashing_helper.py	10.78 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_help.py	3.5 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_hmac.py	23.9 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_iam.py	90.67 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_kms.py	16.68 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_label.py	11.55 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_lifecycle.py	13.8 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_logging.py	3.5 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_ls.py	53.16 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_mb.py	19.64 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_metrics.py	51.65 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_mtls.py	2.01 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_mv.py	13.04 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_naming.py	63.11 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_notification.py	5.9 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_notification_pubsub.py	5.46 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_pap.py	5.91 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_parallel_cp.py	10.15 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_parallelism_framework.py	33.09 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_perfdiag.py	12.62 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_plurality_checkable_iterator.py	7.53 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_posix_util.py	2.03 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_psc.py	5.88 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_rb.py	2.93 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_requester_pays.py	11.7 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_resumable_streaming.py	12.36 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_retention.py	28.9 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_retention_util.py	5.49 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_rewrite.py	31.62 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_rm.py	33.72 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_rpo.py	10.22 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_rsync.py	149.33 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_rsync_funcs.py	3.36 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_seek_ahead_thread.py	8.79 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_setmeta.py	12.54 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_shim_util.py	64.19 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_signurl.py	24.69 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_stat.py	11.38 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_stet_cp.py	5.77 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_stet_util.py	7.38 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_storage_url.py	7.02 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_tabcomplete.py	14.31 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_temporary_file_util.py	1.54 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_trace.py	1.76 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_tracker_file.py	9.9 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_ubla.py	3.88 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_ui.py	67.42 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_update.py	10.37 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_user_agent_helper.py	5.34 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_util.py	19.85 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_versioning.py	3.61 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_web.py	6.54 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_wildcard_iterator.py	22.18 KB	12/09/2024 05:26:03 PM	rw-r--r--
📄 test_wrapped_credentials.py	13.95 KB	12/09/2024 05:26:03 PM	rw-r--r--
📁 testcase	-	12/09/2024 05:26:03 PM	rwxr-xr-x
📄 util.py	29.01 KB	12/09/2024 05:26:03 PM	rw-r--r--

Editing: test_perfdiag.py

# -*- coding: utf-8 -*-
# Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Integration tests for perfdiag command."""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals

import os
import socket
import sys

import six

import boto
from gslib.commands.perfdiag import _GenerateFileData
import gslib.tests.testcase as testcase
from gslib.tests.testcase.integration_testcase import SkipForXML
from gslib.tests.util import ObjectToURI as suri
from gslib.tests.util import RUN_S3_TESTS
from gslib.tests.util import unittest
from gslib.utils.system_util import IS_WINDOWS

from six import add_move, MovedModule

add_move(MovedModule('mock', 'mock', 'unittest.mock'))
from six.moves import mock

class TestPerfDiag(testcase.GsUtilIntegrationTestCase):
  """Integration tests for perfdiag command."""

@classmethod
  def setUpClass(cls):
    super(TestPerfDiag, cls).setUpClass()
    # We want to test that perfdiag works both when connecting to the standard
    # gs endpoint, and when connecting to a specific IP or host while setting
    # the host header. For the 2nd case we resolve gs_host (normally
    # storage.googleapis.com) to a specific IP and connect to that explicitly.
    gs_host = boto.config.get('Credentials', 'gs_host',
                              boto.gs.connection.GSConnection.DefaultHost)
    gs_ip = None
    for address_tuple in socket.getaddrinfo(gs_host, None):
      # Index 0 holds IP version. AF_INET = IPv4.
      if address_tuple[0].name in ('AF_INET', 'AF_INET6'):
        # Index 4 holds IP tuple, where first item is IP.
        gs_ip = address_tuple[4][0]
        break
    if not gs_ip:
      raise ConnectionError('Count not find IP for ' + gs_host)

cls._custom_endpoint_flags = [
        '-o', 'Credentials:gs_host=' + gs_ip, '-o',
        'Credentials:gs_host_header=' + gs_host, '-o',
        'Boto:https_validate_certificates=False'
    ]

def _should_run_with_custom_endpoints(self):
    # Host headers are only supported for XML, and not when
    # using environment variables for proxies.
    # TODO: Currently this is disabled for Python versions
    # >= 2.7.9 which cause certificate errors due to validation
    # added in https://www.python.org/dev/peps/pep-0466/
    # If https://github.com/boto/boto/pull/2857 or its analog
    # is accepted in boto, set https_validate_certificates to False
    # in these tests and re-enable them.
    python_version_less_than_2_7_9 = (sys.version_info[0] == 2 and (
        (sys.version_info[1] < 7) or
        (sys.version_info[1] == 7 and sys.version_info[2] < 9)))
    return (self.test_api == 'XML' and not RUN_S3_TESTS and
            python_version_less_than_2_7_9 and
            not (os.environ.get('http_proxy') or os.environ.get('https_proxy')
                 or os.environ.get('HTTPS_PROXY')))

def test_latency(self):
    bucket_uri = self.CreateBucket()
    cmd = ['perfdiag', '-n', '1', '-t', 'lat', suri(bucket_uri)]
    self.RunGsUtil(cmd)
    if self._should_run_with_custom_endpoints():
      self.RunGsUtil(self._custom_endpoint_flags + cmd)
    self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)

def _run_throughput_test(self,
                           test_name,
                           num_processes,
                           num_threads,
                           parallelism_strategy=None,
                           compression_ratio=None):
    bucket_uri = self.CreateBucket()

cmd = [
        'perfdiag', '-n',
        str(num_processes * num_threads), '-s', '1024', '-c',
        str(num_processes), '-k',
        str(num_threads), '-t', test_name
    ]
    if compression_ratio is not None:
      cmd += ['-j', str(compression_ratio)]
    if parallelism_strategy is not None:
      cmd += ['-p', parallelism_strategy]
    cmd += [suri(bucket_uri)]

stderr_default = self.RunGsUtil(cmd, return_stderr=True)
    stderr_custom = None
    if self._should_run_with_custom_endpoints():
      stderr_custom = self.RunGsUtil(self._custom_endpoint_flags + cmd,
                                     return_stderr=True)
    self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
    return (stderr_default, stderr_custom)

def _run_each_parallel_throughput_test(self,
                                         test_name,
                                         num_processes,
                                         num_threads,
                                         compression_ratio=None):
    self._run_throughput_test(test_name,
                              num_processes,
                              num_threads,
                              'fan',
                              compression_ratio=compression_ratio)
    if not RUN_S3_TESTS:
      self._run_throughput_test(test_name,
                                num_processes,
                                num_threads,
                                'slice',
                                compression_ratio=compression_ratio)
      self._run_throughput_test(test_name,
                                num_processes,
                                num_threads,
                                'both',
                                compression_ratio=compression_ratio)

def test_write_throughput_single_process_single_thread(self):
    self._run_throughput_test('wthru', 1, 1)
    self._run_throughput_test('wthru_file', 1, 1)

def test_write_throughput_single_process_multi_thread(self):
    self._run_each_parallel_throughput_test('wthru', 1, 2)
    self._run_each_parallel_throughput_test('wthru_file', 1, 2)

@unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows')
  def test_write_throughput_multi_process_single_thread(self):
    self._run_each_parallel_throughput_test('wthru', 2, 1)
    self._run_each_parallel_throughput_test('wthru_file', 2, 1)

@unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows')
  def test_write_throughput_multi_process_multi_thread(self):
    self._run_each_parallel_throughput_test('wthru', 2, 2)
    self._run_each_parallel_throughput_test('wthru_file', 2, 2)

def test_read_throughput_single_process_single_thread(self):
    self._run_throughput_test('rthru', 1, 1)
    self._run_throughput_test('rthru_file', 1, 1)

def test_read_throughput_single_process_multi_thread(self):
    self._run_each_parallel_throughput_test('rthru', 1, 2)
    self._run_each_parallel_throughput_test('rthru_file', 1, 2)

@unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows')
  def test_read_throughput_multi_process_single_thread(self):
    self._run_each_parallel_throughput_test('rthru', 2, 1)
    self._run_each_parallel_throughput_test('rthru_file', 2, 1)

@unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows')
  def test_read_throughput_multi_process_multi_thread(self):
    self._run_each_parallel_throughput_test('rthru', 2, 2)
    self._run_each_parallel_throughput_test('rthru_file', 2, 2)

@unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows')
  def test_read_and_write_file_ordering(self):
    """Tests that rthru_file and wthru_file work when run together."""
    self._run_throughput_test('rthru_file,wthru_file', 1, 1)
    self._run_throughput_test('rthru_file,wthru_file', 2, 2, 'fan')
    if not RUN_S3_TESTS:
      self._run_throughput_test('rthru_file,wthru_file', 2, 2, 'slice')
      self._run_throughput_test('rthru_file,wthru_file', 2, 2, 'both')

def test_input_output(self):
    outpath = self.CreateTempFile()
    bucket_uri = self.CreateBucket()
    self.RunGsUtil(
        ['perfdiag', '-o', outpath, '-n', '1', '-t', 'lat',
         suri(bucket_uri)])
    self.RunGsUtil(['perfdiag', '-i', outpath])

def test_invalid_size(self):
    stderr = self.RunGsUtil(
        ['perfdiag', '-n', '1', '-s', 'foo', '-t', 'wthru', 'gs://foobar'],
        expected_status=1,
        return_stderr=True)
    self.assertIn('Invalid -s', stderr)

def test_toobig_size(self):
    stderr = self.RunGsUtil(
        ['perfdiag', '-n', '1', '-s', '3pb', '-t', 'wthru', 'gs://foobar'],
        expected_status=1,
        return_stderr=True)
    self.assertIn('in-memory tests maximum file size', stderr)

def test_listing(self):
    bucket_uri = self.CreateBucket()
    stdout = self.RunGsUtil(
        ['perfdiag', '-n', '1', '-t', 'list',
         suri(bucket_uri)],
        return_stdout=True)
    self.assertIn('Number of listing calls made:', stdout)
    self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)

@SkipForXML('No compressed transport encoding support for the XML API.')
  def test_gzip_write_throughput_single_process_single_thread(self):
    (stderr_default, _) = self._run_throughput_test('wthru',
                                                    1,
                                                    1,
                                                    compression_ratio=50)
    self.assertIn('Gzip compression ratio: 50', stderr_default)
    self.assertIn('Gzip transport encoding writes: True', stderr_default)
    (stderr_default, _) = self._run_throughput_test('wthru_file',
                                                    1,
                                                    1,
                                                    compression_ratio=50)
    self.assertIn('Gzip compression ratio: 50', stderr_default)
    self.assertIn('Gzip transport encoding writes: True', stderr_default)

@SkipForXML('No compressed transport encoding support for the XML API.')
  def test_gzip_write_throughput_single_process_multi_thread(self):
    self._run_each_parallel_throughput_test('wthru', 1, 2, compression_ratio=50)
    self._run_each_parallel_throughput_test('wthru_file',
                                            1,
                                            2,
                                            compression_ratio=50)

@unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows')
  @SkipForXML('No compressed transport encoding support for the XML API.')
  def test_gzip_write_throughput_multi_process_multi_thread(self):
    self._run_each_parallel_throughput_test('wthru', 2, 2, compression_ratio=50)
    self._run_each_parallel_throughput_test('wthru_file',
                                            2,
                                            2,
                                            compression_ratio=50)

class TestPerfDiagUnitTests(testcase.GsUtilUnitTestCase):
  """Unit tests for perfdiag command."""

def test_listing_does_not_list_preexisting_objects(self):
    test_objects = 1
    bucket_uri = self.CreateBucket()
    # Create two objects in the bucket before executing perfdiag.
    self.CreateObject(bucket_uri=bucket_uri, contents=b'foo')
    self.CreateObject(bucket_uri=bucket_uri, contents=b'bar')
    mock_log_handler = self.RunCommand(
        'perfdiag',
        ['-n', str(test_objects), '-t', 'list',
         suri(bucket_uri)],
        return_log_handler=True)
    self.assertNotIn(
        'Listing produced more than the expected %d object(s).' % test_objects,
        mock_log_handler.messages['warning'])

@mock.patch('os.urandom')
  def test_generate_file_data(self, mock_urandom):
    """Test the right amount of random and sequential data is generated."""

def urandom(length):
      return b'a' * length

mock_urandom.side_effect = urandom

fp = six.BytesIO()
    _GenerateFileData(fp, 1000, 100, 1000)
    self.assertEqual(b'a' * 1000, fp.getvalue())
    self.assertEqual(1000, fp.tell())

fp = six.BytesIO()
    _GenerateFileData(fp, 1000, 50, 1000)
    self.assertIn(b'a' * 500, fp.getvalue())
    self.assertIn(b'x' * 500, fp.getvalue())
    self.assertEqual(1000, fp.tell())

fp = six.BytesIO()
    _GenerateFileData(fp, 1001, 50, 1001)
    self.assertIn(b'a' * 501, fp.getvalue())
    self.assertIn(b'x' * 500, fp.getvalue())
    self.assertEqual(1001, fp.tell())

@mock.patch('os.urandom')
  def test_generate_file_data_repeat(self, mock_urandom):
    """Test that random data repeats when exhausted."""

def urandom(length):
      return b'a' * length

mock_urandom.side_effect = urandom

fp = six.BytesIO()
    _GenerateFileData(fp, 8, 50, 4)
    self.assertEqual(b'aaxxaaxx', fp.getvalue())
    self.assertEqual(8, fp.tell())