From 7739890bccf068b252397a9ce9f6035bbe9bf828 Mon Sep 17 00:00:00 2001 From: jgoutin Date: Fri, 29 Mar 2019 16:52:22 +0100 Subject: [PATCH] Fix documentation, test and release 1.3.0 --- README.md | 8 +- docs/api_storage.rst | 2 +- docs/api_storage_azure_blob.rst | 86 +++++++++++++++++++++- docs/api_storage_azure_file.rst | 33 ++++++++- docs/api_storage_http.rst | 7 +- docs/api_storage_oss.rst | 4 +- docs/api_storage_s3.rst | 6 +- docs/api_storage_swift.rst | 2 +- docs/changes.rst | 47 ++++++------ docs/getting_started.rst | 10 +-- docs/index.rst | 12 +-- pycosio/__init__.py | 2 +- pycosio/_core/io_base_buffered.py | 64 +++++++++------- pycosio/_core/io_base_raw.py | 8 +- pycosio/storage/azure_blob/_append_blob.py | 4 + tests/test_core_io_buffered.py | 51 ------------- tests/test_storage.py | 53 +++++++++++++ 17 files changed, 271 insertions(+), 128 deletions(-) diff --git a/README.md b/README.md index 8573a9f..0297f4c 100644 --- a/README.md +++ b/README.md @@ -17,20 +17,20 @@ Pycosio brings standard Python I/O to cloud objects by providing: * Features equivalent to the standard library (``io``, ``os``, ``os.path``, ``shutil``) for seamlessly managing cloud objects and local files. -Theses functions are source agnostic and always provide the same interface for +These functions are source agnostic and always provide the same interface for all files from cloud storage or local file systems. -Buffered cloud objects also support following features: +Buffered cloud objects also support the following features: * Buffered asynchronous writing of any object size. -* Buffered asynchronous preloading in read mode. +* Buffered asynchronous preloading in reading mode. * Write or read lock depending on memory usage limitation. * Maximization of bandwidth using parallels connections. Supported Cloud storage ----------------------- -Pycosio is compatible with following cloud objects storage services: +Pycosio is compatible with the following cloud objects storage services: * Alibaba Cloud OSS * Amazon Web Services S3 diff --git a/docs/api_storage.rst b/docs/api_storage.rst index 055bf42..80f0dcf 100644 --- a/docs/api_storage.rst +++ b/docs/api_storage.rst @@ -15,7 +15,7 @@ The following table shows features available for each storage. API --- -The following pages describes each storage. +The following pages describe each storage. .. automodule:: pycosio.storage :members: diff --git a/docs/api_storage_azure_blob.rst b/docs/api_storage_azure_blob.rst index 2ec0d03..2e8cab3 100644 --- a/docs/api_storage_azure_blob.rst +++ b/docs/api_storage_azure_blob.rst @@ -12,7 +12,7 @@ An Azure storage account can be mounted using the Pycosio ``mount`` function. ``azure.storage.blob.baseblobservice.BaseBlobService`` class from ``azure-storage-blob`` Python library. -This example show the mount of Azure Storage Blob with the minimal +This example shows the mount of Azure Storage Blob with the minimal configuration: .. code-block:: python @@ -45,6 +45,90 @@ Limitation Only one configuration per Azure Storage account can be mounted simultaneously. +Azure blob type selection +------------------------- + +It is possible to select the blob type for new files created using the +``blob_type`` argument. + +Possible blob types are ``BlockBlob``, ``AppendBlob`` & ``PageBlob``. + +The default blob type can be set when mounting the storage +(if not specified, the ``BlockBlob`` type is used by default): + +.. code-block:: python + + import pycosio + + pycosio.mount(storage='azure_blob', storage_parameters=dict( + account_name='my_account_name', account_key='my_account_key', + + # Using PageBlob by default for new files + blob_type='PageBlob', + ) + ) + +It can also be selected for a specific file when opening it in write mode: + +.. code-block:: python + + # Open a new file in write mode as PageBlob + with pycosio.open( + 'https://my_account.blob.core.windows.net/my_container/my_blob', + 'wb', blob_type='PageBlob') as file: + file.write(b'0') + +Page blobs specific features +---------------------------- + +The page blob supports the following specific features. + +Preallocating pages +~~~~~~~~~~~~~~~~~~~ + +When flushing a page blob out of its current size, pycosio first resize the +blob to allow the flush of the new data. + +In case of multiple flushes on a raw IO or when using a buffered IO, this is +done with extra requests to the Azure server. If The size to write is known +before opening the file, it is possible to avoid these extra requests by +to preallocate the required size in only one initial request. + +The ``content_length`` argument allow preallocating a Page blob to a defined +size when opening it in write mode: + +.. code-block:: python + + # Open a new page blob and preallocate it with 1024 bytes. + with pycosio.open( + 'https://my_account.blob.core.windows.net/my_container/my_blob', + 'wb', blob_type='PageBlob', content_length=1024) as file: + file.write(b'1') + + # Append on an existing page blob and pre-resize it to 2048 bytes. + with pycosio.open( + 'https://my_account.blob.core.windows.net/my_container/my_blob', + 'ab', blob_type='PageBlob', content_length=2048) as file: + file.write(b'1') + +The preallocation is done with padding of null characters (``b'\0'``). + +End page padding handling +~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, Pycosio tries to handle page blobs like standard files by ignoring +trailing page padding of null characters: + +* When opening a file in append mode (Seek to the end of file after ignoring + trailing padding of the last page). +* When reading data (Read until a null character reaches). +* When using the ``seek()`` method with ``whence=os.SEEK_END`` (Ignore the + trailing padding when determining the end of the file to use as the reference + position) + +This behaviour can be disabled using the ``ignore_padding=False`` argument when +opening the page blob. + Files objects classes --------------------- diff --git a/docs/api_storage_azure_file.rst b/docs/api_storage_azure_file.rst index 789ebbd..afe0bf8 100644 --- a/docs/api_storage_azure_file.rst +++ b/docs/api_storage_azure_file.rst @@ -12,7 +12,7 @@ An Azure storage account can be mounted using the Pycosio ``mount`` function. ``azure.storage.file.fileservice.FileService`` class from ``azure-storage-file`` Python library. -This example show the mount of Azure Storage File with the minimal +This example shows the mount of Azure Storage File with the minimal configuration: .. code-block:: python @@ -45,6 +45,37 @@ Limitation Only one configuration per Azure Storage account can be mounted simultaneously. +Preallocating files +------------------- + +When flushing a file out of its current size, pycosio first resize the +file to allow the flush of the new data. + +In case of multiple flushes on a raw IO or when using a buffered IO, this is +done with extra requests to the Azure server. If The size to write is known +before opening the file, it is possible to avoid these extra requests by +to preallocate the required size in only one initial request. + +The ``content_length`` argument allow preallocating a file to a defined +size when opening it in write mode: + +.. code-block:: python + + # Open a new file and preallocate it with 1024 bytes. + with pycosio.open( + '//my_account.file.core.windows.net/my_share/my_file', + 'wb', content_length=1024) as file: + file.write(b'1') + + # Append on an existing file and pre-resize it to 2048 bytes. + with pycosio.open( + '//my_account.file.core.windows.net/my_share/my_file', + 'ab', content_length=2048) as file: + file.write(b'1') + +The preallocation is done with padding of null characters (``b'\0'``). + + Files objects classes --------------------- diff --git a/docs/api_storage_http.rst b/docs/api_storage_http.rst index d4e0b19..948f602 100644 --- a/docs/api_storage_http.rst +++ b/docs/api_storage_http.rst @@ -1,14 +1,15 @@ pycosio.storage.http ==================== -HTTP/HTTPS object read only access. +HTTP/HTTPS object read-only access. Mount ----- -The HTTP storage does not require to be mounted prior to be used. +The HTTP storage does not require to be mounted prior to being used. -Function can be used directly on any HTTP object reachable by the Pycosio host: +The function can be used directly on any HTTP object reachable by the Pycosio +host: .. code-block:: python diff --git a/docs/api_storage_oss.rst b/docs/api_storage_oss.rst index 5e14242..cdee0c4 100644 --- a/docs/api_storage_oss.rst +++ b/docs/api_storage_oss.rst @@ -13,11 +13,11 @@ OSS can be mounted using the Pycosio ``mount`` function. ``oss2`` Python library (The class selection is done automatically based on parameters found in ``storage_parameters``). -Pycosio also require one extra argument, the ``endpoint``, which is basically +Pycosio also requires one extra argument, the ``endpoint``, which is basically the URL of the OSS Alibaba region to use. (See ``endpoint`` argument of the ``oss2.Bucket`` class) -This example show the mount of OSS with the minimal configuration: +This example shows the mount of OSS with the minimal configuration: .. code-block:: python diff --git a/docs/api_storage_s3.rst b/docs/api_storage_s3.rst index bbc2d48..016dc30 100644 --- a/docs/api_storage_s3.rst +++ b/docs/api_storage_s3.rst @@ -15,7 +15,7 @@ arguments to pass to ``boto3.session.Session(**session_parameters)`` from the It can also include a sub-directory ``client`` that is used to pass arguments to ``boto3.session.Session.client('s3', **client_parameters)``. -This example show the mount of S3 with the minimal configuration: +This example shows the mount of S3 with the minimal configuration: .. code-block:: python @@ -39,12 +39,12 @@ This example show the mount of S3 with the minimal configuration: Automatic mount ~~~~~~~~~~~~~~~ -It is not required to mount S3 explicitly When using pycosio on a host +It is not required to mount S3 explicitly when using pycosio on a host configured to handle AWS S3 access (Through IAM policy, configuration files, environment variables, ...). In this case, mounting is done transparently on the first call of a Pycosio -function on a S3 object and no configuration or extra steps are required: +function on an S3 object and no configuration or extra steps are required: .. code-block:: python diff --git a/docs/api_storage_swift.rst b/docs/api_storage_swift.rst index 2e3555b..bc0bc74 100644 --- a/docs/api_storage_swift.rst +++ b/docs/api_storage_swift.rst @@ -12,7 +12,7 @@ An OpenStack Swift project can be mounted using the Pycosio ``mount`` function. ``swiftclient.client.Connection`` class from ``python-swiftclient`` Python library. -This example show the mount of OpenStack Swift with the minimal configuration: +This example shows the mount of OpenStack Swift with a minimal configuration: .. code-block:: python diff --git a/docs/changes.rst b/docs/changes.rst index fe1edba..af91659 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,7 +1,7 @@ Changelog ========= -1.3.0 (2019/??) +1.3.0 (2019/03) --------------- Add support for following cloud storage: @@ -11,8 +11,8 @@ Add support for following cloud storage: Improvements: -* ``io.RawIOBase`` can now be used for storage that support random write access. -* OSS: Copy objects between OSS buckets without copying data on client when +* ``io.RawIOBase`` can now be used for storage that supports random write access. +* OSS: Copy objects between OSS buckets without copying data on the client when possible. Deprecations: @@ -21,23 +21,25 @@ Deprecations: Fixes: -* Fix file methods not translate Cloud Storage exception into OSError. +* Fix unsupported operation not risen in all cases with raw and buffered IO. +* Fix call of ``flush()`` in buffered IO. +* Fix file methods not translate cloud storage exception into ``OSError``. * Fix file not create on open in write mode (Was only created on flush). * Fix file closed twice when using context manager. * Fix root URL detection in some cases. * Fix too many returned result when listing objects with a count limit. * Fix error when trying to append on a not existing file. -* Fix ``io.RawIOBase`` not generating padding when seeking after end of file. +* Fix ``io.RawIOBase`` not generating padding when seeking after the end of the file. * OSS: Fix error when listing objects in a not existing directory. -* OSS: Fix read error if try to read after end of file. +* OSS: Fix read error if try to read after the end of the file. * OSS: Fix buffered write minimum buffer size. * OSS: Clean up multipart upload parts on failed uploads. -* OSS: Fix error when opening existing file in 'a' mode. -* S3: Fix error when creating a bucket due to unspecified region. -* S3: Fix unprocessed error in listing bucket content of an not existing bucket. +* OSS: Fix error when opening an existing file in 'a' mode. +* S3: Fix error when creating a bucket due to an unspecified region. +* S3: Fix unprocessed error in listing bucket content of a not existing bucket. * S3: Clean up multipart upload parts on failed uploads. * S3: Fix missing transfer acceleration endpoints. -* Swift: Fix error when opening existing file in 'a' mode. +* Swift: Fix error when opening an existing file in 'a' mode. 1.2.0 (2018/10) --------------- @@ -50,23 +52,23 @@ New standard library equivalent functions: Improvements: -* Copy of objects from and to a same storage is performed directly on remote +* Copy of objects from and to the same storage is performed directly on remote server if possible. * Pycosio now raises ``io.UnsupportedOperation`` if an operation is not - compatible with the current storage, this apply to all newly created function + compatible with the current storage, this applies to all newly created function and following existing functions: ``getsize``, ``getmtime``, ``mkdir``. Fixes: * ``io.BufferedIOBase.read`` now returns empty bytes instead of raising - exception when trying to read if seek already at end of file. + exception when trying to read if seek already at end of the file. * ``copy`` destination can now be a storage directory and not only a local directory. * ``copy`` now checks if destination parent directory exists and if files - are not same file and raise proper exceptions. + are not the same file and raise proper exceptions. * ``mkdir``: missing ``dir_fd`` argument. * ``isdir`` now correctly handle "virtual" directories (Directory that don't - exist as proper object, but exists in another object path). + exist as a proper object, but exists in another object path). 1.1.0 (2018/10) --------------- @@ -84,39 +86,38 @@ Backward incompatible change: Improvements: * No buffer copy when using ``io.BufferedIOBase.read`` with exactly - buffer size. This may lead performance improvement. + buffer size. This may lead to performance improvement. * Minimum packages versions are set in setup based on packages changelog or date. Fixes: -* ``isfile`` now correctly returns ``False`` when used on directory. +* ``isfile`` now correctly returns ``False`` when used on a directory. * ``relpath`` now keeps ending ``/`` on cloud storage path (Directory marker). 1.0.0 (2018/08) --------------- -First version that implement the core machinery. +The first version that implements the core machinery. Provides cloud storage equivalent functions of: * ``open`` / ``io.open``, ``shutil.copy``, ``os.path.getmtime``, ``os.path.getsize``, ``os.path.isfile``, ``os.path.relpath``. -Provides cloud objects abstract classes with following interfaces: +Provide cloud objects abstract classes with the following interfaces: * ``io.RawIOBase``, ``io.BufferedIOBase``. -Adds support for following cloud storage: +Add support for following cloud storage: * Alibaba Cloud OSS * AWS S3 * OpenStack Swift -Adds read only generic HTTP/HTTPS objects support. +Add read-only generic HTTP/HTTPS objects support. Known issues ------------ -* Append mode don't work with ``ObjectBufferedIOBase``. -* ``unsecure`` parameter is not supported on Google Cloud Storage. \ No newline at end of file +* Append mode doesn't work with ``ObjectBufferedIOBase``. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index f16876a..3e63971 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -6,7 +6,7 @@ Installation Supported Python versions: 2.7, 3.4, 3.5, 3.6, 3.7 -Python >= 3.6 is recommanded to provide fully featured local file support. +Python >= 3.6 is recommended to provide fully featured local file support. Installation is performed using PIP: @@ -24,7 +24,7 @@ You can also install these optional extras: * ``s3``: Amazon Web Services S3 support. * ``swift``: OpenStack Swift support. -Example for installing Pycosio with all dependencies: +Example of installing Pycosio with all dependencies: .. code-block:: bash @@ -78,13 +78,13 @@ Examples of functions: Cloud storage configuration --------------------------- -Like with file systems, Cloud storage need to be *mounted* to be used. +Like with file systems, Cloud storage needs to be *mounted* to be used. Some storage requires configuration before use (such as user access keys). For the required parameter detail, see the targeted storage class or the targeted storage documentation. -Storage that does not require configuration are automatically mounted. +Storage that does not require configuration is automatically mounted. All storage parameters must be defined in a ``storage_parameters`` dictionary. This dictionary must be transmitted either to the ``pycosio.mount`` function @@ -128,7 +128,7 @@ the ``storage_parameters`` dictionary. storage_parameters=storage_parameters) as file: file.read() - # Next calls uses mounted storage transparently + # Next calls use mounted storage transparently with pycosio.open( 'https://my_cloud.com/my_other_object', 'rt') as file: file.read() diff --git a/docs/index.rst b/docs/index.rst index 1c34a62..b3ffb08 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,13 +8,13 @@ Pycosio brings standard Python I/O to cloud objects by providing: * Features equivalent to the standard library (``io``, ``os``, ``os.path``, ``shutil``) for seamlessly managing cloud objects and local files. -Theses functions are source agnostic and always provide the same interface for +These functions are source agnostic and always provide the same interface for all files from cloud storage or local file systems. -Buffered cloud objects also support following features: +Buffered cloud objects also support the following features: * Buffered asynchronous writing of any object size. -* Buffered asynchronous preloading in read mode. +* Buffered asynchronous preloading in reading mode. * Write or read lock depending on memory usage limitation. * Maximization of bandwidth using parallels connections. @@ -32,7 +32,7 @@ Example of code: with pycosio.open('s3://my_bucket/my_object.bin', 'wb') as file: file.write(b'binary_data') - # Copy file from local file system to OpenStack Swift + # Copy file from the local file system to OpenStack Swift pycosio.copy( 'my_file', 'https://objects.my_cloud.com/v1/12345678912345/my_container/my_object') @@ -44,7 +44,7 @@ Example of code: Supported Cloud storage ----------------------- -Pycosio is compatible with following cloud objects storage services: +Pycosio is compatible with the following cloud objects storage services: * Alibaba Cloud OSS * Amazon Web Services S3 @@ -58,7 +58,7 @@ Pycosio can also access any publicly accessible file via HTTP/HTTPS Limitations ----------- -Cloud object storage are not file systems and have following limitations: +Cloud object storage is not file systems and has the following limitations: - Cloud objects are not seekable in write mode. - Cloud objects must be written entirely at once. diff --git a/pycosio/__init__.py b/pycosio/__init__.py index d8442e1..876a0a9 100644 --- a/pycosio/__init__.py +++ b/pycosio/__init__.py @@ -16,7 +16,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -__version__ = '1.2.0' +__version__ = '1.3.0' __copyright__ = "Copyright 2018 Accelize" __licence__ = "Apache 2.0" diff --git a/pycosio/_core/io_base_buffered.py b/pycosio/_core/io_base_buffered.py index be269d5..f2a6bcf 100644 --- a/pycosio/_core/io_base_buffered.py +++ b/pycosio/_core/io_base_buffered.py @@ -52,6 +52,11 @@ class ObjectBufferedIOBase(BufferedIOBase, ObjectIOBase, WorkerPoolBase): def __init__(self, name, mode='r', buffer_size=None, max_buffers=0, max_workers=None, **kwargs): + if 'a' in mode: + # TODO: Implement append mode and remove this exception + raise NotImplementedError( + 'Not implemented yet in Pycosio') + BufferedIOBase.__init__(self) ObjectIOBase.__init__(self, name, mode=mode) WorkerPoolBase.__init__(self, max_workers) @@ -120,22 +125,10 @@ def close(self): if self._writable and not self._closed: self._closed = True with self._seek_lock: - # Flush on close only if bytes written - # This avoid no required process/thread - # creation and network call. - # This step is performed by raw stream. - if self._buffer_seek and self._seek: - self._seek += 1 - with handle_os_exceptions(): - self._flush() - self._close_writable() - - # If closed and data lower than buffer size - # flush data with raw stream to reduce IO calls - elif self._buffer_seek: - self._raw._write_buffer = self._get_buffer() - self._raw._seek = self._buffer_seek - self._raw.flush() + self._flush_raw_or_buffered() + if self._seek: + with handle_os_exceptions(): + self._close_writable() def _close_writable(self): """ @@ -154,19 +147,32 @@ def flush(self): """ if self._writable: with self._seek_lock: - # Advance seek - # This is the number of buffer flushed - # including the current one - self._seek += 1 - - # Write buffer to cloud object - with handle_os_exceptions(): - self._flush() + self._flush_raw_or_buffered() # Clear the buffer self._write_buffer = bytearray(self._buffer_size) self._buffer_seek = 0 + def _flush_raw_or_buffered(self): + """ + Flush using raw of buffered methods. + """ + # Flush only if bytes written + # This avoid no required process/thread + # creation and network call. + # This step is performed by raw stream. + if self._buffer_seek and self._seek: + self._seek += 1 + with handle_os_exceptions(): + self._flush() + + # If data lower than buffer size + # flush data with raw stream to reduce IO calls + elif self._buffer_seek: + self._raw._write_buffer = self._get_buffer() + self._raw._seek = self._buffer_seek + self._raw.flush() + @abstractmethod def _flush(self): """ @@ -196,6 +202,9 @@ def peek(self, size=-1): Returns: bytes: bytes read """ + if not self._readable: + raise UnsupportedOperation('read') + with self._seek_lock: self._raw.seek(self._seek) return self._raw._peek(size) @@ -315,6 +324,9 @@ def readinto(self, b): Returns: int: number of bytes read """ + if not self._readable: + raise UnsupportedOperation('read') + with self._seek_lock: # Gets seek seek = self._seek @@ -451,11 +463,13 @@ def seek(self, offset, whence=SEEK_SET): # Set seek using raw method and # sync buffered seek with raw seek self.raw.seek(offset, whence) - self._seek = self.raw._seek + self._seek = seek = self.raw._seek # Preload starting from current seek self._preload_range() + return seek + def write(self, b): """ Write the given bytes-like object, b, to the underlying raw stream, diff --git a/pycosio/_core/io_base_raw.py b/pycosio/_core/io_base_raw.py index 6b2925c..b8e11e0 100644 --- a/pycosio/_core/io_base_raw.py +++ b/pycosio/_core/io_base_raw.py @@ -108,7 +108,7 @@ def _init_append(self): Initializes file on 'a' mode. """ # Require to load the full file content in buffer - self._write_buffer[:] = self.readall() + self._write_buffer[:] = self._readall() # Make initial seek position to current end of file self._seek = self._size @@ -252,6 +252,9 @@ def readall(self): Returns: bytes: Object content """ + if not self._readable: + raise UnsupportedOperation('read') + with self._seek_lock: # Get data starting from seek with handle_os_exceptions(): @@ -286,6 +289,9 @@ def readinto(self, b): Returns: int: number of bytes read """ + if not self._readable: + raise UnsupportedOperation('read') + # Get and update stream positions size = len(b) with self._seek_lock: diff --git a/pycosio/storage/azure_blob/_append_blob.py b/pycosio/storage/azure_blob/_append_blob.py index 3c58627..36468cb 100644 --- a/pycosio/storage/azure_blob/_append_blob.py +++ b/pycosio/storage/azure_blob/_append_blob.py @@ -18,6 +18,8 @@ class AzureAppendBlobRawIO(AzureBlobRawIO, ObjectRawIORandomWriteBase): """Binary Azure Append Blobs Storage Object I/O + This blob type is not seekable in write mode. + Args: name (path-like object): URL or path to the file which will be opened. mode (str): The mode can be 'r', 'w', 'a' @@ -91,6 +93,8 @@ class AzureAppendBlobBufferedIO(AzureBlobBufferedIO, ObjectBufferedIORandomWriteBase): """Buffered binary Azure Append Blobs Storage Object I/O + This blob type is not seekable in write mode. + Args: name (path-like object): URL or path to the file which will be opened. mode (str): The mode can be 'r', 'w' for reading (default) or writing diff --git a/tests/test_core_io_buffered.py b/tests/test_core_io_buffered.py index 9bba970..042133a 100644 --- a/tests/test_core_io_buffered.py +++ b/tests/test_core_io_buffered.py @@ -1,15 +1,11 @@ # coding=utf-8 """Test pycosio._core.io_buffered""" -import io import os import time -import pytest - def test_object_buffered_base_io(): """Tests pycosio._core.io_buffered.ObjectBufferedIOBase""" - pytest.skip('') from pycosio._core.io_base_raw import ObjectRawIOBase from pycosio._core.io_base_buffered import ObjectBufferedIOBase from pycosio._core.io_random_write import ( @@ -111,23 +107,6 @@ class DummyBufferedIOPartFlush(ObjectBufferedIORandomWriteBase): """Dummy buffered IO with part flush support""" _RAW_CLASS = DummyRawIOPartFlush - # Test raw - object_io = DummyBufferedIO(name) - assert isinstance(object_io.raw, object_io._RAW_CLASS) - assert object_io._size == object_io.raw._size - - assert object_io.raw.tell() == 0 - assert object_io.peek(10) == 10 * b'0' - assert object_io.raw.tell() == 0 - - assert object_io.read1(10) == 10 * b'0' - assert object_io.raw.tell() == 10 - - buffer = bytearray(10) - assert object_io.readinto1(buffer) == 10 - assert bytes(buffer) == 10 * b'0' - assert object_io.raw.tell() == 20 - # Tests: Read until end object_io = DummyBufferedIO(name) assert object_io.read() == size * b'0' @@ -173,11 +152,6 @@ class DummyBufferedIOPartFlush(ObjectBufferedIORandomWriteBase): assert sorted(object_io._read_queue) == list(range( 700, 700 + buffer_size * 5, buffer_size)) - # Tests: flush should do nothing - seek = object_io._seek - object_io.flush() - assert object_io._seek == seek - # Tests: Read buffer size (No copy mode) object_io.seek(0) assert object_io.read(buffer_size) == buffer_size * b'0' @@ -243,31 +217,6 @@ def read_range(*_, **__): assert object_io.write(1000 * b'0') == 1000 flush_sleep = 0 - # Test read in write mode - object_io = DummyBufferedIO(name, mode='w') - with pytest.raises(io.UnsupportedOperation): - object_io.read() - - # Test seek in write mode - object_io = DummyBufferedIO(name, mode='w') - with pytest.raises(io.UnsupportedOperation): - object_io.seek(0) - - # Test write in read mode - object_io = DummyBufferedIO(name) - with pytest.raises(io.UnsupportedOperation): - object_io.write(b'0') - - # Test buffer size - object_io = DummyBufferedIO(name, mode='w') - assert object_io._buffer_size == DummyBufferedIO.DEFAULT_BUFFER_SIZE - object_io = DummyBufferedIO(name, mode='w', buffer_size=1000) - assert object_io._buffer_size == 1000 - object_io = DummyBufferedIO(name, mode='w', buffer_size=1) - assert object_io._buffer_size == DummyBufferedIO.MINIMUM_BUFFER_SIZE - object_io = DummyBufferedIO(name, mode='w', buffer_size=1000000) - assert object_io._buffer_size == DummyBufferedIO.MAXIMUM_BUFFER_SIZE - # Test default implementation with part flush support raw_flushed[:] = b'' content = os.urandom(100) diff --git a/tests/test_storage.py b/tests/test_storage.py index ab5c7e8..05cfe9a 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -193,6 +193,13 @@ def _test_raw_io(self): with _pytest.raises(_UnsupportedOperation): file.seek(0) + # Test: read in write mode is not supported + with _pytest.raises(_UnsupportedOperation): + file.read() + + with _pytest.raises(_UnsupportedOperation): + file.readinto(bytearray(100)) + else: is_seekable = False max_flush_size = 0 @@ -396,6 +403,33 @@ def _test_buffered_io(self): with self._buffered_io(file_path, 'wb', buffer_size=buffer_size, **self._system_parameters) as file: file.write(content) + + # Test: Flush manually + file.flush() + + # Test: read in write mode is not supported + with _pytest.raises(_UnsupportedOperation): + file.read() + + with _pytest.raises(_UnsupportedOperation): + file.read1() + + with _pytest.raises(_UnsupportedOperation): + file.readinto(bytearray(100)) + + with _pytest.raises(_UnsupportedOperation): + file.readinto1(bytearray(100)) + + with _pytest.raises(_UnsupportedOperation): + file.peek() + + # Test: Unsupported if not seekable + if not file.seekable(): + with _pytest.raises(_UnsupportedOperation): + file.tell() + + with _pytest.raises(_UnsupportedOperation): + file.seek(0) else: # Create pre-existing file if self._storage_mock: @@ -405,9 +439,28 @@ def _test_buffered_io(self): # Test: Read data, multiple of buffer with self._buffered_io(file_path, 'rb', buffer_size=buffer_size, **self._system_parameters) as file: + + # Test full data read assert content == file.read(),\ 'Buffered read, multiple of buffer size' + # Test: seek + assert file.seek(10) == 10, 'Buffered read, seek' + assert file.tell() == 10, 'Buffered read, tell match seek' + + # Test: peek: + assert content[10:110] == file.peek(100), \ + 'Buffered read, peek content match' + assert file.tell() == 10, \ + 'Buffered read, peek tell match' + + # Test: Cannot write in read mode + with _pytest.raises(_UnsupportedOperation): + file.write(b'0') + + # Test: Flush has no effect in read mode + file.flush() + # Check if pycosio subclass is_pycosio_subclass = isinstance(file, ObjectBufferedIOBase)