-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #250 from wiwski/feat/stream-zip-files
download share directory and stream back a zip on the fly
- Loading branch information
Showing
12 changed files
with
495 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
""" | ||
Streaming zip files from Azure file shares. | ||
This module provides functions for streaming and zipping files from Azure file shares. | ||
""" | ||
|
||
from typing import Any, Generator | ||
from stat import S_IFREG | ||
|
||
from azure.storage.fileshare._download import StorageStreamDownloader | ||
|
||
from stream_zip import stream_zip, ZIP_AUTO | ||
|
||
|
||
def iter_files_zip_attr(files: Generator[StorageStreamDownloader, Any, None]): | ||
""" | ||
Iterates over a generator of `StorageStreamDownloader` objects and | ||
yields file attributes required for zipping. | ||
Args: | ||
files: A generator of `StorageStreamDownloader` objects. | ||
Yields: | ||
A tuple containing the file attributes required for zipping: | ||
- File name | ||
- Last modified timestamp | ||
- File mode (S_IFREG | 0o600) | ||
- File size (ZIP_AUTO) | ||
- File contents (generator) | ||
""" | ||
|
||
def contents(stream_obj: StorageStreamDownloader): | ||
for chunk in stream_obj.chunks(): | ||
yield chunk | ||
|
||
for stream_obj in files: | ||
yield ( | ||
stream_obj.name, | ||
stream_obj.properties.last_modified, | ||
S_IFREG | 0o600, | ||
ZIP_AUTO(stream_obj.size), | ||
contents(stream_obj), | ||
) | ||
|
||
|
||
def stream_zip_from_azure_files(files: Generator[StorageStreamDownloader, Any, None]): | ||
""" | ||
Streams a zip file from Azure files. | ||
Args: | ||
files: A generator of `StorageStreamDownloader` objects. | ||
Returns: | ||
A streaming zip file. | ||
""" | ||
return stream_zip(iter_files_zip_attr(files)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
import datetime | ||
from typing import Any, Callable, Generator | ||
|
||
def NO_COMPRESSION_32(uncompressed_size, crc_32): ... | ||
def NO_COMPRESSION_64(uncompressed_size, crc_32): ... | ||
def ZIP_32(offset, default_get_compressobj): ... | ||
def ZIP_64(offset, default_get_compressobj): ... | ||
def ZIP_AUTO(uncompressed_size, level: int = ...): ... | ||
def stream_zip( | ||
files: Generator[ | ||
tuple[str, datetime.datetime, int, Callable, Generator[bytes, Any, None]], | ||
Any, | ||
None, | ||
] | ||
| tuple[str, datetime.datetime, int, Callable, Generator[bytes, Any, None]] | ||
| tuple[str, datetime.datetime, int, Callable, bytes], | ||
chunk_size: int = ..., | ||
get_compressobj=..., | ||
extended_timestamps: bool = ..., | ||
): ... | ||
|
||
class ZipError(Exception): ... | ||
class ZipValueError(ZipError, ValueError): ... | ||
class ZipIntegrityError(ZipValueError): ... | ||
class CRC32IntegrityError(ZipIntegrityError): ... | ||
class UncompressedSizeIntegrityError(ZipIntegrityError): ... | ||
class ZipOverflowError(ZipValueError, OverflowError): ... | ||
class UncompressedSizeOverflowError(ZipOverflowError): ... | ||
class CompressedSizeOverflowError(ZipOverflowError): ... | ||
class CentralDirectorySizeOverflowError(ZipOverflowError): ... | ||
class OffsetOverflowError(ZipOverflowError): ... | ||
class CentralDirectoryNumberOfEntriesOverflowError(ZipOverflowError): ... | ||
class NameLengthOverflowError(ZipOverflowError): ... |
Oops, something went wrong.