From b873d1e75f114f3cf58f8df0dba64d0b14af0628 Mon Sep 17 00:00:00 2001 From: aryanvikash Date: Tue, 6 Oct 2020 19:47:18 +0530 Subject: [PATCH] Bump 0.0.4 --- README.md | 96 ++++++++++++++++++++++++++++----------------- example.py | 55 ++++++++++++++++++++++++++ pyaiodl/__init__.py | 51 +++++++++++++++++++++--- pyaiodl/errors.py | 8 ++-- pyaiodl/pyaiodl.py | 85 ++++++++++++++++++++++++++------------- setup.py | 2 +- version.py | 2 +- 7 files changed, 224 insertions(+), 75 deletions(-) create mode 100644 example.py diff --git a/README.md b/README.md index d5c1b01..5b1f386 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ### Don't Use it in Production or Live Projects Currently Its Unstable ___ - [![Python 3.8](https://img.shields.io/badge/python-3.8-blue.svg)](https://www.python.org/downloads/release/python-360/) + [![Python 3.6](https://img.shields.io/badge/python-3-blue.svg)](https://www.python.org/downloads/release/python-360/) [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://github.com/aryanvikash/pyaiodl) [![PyPI license](https://img.shields.io/pypi/l/ansicolortags.svg)](https://github.com/aryanvikash/pyaiodl) [![Open Source Love png3](https://badges.frapsoft.com/os/v3/open-source.png?v=103)](https://github.com/aryanvikash/pyaiodl) @@ -11,11 +11,11 @@ ___ ## Version [![Beta badge](https://img.shields.io/badge/STATUS-BETA-red.svg)](https://github.com/aryanvikash/pyaiodl) -[![PyPI version](https://badge.fury.io/py/pyaiodl.svg)](https://badge.fury.io/py/pyaiodl) +[![PyPI version](https://badge.fury.io/py/pyaiodl.svg)](https://pypi.org/project/pyaiodl/) ## installation -pypi Method (recommanded) +pypi Method (Recommended) pip3 install pyaiodl @@ -24,23 +24,32 @@ Github repo method pip3 install git+https://github.com/aryanvikash/pyaiodl.git -# Avalible Methods +# Available Methods - Downloader class Instance `Non-blocking , params = [fake_useragent:bool,chunk_size:int ,download_path:str] optinals` dl = Downloader() -- Download [ `download(self.url)` ] +- Download [ `download(self,url)` ] uuid = await dl.download(url) +- Errors [` iserror(self, uuid) `] + ` : Returns - Error Or None + , Even On cancel It returns an error "{uuid} Cancelled"` + + ``` + await dl.iserror(uuid) + ``` + + - cancel [ `cancel(self, uuid)` ] - + await dl.cancel(uuid) - Get Status [ `status(self, uuid)` ] response = await dl.status(uuid) - + returns a dict """ @@ -52,12 +61,12 @@ Github repo method downloaded_str :str progress:int download_speed:str - active: bool complete :bool download_path:str + """ -- is_active returns : bool [ `is_active( self,uuid )` ]` on cancel and download complete it will return False` +- is_active returns : bool [ `is_active( self,uuid )` ]` - on cancel ,error , download complete return False` result = await dl.is_active(uuid) @@ -68,56 +77,71 @@ ___ ```py +from pyaiodl import Downloader, errors import asyncio -from pyaiodl import Downloader - url = "https://speed.hetzner.de/100MB.bin" -async def main(url): - dl = Downloader() +async def main(): + dl = Downloader() + # you can pass your + # custom chunk size and Download Path + # dl = Downloader(download_path="/your_dir/", chunk_size=10000) + uuid = await dl.download(url) try: - #Non-blocking - uuid = await dl.download(url) - - - - #progress while await dl.is_active(uuid): - + r = await dl.status(uuid) - #cancel - # await dl.cancel(uuid) - print(f""" - Filename: {r['filename']} - Total : {r['total_size_str']} - Downloaded : {r['downloaded_str']} - Download Speed : {r['download_speed']} - progress: {r['progress']} - """) + #cancel + if r['progress'] > 0: + try: + await dl.cancel("your_uuid") + except errors.DownloadNotActive as na: + print(na) + + + print(f""" + Filename: {r['filename']} + Total : {r['total_size_str']} + Downloaded : {r['downloaded_str']} + Download Speed : {r['download_speed']} + progress: {r['progress']} + """) + + # let him breath for a second:P await asyncio.sleep(1) - + + # If You are putting uuid manually Than its better handle This Exception + except errors.InvalidId: + print("not valid uuid") + return + + # when loop Breaks There are 2 Possibility + # either Its An error Or Download Complete + # Cancelled Is also count as error + if await dl.iserror(uuid): + print(await dl.iserror(uuid)) + + else: # Final filename / path - print( "download completed : ",r['download_path']) + print("Download completed : ", r['download_path']) - except Exception as e: - print(e) +asyncio.get_event_loop().run_until_complete(main()) -asyncio.get_event_loop().run_until_complete(main(url)) ``` ___ ### known Bugs - - - Error is Not Handled Correctly + - None Please Report :) ___ # TODO - Multipart Download - Queue Download / Parallel Downloads Limit -- Better Error Handling +- [x] Better Error Handling diff --git a/example.py b/example.py new file mode 100644 index 0000000..2e6189a --- /dev/null +++ b/example.py @@ -0,0 +1,55 @@ +from pyaiodl import Downloader, errors +import asyncio +url = "https://speed.hetzner.de/100MB.bin" + + +async def main(): + dl = Downloader() + + # or + # dl = Downloader(download_path="/your_dir/", chunk_size=10000) + + uuid = await dl.download(url) + + + try: + while await dl.is_active(uuid): + + r = await dl.status(uuid) + + #cancel + if r['progress'] > 0: + try: + await dl.cancel("your_uuid") + except errors.DownloadNotActive as na: + print(na) + + + print(f""" + Filename: {r['filename']} + Total : {r['total_size_str']} + Downloaded : {r['downloaded_str']} + Download Speed : {r['download_speed']} + progress: {r['progress']} + """) + + # let him breath for a second:P + await asyncio.sleep(1) + + # If You are putting uuid manually Than its better handle This Exception + except errors.InvalidId: + print("not valid uuid") + return + + # when loop Breaks There are 2 Possibility either Its An error Or Download Complete + # Cancelled Is also count as error + if await dl.iserror(uuid): + print(await dl.iserror(uuid)) + + else: + # Final filename / path + print("Download completed : ", r['download_path']) + + +asyncio.get_event_loop().run_until_complete(main()) + diff --git a/pyaiodl/__init__.py b/pyaiodl/__init__.py index 5f0f03a..6d813ab 100644 --- a/pyaiodl/__init__.py +++ b/pyaiodl/__init__.py @@ -13,10 +13,11 @@ from .pyaiodl import PrivateDl +from .errors import DownloadNotActive, InvalidId class Downloader: - def __init__(self, chunk_size=10000, download_path=None): + def __init__(self, chunk_size=None, download_path=None): self._alldownloads = {} self.download_path = download_path # custom chunk size @@ -34,14 +35,52 @@ async def download(self, url): raise Exception(e) return _uuid + # To check If Download Is active async def is_active(self, uuid): - _tempobj = self._alldownloads[uuid]["obj"] - return _tempobj.isActive + try: + _tempobj = self._alldownloads[uuid]["obj"] + except KeyError: + raise InvalidId() + + return not _tempobj.task.done() + # Get Status async def status(self, uuid): - _tempobj = self._alldownloads[uuid]["obj"] + try: + _tempobj = self._alldownloads[uuid]["obj"] + except KeyError: + raise InvalidId() return await _tempobj.getStatus() + #Cancel your Download + async def cancel(self, uuid): - _tempobj = self._alldownloads[uuid]["obj"] - return await _tempobj.cancel(uuid) + try: + _tempobj = self._alldownloads[uuid]["obj"] + + # mark as cancelled + self._alldownloads[uuid]["iscancel"] = True + _tempobj._cancelled = True + + cancelstatus = await _tempobj.cancel(uuid) or False + + except KeyError: + raise InvalidId() + + if _tempobj.task.done(): + + raise DownloadNotActive(f"{uuid} : Download Not active") + + return cancelstatus + + async def iserror(self, uuid): + try: + _tempobj = self._alldownloads[uuid]["obj"] + _iscancel = self._alldownloads[uuid]["iscancel"] + + if _iscancel: + return f"{uuid} Cancelled" + except KeyError: + raise InvalidId() + + return _tempobj.iserror diff --git a/pyaiodl/errors.py b/pyaiodl/errors.py index 550224c..e37ab9a 100644 --- a/pyaiodl/errors.py +++ b/pyaiodl/errors.py @@ -1,7 +1,9 @@ -__all_ = ['download_not_active'] +__all_ = ['DownloadNotActive', 'InvalidId'] +class DownloadNotActive(Exception): + """ Download Not active """ -class download_not_active(Exception): - """ Download Not active """ \ No newline at end of file +class InvalidId(Exception): + """ on Invalid uuid """ diff --git a/pyaiodl/pyaiodl.py b/pyaiodl/pyaiodl.py index 21ba961..1185f9b 100644 --- a/pyaiodl/pyaiodl.py +++ b/pyaiodl/pyaiodl.py @@ -4,9 +4,11 @@ import os import aiohttp from .utils import human_size, gen_uuid, getspeed -from .errors import download_not_active + import aiofiles from time import time +import socket +from contextlib import suppress class PrivateDl: @@ -19,7 +21,7 @@ class PrivateDl: or dl.cancel() """ - def __init__(self, fake_useragent: bool = False, chunk_size: int = 10000, download_path=None): + def __init__(self, fake_useragent: bool = False, chunk_size = None, download_path=None): self.chunk_size = chunk_size self.total_size = 0 self.downloaded = 0 @@ -32,7 +34,6 @@ def __init__(self, fake_useragent: bool = False, chunk_size: int = 10000, downlo self.file_type = None self.session = None # incase if download is cancelled we can check it here - self.isActive = True # both are protected bcz we don't need mutiple value to check status self._cancelled = False @@ -41,13 +42,18 @@ def __init__(self, fake_useragent: bool = False, chunk_size: int = 10000, downlo self.task = None self.fake_useragent = fake_useragent + self.conn = aiohttp.TCPConnector( + family=socket.AF_INET, + verify_ssl=False) + # TODO add retry self.max_tries = 3 self.start_time = 0 # basically i will hold only one uuid ;) self.__toatal_downloads = {} self.real_url = None - + # error goes here + self.iserror = None self.downloadedstr = 0 # 10MiB if fake_useragent: from fake_useragent import UserAgent @@ -63,7 +69,7 @@ async def download(self, url: str) -> str: __uuid = gen_uuid() self.uuid = __uuid self.url = url - __task = asyncio.create_task(self.__down()) + __task = asyncio.ensure_future(self.__down()) self.__toatal_downloads[__uuid] = {} self.__toatal_downloads[__uuid]["obj"] = download_obj @@ -72,7 +78,8 @@ async def download(self, url: str) -> str: return self.uuid except Exception as e: - raise Exception(e) + await self.mark_done(e) + return async def __down(self) -> None: @@ -84,15 +91,15 @@ async def __down(self) -> None: "User-Agent": self.userAgent } self.session = aiohttp.ClientSession( - headers=headers, raise_for_status=True) + headers=headers, raise_for_status=True, connector=self.conn) else: - self.session = aiohttp.ClientSession(raise_for_status=True) + self.session = aiohttp.ClientSession( + raise_for_status=True, connector=self.conn) try: self.filename, self.total_size, self.content_type, self.real_url = await self.__getinfo() except Exception as e: - self.isActive = False - await self.session.close() - raise Exception(e) + await self.mark_done(e) + return try: async with self.session.get(self.url) as r: @@ -102,7 +109,8 @@ async def __down(self) -> None: try: os.makedirs(self.download_path) except Exception as e: - raise Exception(e) + await self.mark_done(e) + return self.download_path = os.path.join( self.download_path, self.filename) @@ -110,17 +118,25 @@ async def __down(self) -> None: self.download_path = self.filename async with aiofiles.open(self.download_path, mode="wb") as f: - # removed iter_chunked(bytes) for max performance - async for chunk in r.content.iter_chunked(self.chunk_size): - await f.write(chunk) - downloaded_chunk += len(chunk) - await self.__updateStatus(downloaded_chunk) + + if self.chunk_size: + async for chunk in r.content.iter_chunked(self.chunk_size): + await f.write(chunk) + downloaded_chunk += len(chunk) + await self.__updateStatus(downloaded_chunk) + else: + async for chunk in r.content.iter_any(): + await f.write(chunk) + downloaded_chunk += len(chunk) + await self.__updateStatus(downloaded_chunk) + except Exception as e: - raise Exception(e) + await self.mark_done(e) + return # session close self._complete = True - self.isActive = False + await self.session.close() async def __updateStatus(self, downloaded_chunks): @@ -140,7 +156,9 @@ async def __getinfo(self) -> tuple: """ get Url Info like filename ,size and filetype """ async with self.session.get( - self.url, allow_redirects=True + self.url, + allow_redirects=True + ) as response: # Use redirected URL @@ -174,9 +192,9 @@ async def getStatus(self) -> dict: downloaded_str :str progress:int download_speed:str - active: bool complete :bool download_path:str + """ return { @@ -188,25 +206,36 @@ async def getStatus(self) -> dict: "downloaded_str": human_size(self.downloaded), "progress": self.progress, "download_speed": self.download_speed, - "active": self.isActive, "complete": self._complete, - "download_path": self.download_path + "download_path": self.download_path, + } + async def mark_done(self, error): + + self.iserror = error + await self.session.close() + + self.task.cancel() + + # supress CanceledError raised by asyncio cancel task + with suppress(asyncio.CancelledError): + await self.task + + async def cancel(self, uuid) -> bool: """ provide uuid returned by download method to cancel it return : bool """ await self.session.close() # check task is active or cancelled + if not self.task.done(): - # return True or False + __task = self.__toatal_downloads[uuid]["task"] __iscancel: bool = __task.cancel() - if __iscancel: - self.isActive = False + return __iscancel else: - - raise download_not_active(f"{uuid} : Download not active") + return True diff --git a/setup.py b/setup.py index 5a2c856..ceccf98 100644 --- a/setup.py +++ b/setup.py @@ -26,5 +26,5 @@ "Operating System :: OS Independent", ], - python_requires='>=3.7', + python_requires='>=3.6', ) diff --git a/version.py b/version.py index 093c92d..5bd3c24 100644 --- a/version.py +++ b/version.py @@ -1 +1 @@ -version__ = '0.0.3' +version__ = '0.0.4'