diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fe86a22 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +__pycache__/ +.vscode +_other +downs +folder +build +dist +*.spec +*.manifest +*.egg-info \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0026ba4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 agmmnn + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.MD b/README.MD new file mode 100644 index 0000000..b4f3ab2 --- /dev/null +++ b/README.MD @@ -0,0 +1,77 @@ +![screenshot](https://user-images.githubusercontent.com/16024979/134737875-9e9a5daf-d6ed-414b-937f-54e67feb0025.png) +
+ +GitHub release (latest by date) + +PyPI + +Batch downloader for [polyhaven.com](https://polyhaven.com/). You can download hdris, textures and models in all sizes. This project uses Poly Haven's [Public API](https://github.com/Poly-Haven/Public-API). +
+ +# How to Install + +- `pip install polydown` + +# How to Use +``` +$ polydown hdris + +# download all available sizes of all hdris into current folder. +> πŸ”—(polyhaven.com/hdris['all sizes'])=>🏠 +``` +``` +$ polydown + +# download all assets of this asset type to the current folder in all available sizes. +# asset types: "hdris", "textures", "models". +``` +``` +$ polydown textures -c + +# list of category in the given asset type. +``` +``` +$ polydown hdris -f hdris_down -s 2k 4k + +# download all hdris with given sizes into "hdris_down" folder. +# /if there is no such folder it will create it./ +> πŸ”—(polyhaven.com/hdris['2k', '4k'])=>🏠(hdris_down) +``` +``` +$ polydown models -c decorative -f folder -s 1k + +# download all models in the "decorative" category into the "folder". +> πŸ”—(polyhaven.com/models/decorative['all sizes'])=>🏠 + +Downloaded files structure: +``` +![model files structure](https://user-images.githubusercontent.com/16024979/134737874-cc04a42e-5855-4acb-9394-dac08352efee.png) + +# Arguments: + +``` + "hdris, textures, models" +-h, --help show this help message and exit +-f, --folder target download folder. +-c, --category category to download. +-s, --sizes size(s) of downloaded asset files. eg: 1k 2k 4k +-o, --overwrite overwrite if the files already exists. otherwise the current task will be skipped. +-no, --noimgs do not download 'preview, render, thumbnail...' images. +-v, --version show program's version number and exit +``` + +# To-Do +- [ ] Unit Tests +- [ ] Progressbar for current download task(s) +- [ ] Select the file format to download +- [ ] Download a specific asset, "polydown hdris stuttgart_suburbs" + +# Requirements +- Python >3.5 + +## Dependencies +- [requests](https://pypi.org/project/requests/) +- [rich](https://github.com/willmcgugan/rich) + +# License +[MIT](https://github.com/agmmnn/polydown/blob/master/LICENSE) \ No newline at end of file diff --git a/polydown/__init__.py b/polydown/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/polydown/__main__.py b/polydown/__main__.py new file mode 100644 index 0000000..6ce5245 --- /dev/null +++ b/polydown/__main__.py @@ -0,0 +1,80 @@ +import argparse +import datetime +from .cli import polycli + +__version__ = "0.1" + +ap = argparse.ArgumentParser() +ap.add_argument( + "asset_type", + type=str, + nargs="*", + help='"hdris, textures, models"', +) +ap.add_argument( + "-f", + "--folder", + action="store", + type=str, + default="", + help="target download folder.", +) +ap.add_argument( + "-c", + "--category", + nargs="?", + const="", + help="category to download.", +) +ap.add_argument( + "-s", + "--sizes", + nargs="+", + default=[], + help="size(s) of downloaded asset files. eg: 1k 2k 4k", +) +# ap.add_argument( +# "-ff", +# "--file_format", +# action="store", +# type=str, +# help="target download folder.", +# ) +ap.add_argument( + "-o", + "--overwrite", + action="store_true", + default=False, + help="Overwrite if the files already exists. otherwise the current task will be skipped.", +) +ap.add_argument( + "-no", + "--noimgs", + action="store_true", + default=False, + help="Do not download 'preview, render, thumbnail...' images.", +) +ap.add_argument("-v", "--version", action="version", version="%(prog)s v" + __version__) +args = ap.parse_args() + + +def cli(): + if args.asset_type == []: + print(" is required.") + exit(0) + + execution_start_time = datetime.datetime.now() + + try: + polycli(args) + except KeyboardInterrupt: + print("\nKeyboardInterrupt!") + + print("Total runtime: {}".format(datetime.datetime.now() - execution_start_time)) + + +if __name__ == "__main__": + try: + cli() + except KeyboardInterrupt: + print("\nKeyboardInterrupt!") diff --git a/polydown/cli.py b/polydown/cli.py new file mode 100644 index 0000000..16beedb --- /dev/null +++ b/polydown/cli.py @@ -0,0 +1,81 @@ +import os +import requests +import json +from rich import print as rprint + +from .poly import Poly + + +def polycli(args): + # rprint(args, "\n") + asset_type = args.asset_type[0] + folder = args.folder + overwrite = args.overwrite + sizes = args.sizes + category = args.category + noimgs = args.noimgs + s = requests.Session() + + # ->πŸ”’asset type-> + asset_type_list = list(json.loads(s.get("https://api.polyhaven.com/types").content)) + if asset_type not in asset_type_list: + rprint(f"'{asset_type}' is not a valid asset type!") + exit() + + # ->πŸ”’category-> + if category == "": + js = json.loads( + s.get(f"https://api.polyhaven.com/categories/{asset_type}").content + ) + rprint(f"[green]There are {len(js)} available categories for {asset_type}:") + rprint(js) + exit() + elif category != None: + asset_category_list = list( + json.loads( + s.get(f"https://api.polyhaven.com/categories/{asset_type}").content + ) + ) + if category not in asset_category_list: + rprint( + f"[red]{category} is not a valid category.[/red]\nEnter empty '-c' argument to get the category list of the {asset_type}." + ) + exit() + + # ->πŸ”’file_format-> + # if file_format == None: + # pass + # elif asset_type == "hdris" and file_format not in ["exr", "hdr"]: + # rprint(f"[red]{file_format} is not a valid file format for {asset_type}.[/red]") + # exit() + # elif asset_type in ["models", "textures"] and file_format not in [ + # "jpg", + # "png", + # "exr", + # ]: + # rprint(f"[red]{file_format} is not a valid file format for {asset_type}.[/red]") + # exit() + + # ->πŸ”’folder-> + if os.path.exists(folder): + down_folder = folder + "\\" if folder[:-1] != "\\" else "" + else: + down_folder = os.getcwd() + "\\" + folder + "\\" if folder[:-1] != "\\" else "" + try: + if os.path.exists(down_folder) == False: + os.mkdir(down_folder) + print("Folder not found, creating...") + except Exception as e: + rprint("[red]Error: " + str(e)) + exit() + + rprint( + f"\n[cyan]πŸ”—(polyhaven.com/{asset_type}" + + (f"/{category}" if category != None else "") + + ("['all sizes']" if sizes == [] else str(sizes)) + + f")=>🏠" + + (f"({folder})" if not folder == "" else "") + + "\n" + ) + + Poly(asset_type, s, category, down_folder, sizes, overwrite, noimgs) diff --git a/polydown/downloader.py b/polydown/downloader.py new file mode 100644 index 0000000..c13baa8 --- /dev/null +++ b/polydown/downloader.py @@ -0,0 +1,185 @@ +from rich import print as rprint +import os +from .hash_check import hash_check + +# theme +t_skipped_file = "[on dark_khaki]πŸ“β†³[/on dark_khaki][green]" +t_down_file = "[grey11 on cyan]πŸ“β†“[/grey11 on cyan][cyan]" + +t_skipped_img = "[on dark_khaki]πŸ–ΌοΈβ†³[/on dark_khaki][green]" +t_down_img = "[grey11 on cyan]πŸ–ΌοΈβ†“[/grey11 on cyan][cyan]" + +v = " (MD5βœ”)" +x = " [red](MD5❌)" +# /theme + + +class Downloader: + def __init__( + self, + type, + session, + down_folder, + subfolder, + filename, + asset, + k, + url, + md5, + overwrite, + b, + ): + self.type = type + self.s = session + self.down_folder = down_folder + self.subfolder = subfolder + self.filename = filename + self.asset = asset + self.k = k + self.url = url + self.md5 = md5 + self.overwrite = overwrite + self.b = b + self.asset_k_folder = f"{subfolder}\\{asset}_{k}" + self.textures_folder = f"{subfolder}\\{asset}_{k}\\textures" + + if type == "hdris": + self.folder = down_folder + filename + elif type == "models": + self.folder = ( + f"{self.textures_folder}\\{self.filename}" + if not b + else f"{self.asset_k_folder}\\{self.filename}" + ) + else: + self.folder = ( + f"{self.textures_folder}\\{self.filename}" + if not self.b + else f"{self.asset_k_folder}\\{self.filename}" + ) + + if type == "hdris": + self.filelist = [ + entry.name + for entry in os.scandir(path=down_folder) + if entry.is_file() and entry.name.endswith((".hdr", ".exr", ".png")) + ] + else: + self.filelist = ( + [t.name for t in os.scandir(path=self.textures_folder) if t.is_file()] + + [ + bl.name + for bl in os.scandir(path=self.asset_k_folder) + if bl.is_file() + ] + + [pr.name for pr in os.scandir(path=subfolder) if pr.is_file()] + ) + + def file(self): + def save_file(): + r = self.s.get(self.url) + with open(self.folder, "wb") as f: + f.write(r.content) + + if self.filename in self.filelist and self.overwrite == False: + h = hash_check( + self.type, + self.down_folder, + self.subfolder, + self.asset, + self.k, + self.filename, + self.md5, + self.b, + ) + return ( + t_skipped_file + + " Already exist (skipped): " + + self.filename + + (v if h == True else x), + "exist", + h, + ) + elif self.filename in self.filelist and self.overwrite: + save_file() + h = hash_check( + self.type, + self.down_folder, + self.subfolder, + self.asset, + self.k, + self.filename, + self.md5, + self.b, + ) + return ( + t_down_file + + " Already exist (overwritten): " + + self.filename + + (v if h == True else x), + "downloaded", + h, + ) + else: + save_file() + h = hash_check( + self.type, + self.down_folder, + self.subfolder, + self.asset, + self.k, + self.filename, + self.md5, + self.b, + ) + return ( + t_down_file + + " Download complete: " + + self.filename + + (v if h == True else x), + "downloaded", + h, + ) + + def img(self): + if self.type == "hdris": + imgs_dict = { + "thumb": f"https://cdn.polyhaven.com/asset_img/thumbs/{self.asset}.png", + "primary": f"https://cdn.polyhaven.com/asset_img/primary/{self.asset}.png", + "renders_lone_monk": f"https://cdn.polyhaven.com/asset_img/renders/{self.asset}/lone_monk.png", + } + elif self.type == "textures": + imgs_dict = { + "primary": f"https://cdn.polyhaven.com/asset_img/primary/{self.asset}.png", + "thumb": f"https://cdn.polyhaven.com/asset_img/thumbs/{self.asset}.png", + "renders_clay": f"https://cdn.polyhaven.com/asset_img/renders/{self.asset}/clay.png", + } + elif self.type == "models": + imgs_dict = { + "primary": f"https://cdn.polyhaven.com/asset_img/primary/{self.asset}.png", + "renders_clay": f"https://cdn.polyhaven.com/asset_img/renders/{self.asset}/clay.png", + "renders_orth_front": f"https://cdn.polyhaven.com/asset_img/renders/{self.asset}/orth_front.png", + "renders_orth_side": f"https://cdn.polyhaven.com/asset_img/renders/{self.asset}/orth_side.png", + "renders_orth_top": f"https://cdn.polyhaven.com/asset_img/renders/{self.asset}/orth_top.png", + } + + def save_file(url, filename): + r = self.s.get(url) + with open( + f"{self.subfolder}\\{filename}" + if self.type != "hdris" + else f"{self.down_folder}\\{filename}", + "wb", + ) as f: + f.write(r.content) + + for i in imgs_dict: + filename = f"{self.asset}_{i}.png" + if filename in self.filelist and self.overwrite == False: + rprint(t_skipped_img + "Already exist (skipped): " + filename) + elif filename in self.filelist and self.overwrite: + save_file(imgs_dict[i], filename) + rprint(t_down_img + "Already exist (overwritten): " + filename) + else: + save_file(imgs_dict[i], filename) + rprint(t_down_img + "Download complete: " + filename) diff --git a/polydown/hash_check.py b/polydown/hash_check.py new file mode 100644 index 0000000..b0ab627 --- /dev/null +++ b/polydown/hash_check.py @@ -0,0 +1,21 @@ +import hashlib + + +def hash_check(type, down_folder, subfolder, asset, k, filename, hash, b): + if type == "hdris": + file = down_folder + filename + else: + file = ( + f"{subfolder}\\{asset}_{k}\\textures\\{filename}" + if not b + else f"{subfolder}\\{asset}_{k}\\{filename}" + ) + + with open(file, "rb") as f: + file_hash = hashlib.md5() + while chunk := f.read(8192): + file_hash.update(chunk) + + if hash != file_hash.hexdigest(): + return False + return True diff --git a/polydown/poly.py b/polydown/poly.py new file mode 100644 index 0000000..056f4ea --- /dev/null +++ b/polydown/poly.py @@ -0,0 +1,151 @@ +import os, json +from rich import print as rprint + +from .report import Report +from .downloader import Downloader + +# themes +t_skipped_file = "[on dark_khaki]πŸ“β†³[/on dark_khaki][green]" +t_down_file = "[grey11 on cyan]πŸ“β†“[/grey11 on cyan][cyan]" + +t_skipped_img = "[on dark_khaki]πŸ–ΌοΈβ†³[/on dark_khaki][green]" +t_down_img = "[grey11 on cyan]πŸ–ΌοΈβ†“[/grey11 on cyan][cyan]" +# /themes + + +class Poly: + def __init__(self, type, session, category, down_folder, sizes, overwrite, noimgs): + self.s = session + self.type = type + self.asset_url = f"https://api.polyhaven.com/assets?t={type}" + if category != None: + self.asset_url = f"https://api.polyhaven.com/assets?t={type}&c={category}" + self.asset_list = [i for i in json.loads(self.s.get(self.asset_url).content)] + + self.down_folder = down_folder + self.down_sizes = sizes + self.overwrite = overwrite + self.noimgs = noimgs + + self.corrupted_files = [] + self.exist_files = 0 + self.downloaded_files = 0 + + self.report = Report() + if type == "textures" or type == "models": + self.main() + else: + self.hdris() + self.report.show_report(self.overwrite, self.corrupted_files) + + def main(self): + count = 0 + for asset in self.asset_list: + files_url = "https://api.polyhaven.com/files/" + asset + file_js = json.loads(self.s.get(files_url).content) + k_list = [i for i in file_js["blend"]] + k_list.sort(key=lambda fname: int(fname.split("k")[0])) + include = file_js["blend"]["1k"]["blend"]["include"] + + def create_subfolder(k): + # downfolder>ArmChair_01>ArmChair_01_1k>textures + self.subfolder = self.down_folder + asset + if not os.path.exists(self.subfolder): + os.mkdir(self.subfolder) + if not os.path.exists(self.subfolder + f"\\{asset}_{k}"): + os.mkdir(self.subfolder + f"\\{asset}_{k}") + os.mkdir(self.subfolder + f"\\{asset}_{k}\\textures") + + rprint("[grey50]" + asset + ":") + for k in k_list if self.down_sizes == [] else self.down_sizes: + if k in k_list: + # download blend file + create_subfolder(k) + bl_url = file_js["blend"][k]["blend"]["url"] + bl_md5 = file_js["blend"][k]["blend"]["md5"] + filename = bl_url.split("/")[-1] + dw = Downloader( + self.type, + self.s, + self.down_folder, + self.subfolder, + filename, + asset, + k, + bl_url, + bl_md5, + self.overwrite, + True, + ) + d = dw.file() + rprint(d[0]) + self.report.add(d[1]) + if d[2] == False: + self.corrupted_files.append(filename) + # download texture files + for i in include: + url = include[i]["url"] + md5 = include[i]["md5"] + filename = url.split("/")[-1] + dw = Downloader( + self.type, + self.s, + self.down_folder, + self.subfolder, + filename, + asset, + k, + url, + md5, + self.overwrite, + False, + ) + d = dw.file() + rprint(d[0]) + self.report.add(d[1]) + if d[2] == False: + self.corrupted_files.append(filename) + + if self.noimgs != True: + dw.img() + # count += 1 + # if count == 4: + # break + + def hdris(self): + count = 0 + for asset in self.asset_list: + files_url = "https://api.polyhaven.com/files/" + asset + file_js = json.loads(self.s.get(files_url).content) + file_sizes_list = [i for i in file_js["hdri"]] + file_sizes_list.sort(key=lambda fname: int(fname.split("k")[0])) + rprint("[grey50]" + asset + ":") + + for k in file_sizes_list if self.down_sizes == [] else self.down_sizes: + url = file_js["hdri"][k]["hdr"]["url"] + md5 = file_js["hdri"][k]["hdr"]["md5"] + filename = url.split("/")[-1] + dw = Downloader( + self.type, + self.s, + self.down_folder, + "no", + filename, + asset, + k, + url, + md5, + self.overwrite, + False, + ) + d = dw.file() + rprint(d[0]) + self.report.add(d[1]) + if d[2] == False: + self.corrupted_files.append(filename) + + if self.noimgs != True: + dw.img() + # count += 1 + # if count == 2: + # break diff --git a/polydown/report.py b/polydown/report.py new file mode 100644 index 0000000..5bec341 --- /dev/null +++ b/polydown/report.py @@ -0,0 +1,35 @@ +from rich import print as rprint + +# themes +t_tick = "[on dark_khaki]πŸ“βœ“[/on dark_khaki][green]" +t_tick_d = "[on cyan]πŸ“βœ“[/on cyan][cyan]" +t_cross = "[on red]πŸ“Γ—[/on red][red]" +# /themes + + +class Report: + def __init__(self): + self.exist_files = 0 + self.downloaded_files = 0 + + def add(self, type): + if type == "exist": + self.exist_files += 1 + elif type == "downloaded": + self.downloaded_files += 1 + + def show_report(self, overwrite, corrupted_files): + print() + if overwrite: + rprint( + f"{t_tick_d}> {self.exist_files} files already exist, downloaded and overwritten." + ) + else: + rprint(f"{t_tick}> {self.exist_files} files already exist, skipped.") + rprint(f"{t_tick_d}> {self.downloaded_files} files downloaded.") + if corrupted_files != []: + rprint( + f"{t_cross}> {len(corrupted_files)} files failed at the MD5 test:[/red]" + ) + for i in corrupted_files: + rprint(f"\t[red]{i}[/red]") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..195624e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests +rich \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..164beca --- /dev/null +++ b/setup.py @@ -0,0 +1,35 @@ +from setuptools import setup +import polydown.__main__ as m + +with open("README.md", "r", encoding="UTF8") as fh: + long_description = fh.read() + +with open("requirements.txt", "r") as file: + requires = [line.strip() for line in file.readlines()] + +VERSION = m.__version__ +DESCRIPTION = "Batch downloader for polyhaven (polyhaven.com)." + +setup( + name="polydown", + version=VERSION, + url="https://github.com/agmmnn/polydown", + description=DESCRIPTION, + long_description=long_description, + long_description_content_type="text/markdown", + author="agmmnn", + license="MIT", + classifiers=[ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + "License :: OSI Approved :: MIT License", + "Environment :: Console", + "Topic :: Utilities", + ], + packages=["polydown"], + install_requires=requires, + include_package_data=True, + package_data={"polydown": ["polydown/*"]}, + python_requires=">=3.5", + entry_points={"console_scripts": ["polydown = polydown.__main__:cli"]}, +)