From a3315a245ed04cf01e7879da2c4c6677f842d3af Mon Sep 17 00:00:00 2001 From: David Youssefi Date: Wed, 24 Jul 2024 13:40:49 +0000 Subject: [PATCH 1/9] fix: add windows compatible lock --- cars/core/cars_logging.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/cars/core/cars_logging.py b/cars/core/cars_logging.py index 4a365b83..2748806a 100644 --- a/cars/core/cars_logging.py +++ b/cars/core/cars_logging.py @@ -25,15 +25,41 @@ and workers """ -import fcntl import logging import logging.config import os +import platform # Standard imports from datetime import datetime from functools import wraps +SYS_PLATFORM = platform.system().lower() +IS_WIN = "windows" == SYS_PLATFORM + +if IS_WIN: + import msvcrt # pylint: disable=E0401 + + def lock(file): + msvcrt.locking( + file.fileno(), msvcrt.LK_LOCK, os.fstat(file.fileno()).st_size + ) + + def unlock(file): + msvcrt.locking( + file.fileno(), msvcrt.LK_UNLCK, os.fstat(file.fileno()).st_size + ) + +else: + import fcntl + + def lock(file): + fcntl.flock(file, fcntl.LOCK_EX) + + def unlock(file): + fcntl.flock(file, fcntl.LOCK_UN) + + PROGRESS = 21 logging.addLevelName(PROGRESS, "PROGRESS") PROFILING_LOG = 15 @@ -90,9 +116,9 @@ def write_log(self, msg) -> None: Write log """ with open(self.log_file, "a", encoding="utf-8") as file: - fcntl.flock(file, fcntl.LOCK_EX) + lock(file) file.write(msg) - fcntl.flock(file, fcntl.LOCK_UN) + unlock(file) def setup_logging( From 51decabe2a46c447932e0a45c36effa11f2ec565 Mon Sep 17 00:00:00 2001 From: David Youssefi Date: Wed, 24 Jul 2024 13:50:12 +0000 Subject: [PATCH 2/9] feat: upgrade shareloc --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 13e62341..1a0bd352 100644 --- a/setup.cfg +++ b/setup.cfg @@ -81,7 +81,7 @@ install_requires = cars-rasterize==0.2.* cars-resample==0.1.* vlsift==0.1.* - shareloc==0.2.1 + shareloc==0.2.2 package_dir = . = cars From bafbf6811ac25d5daacacc54fed58d0b96302117 Mon Sep 17 00:00:00 2001 From: David Youssefi Date: Wed, 24 Jul 2024 13:58:46 +0000 Subject: [PATCH 3/9] fix: spawn multiprocessing if IS_WIN --- .../cluster/mp_cluster/multiprocessing_cluster.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py b/cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py index 67cde9c7..e10520b2 100644 --- a/cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py +++ b/cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py @@ -31,6 +31,7 @@ # Standard imports import multiprocessing as mp import os +import platform import shutil import signal import threading @@ -58,6 +59,9 @@ ) from cars.orchestrator.cluster.mp_cluster.mp_tools import replace_data +SYS_PLATFORM = platform.system().lower() +IS_WIN = "windows" == SYS_PLATFORM + RUN = 0 TERMINATE = 1 @@ -106,7 +110,7 @@ def __init__(self, conf_cluster, out_dir, launch_worker=True): self.factorize_tasks = self.checked_conf_cluster["factorize_tasks"] # Set multiprocessing mode # forkserver is used, to allow OMP to be used in numba - mp_mode = "forkserver" + mp_mode = "spawn" if IS_WIN else "forkserver" self.launch_worker = launch_worker From 6a302ba5ab25aa38561e5339ba4c04ea77c1edc0 Mon Sep 17 00:00:00 2001 From: David Youssefi Date: Wed, 24 Jul 2024 14:06:05 +0000 Subject: [PATCH 4/9] fix: use cpu_count if IS_WIN --- cars/orchestrator/orchestrator.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cars/orchestrator/orchestrator.py b/cars/orchestrator/orchestrator.py index 0d0436f0..6bb42945 100644 --- a/cars/orchestrator/orchestrator.py +++ b/cars/orchestrator/orchestrator.py @@ -22,10 +22,13 @@ this module contains the orchestrator class """ -# Standard imports import collections import logging + +# Standard imports +import multiprocessing import os +import platform import shutil import sys @@ -51,6 +54,9 @@ from cars.orchestrator.registry import replacer_registry, saver_registry from cars.orchestrator.tiles_profiler import TileProfiler +SYS_PLATFORM = platform.system().lower() +IS_WIN = "windows" == SYS_PLATFORM + class Orchestrator: """ @@ -105,7 +111,11 @@ def __init__( "Auto mode is used for orchestator: " "parameters set by user are ignored" ) - available_cpu = len(os.sched_getaffinity(0)) + available_cpu = ( + multiprocessing.cpu_count() + if IS_WIN + else len(os.sched_getaffinity(0)) + ) if available_cpu == 1: logging.warning("Only one CPU detected.") nb_workers = max(1, available_cpu - 1) From 06dcea8300fc2f4d279710ce9856beee7d833c5e Mon Sep 17 00:00:00 2001 From: David Youssefi Date: Wed, 24 Jul 2024 14:17:32 +0000 Subject: [PATCH 5/9] docs: add docstrings --- cars/core/cars_logging.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cars/core/cars_logging.py b/cars/core/cars_logging.py index 2748806a..98c9e432 100644 --- a/cars/core/cars_logging.py +++ b/cars/core/cars_logging.py @@ -41,11 +41,13 @@ import msvcrt # pylint: disable=E0401 def lock(file): + """Lock file for safe writing (Windows version)""" msvcrt.locking( file.fileno(), msvcrt.LK_LOCK, os.fstat(file.fileno()).st_size ) def unlock(file): + """Unlock file for safe writing (Windows version)""" msvcrt.locking( file.fileno(), msvcrt.LK_UNLCK, os.fstat(file.fileno()).st_size ) @@ -54,9 +56,11 @@ def unlock(file): import fcntl def lock(file): + """Lock file for safe writing (Unix version)""" fcntl.flock(file, fcntl.LOCK_EX) def unlock(file): + """Unlock file for safe writing (Unix version)""" fcntl.flock(file, fcntl.LOCK_UN) From 3122fd7dd1ea7ce9055b5d0aa016fa7a58861ec5 Mon Sep 17 00:00:00 2001 From: David Youssefi Date: Wed, 24 Jul 2024 16:25:16 +0000 Subject: [PATCH 6/9] fix: avoid PermissionError --- cars/core/cars_logging.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cars/core/cars_logging.py b/cars/core/cars_logging.py index 98c9e432..6f652225 100644 --- a/cars/core/cars_logging.py +++ b/cars/core/cars_logging.py @@ -42,15 +42,11 @@ def lock(file): """Lock file for safe writing (Windows version)""" - msvcrt.locking( - file.fileno(), msvcrt.LK_LOCK, os.fstat(file.fileno()).st_size - ) + msvcrt.locking(file.fileno(), msvcrt.LK_LOCK, 0) def unlock(file): """Unlock file for safe writing (Windows version)""" - msvcrt.locking( - file.fileno(), msvcrt.LK_UNLCK, os.fstat(file.fileno()).st_size - ) + msvcrt.locking(file.fileno(), msvcrt.LK_UNLCK, 0) else: import fcntl From 2c618df49d8f5f8bd03f7b5492e53e6e62f707a8 Mon Sep 17 00:00:00 2001 From: David Youssefi Date: Wed, 24 Jul 2024 16:52:09 +0000 Subject: [PATCH 7/9] fix: replace symbol by degrees --- .../grid_generation/epipolar_grid_generation.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cars/applications/grid_generation/epipolar_grid_generation.py b/cars/applications/grid_generation/epipolar_grid_generation.py index 38f13c39..3784e852 100644 --- a/cars/applications/grid_generation/epipolar_grid_generation.py +++ b/cars/applications/grid_generation/epipolar_grid_generation.py @@ -176,19 +176,20 @@ def run( ) logging.info( - "Left satellite acquisition angles: Azimuth angle: {:.1f}°, " - "Elevation angle: {:.1f}°".format(left_az, left_elev_angle) + "Left satellite acquisition angles: " + "Azimuth angle: {:.1f} degrees, " + "Elevation angle: {:.1f} degrees°".format(left_az, left_elev_angle) ) logging.info( - "Right satellite acquisition angles: Azimuth angle: {:.1f}°, " - "Elevation angle: {:.1f}°".format(right_az, right_elev_angle) + "Right satellite acquisition angles: " + "Azimuth angle: {:.1f} degrees, " + "Elevation angle: {:.1f} degrees".format(right_az, right_elev_angle) ) logging.info( - "Stereo satellite convergence angle from ground: {:.1f}°".format( - convergence_angle - ) + "Stereo satellite convergence angle from ground: " + "{:.1f} degrees".format(convergence_angle) ) # Generate rectification grids From 68c735c027fda100f29c005ca7d592ac5a7d947e Mon Sep 17 00:00:00 2001 From: David Youssefi Date: Wed, 24 Jul 2024 16:58:39 +0000 Subject: [PATCH 8/9] fix: replace symbol by degrees --- cars/applications/grid_generation/epipolar_grid_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cars/applications/grid_generation/epipolar_grid_generation.py b/cars/applications/grid_generation/epipolar_grid_generation.py index 3784e852..5ef5f4ba 100644 --- a/cars/applications/grid_generation/epipolar_grid_generation.py +++ b/cars/applications/grid_generation/epipolar_grid_generation.py @@ -178,7 +178,7 @@ def run( logging.info( "Left satellite acquisition angles: " "Azimuth angle: {:.1f} degrees, " - "Elevation angle: {:.1f} degrees°".format(left_az, left_elev_angle) + "Elevation angle: {:.1f} degrees".format(left_az, left_elev_angle) ) logging.info( From 31787859e28c9f3b33495c0f140a6fb25c7ed26e Mon Sep 17 00:00:00 2001 From: David Youssefi Date: Thu, 25 Jul 2024 07:26:35 +0000 Subject: [PATCH 9/9] fix: remove taskset if IS_WIN --- .../cluster/mp_cluster/multiprocessing_cluster.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py b/cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py index e10520b2..82397d30 100644 --- a/cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py +++ b/cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py @@ -117,7 +117,10 @@ def __init__(self, conf_cluster, out_dir, launch_worker=True): self.tmp_dir = None # affinity issues caused by numpy - os.system("taskset -p 0xffffffff %d > /dev/null 2>&1" % os.getpid()) + if IS_WIN is False: + os.system( + "taskset -p 0xffffffff %d > /dev/null 2>&1" % os.getpid() + ) if self.launch_worker: # Create wrapper object