From b86edf633289fcddfe0dc14d6b0ef2786c787c7e Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Sat, 19 Jun 2021 21:17:11 +0100 Subject: [PATCH 01/19] Use the MU_LOG_TO_STDOUT envvar to turn on stdout logging and to disable the crash handler Use version-tagged zips for the downloaded packages --- mu/app.py | 3 +- mu/virtual_environment.py | 78 +++++++++++++++++++++------------------ mu/wheels/__init__.py | 66 +++++++++++++++++++++------------ 3 files changed, 87 insertions(+), 60 deletions(-) diff --git a/mu/app.py b/mu/app.py index 39c4b7828..795f4664c 100644 --- a/mu/app.py +++ b/mu/app.py @@ -224,7 +224,6 @@ def setup_logging(): log = logging.getLogger() log.setLevel(logging.DEBUG) log.addHandler(handler) - sys.excepthook = excepthook # Only enable on-screen logging if the MU_LOG_TO_STDOUT env variable is set if "MU_LOG_TO_STDOUT" in os.environ: @@ -232,6 +231,8 @@ def setup_logging(): stdout_handler.setFormatter(formatter) stdout_handler.setLevel(logging.DEBUG) log.addHandler(stdout_handler) + else: + sys.excepthook = excepthook def setup_modes(editor, view): diff --git a/mu/virtual_environment.py b/mu/virtual_environment.py index 72504cba7..3c1b453c4 100644 --- a/mu/virtual_environment.py +++ b/mu/virtual_environment.py @@ -5,7 +5,9 @@ import glob import logging import subprocess +import tempfile import time +import zipfile from PyQt5.QtCore import ( QObject, @@ -228,7 +230,7 @@ def run( result = self.process.run_blocking( self.executable, params, wait_for_s=wait_for_s ) - logger.debug("Process output: %s", result.strip()) + logger.debug("Process output: %s", compact(result.strip())) return result else: if slots.started: @@ -545,8 +547,9 @@ def recreate(self): # # Now reinstall the original user packages # - logger.debug("About to reinstall user packages: %s", user_packages) - self.install_user_packages(user_packages) + if user_packages: + logger.debug("About to reinstall user packages: %s", user_packages) + self.install_user_packages(user_packages) def ensure_and_create(self, emitter=None): """Check whether we have a valid virtual environment in place and, if not, @@ -822,6 +825,28 @@ def install_jupyter_kernel(self): "Unable to install kernel\n%s" % compact(output) ) + def install_from_unpacked_wheels(self, dirpath): + # + # This command should install the baseline packages, picking up the + # precompiled wheels from the wheels path + # + # For dev purposes (where we might not have the wheels) warn where + # the wheels are not already present and download them + # + wheel_filepaths = glob.glob(os.path.join(dirpath, "*.whl")) + if not wheel_filepaths: + raise VirtualEnvironmentCreateError( + "No wheels in %s; try `python -mmu.wheels`" % wheels_dirpath + ) + self.reset_pip() + for wheel in wheel_filepaths: + logger.info( + "About to install from wheel: {}".format( + os.path.basename(wheel) + ) + ) + self.pip.install(wheel, deps=False, index=False) + def install_baseline_packages(self): """ Install all packages needed for non-core activity. @@ -836,40 +861,23 @@ def install_baseline_packages(self): --upgrade is currently used with a thought to upgrade-releases of Mu. """ logger.info("Installing baseline packages.") - logger.info( - "%s %s", - wheels_dirpath, - "exists" if os.path.isdir(wheels_dirpath) else "does not exist", + zipped_wheels_filepath = os.path.join( + wheels_dirpath, "%s.zip" % mu_version ) - # - # This command should install the baseline packages, picking up the - # precompiled wheels from the wheels path - # - # For dev purposes (where we might not have the wheels) warn where - # the wheels are not already present and download them - # - wheel_filepaths = glob.glob(os.path.join(wheels_dirpath, "*.whl")) - if not wheel_filepaths: - logger.warning( - "No wheels found in %s; downloading...", wheels_dirpath - ) - try: - wheels.download(interpreter=self.interpreter, logger=logger) - except wheels.WheelsDownloadError as exc: - raise VirtualEnvironmentCreateError(exc.message) - else: - wheel_filepaths = glob.glob( - os.path.join(wheels_dirpath, "*.whl") - ) + logger.info("Expecting zipped wheels at %s", zipped_wheels_filepath) + if not os.path.exists(zipped_wheels_filepath): + logger.warning("No zipped wheels found; downloading...") + wheels.download(zipped_wheels_filepath) - if not wheel_filepaths: - raise VirtualEnvironmentCreateError( - "No wheels in %s; try `python -mmu.wheels`" % wheels_dirpath - ) - self.reset_pip() - for wheel in wheel_filepaths: - logger.info("About to install from wheels: {}".format(wheel)) - self.pip.install(wheel, deps=False, index=False) + with tempfile.TemporaryDirectory() as unpacked_wheels_dirpath: + # + # The wheel files are shipped in Mu-version-specific zip files to avoid + # cross-contamination when a Mu version change happens and we still have + # wheels from the previous installation. + # + with zipfile.ZipFile(zipped_wheels_filepath) as zip: + zip.extractall(unpacked_wheels_dirpath) + self.install_from_unpacked_wheels(unpacked_wheels_dirpath) def register_baseline_packages(self): """ diff --git a/mu/wheels/__init__.py b/mu/wheels/__init__.py index cc48888be..d941770a0 100644 --- a/mu/wheels/__init__.py +++ b/mu/wheels/__init__.py @@ -6,7 +6,12 @@ import glob import logging import platform +import shutil import subprocess +import tempfile +import zipfile + +from .. import __version__ as mu_version class WheelsError(Exception): @@ -37,6 +42,7 @@ class WheelsBuildError(WheelsError): ("esptool", "esptool==3.*"), ] WHEELS_DIRPATH = os.path.dirname(__file__) +zip_filepath = os.path.join(WHEELS_DIRPATH, mu_version + ".zip") # TODO: Temp app signing workaround https://github.com/mu-editor/mu/issues/1290 if sys.version_info[:2] == (3, 8) and platform.system() == "Darwin": @@ -62,7 +68,7 @@ def compact(text): return "\n".join(line for line in text.splitlines() if line.strip()) -def remove_dist_files(dirpath): +def remove_dist_files(dirpath, logger): # # Clear the directory of any existing wheels and source distributions # (this is especially important because there may have been wheels @@ -79,28 +85,7 @@ def remove_dist_files(dirpath): os.remove(rm_filepath) -def download( - dirpath=WHEELS_DIRPATH, interpreter=sys.executable, logger=logger -): - # - # We allow the interpreter to be overridden so that the newly-created - # virtual environment can pass in its upgraded pip. This solves some issues - # on Linux with recent wheels - # - - # - # We allow the logger to be overridden because output from the virtual_environment - # module logger goes to the splash screen, while output from this module's - # logger doesn't - # - - # - # Download the wheels needed for modes - # - logger.info("Downloading wheels to %s", dirpath) - - remove_dist_files(dirpath) - +def pip_download(dirpath, logger): for name, pip_identifier, *extra_flags in mode_packages: logger.info( "Running pip download for %s / %s / %s", @@ -127,13 +112,15 @@ def download( # # If any wheel fails to download, remove any files already downloaded # (to ensure the download is triggered again) and raise an exception + # NB Probably not necessary now that we're using a temp directory # if process.returncode != 0: - remove_dist_files(dirpath) raise WheelsDownloadError( "Pip was unable to download %s" % pip_identifier ) + +def convert_sdists_to_wheels(dirpath, logger): # # Convert any sdists to wheels # @@ -159,3 +146,34 @@ def download( logger.warning("Unable to build a wheel for %s", filepath) else: os.remove(filepath) + + +def zip_wheels(zip_filepath, dirpath, logger=logger): + """Zip all the wheels into an archive""" + logger.info("Building zip %s of wheels", zip_filepath) + with zipfile.ZipFile(zip_filepath, "w") as z: + for filepath in glob.glob(os.path.join(dirpath, "*.whl")): + filename = os.path.basename(filepath) + logger.debug("Adding %s to zip", filename) + z.write(filepath, filename) + + +def download(zip_filepath=zip_filepath, logger=logger): + # + # We allow the logger to be overridden because output from the virtual_environment + # module logger goes to the splash screen, while output from this module's + # logger doesn't + # + + # + # Download the wheels needed for modes + # + logger.info("Downloading wheels to %s", zip_filepath) + + dirpath = os.path.dirname(zip_filepath) + remove_dist_files(dirpath, logger) + + with tempfile.TemporaryDirectory() as temp_dirpath: + pip_download(temp_dirpath, logger) + convert_sdists_to_wheels(temp_dirpath, logger) + zip_wheels(zip_filepath, temp_dirpath, logger) From 8d16baa16533022f35a1ba761243555038aa9185 Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Sun, 20 Jun 2021 08:45:36 +0100 Subject: [PATCH 02/19] Refactor slightly partly to support easier testing --- mu/virtual_environment.py | 51 ++++++++----------- mu/wheels/__init__.py | 31 ++++++----- .../test_virtual_environment.py | 6 +-- 3 files changed, 42 insertions(+), 46 deletions(-) diff --git a/mu/virtual_environment.py b/mu/virtual_environment.py index 3c1b453c4..d3184e7f9 100644 --- a/mu/virtual_environment.py +++ b/mu/virtual_environment.py @@ -825,27 +825,24 @@ def install_jupyter_kernel(self): "Unable to install kernel\n%s" % compact(output) ) - def install_from_unpacked_wheels(self, dirpath): - # - # This command should install the baseline packages, picking up the - # precompiled wheels from the wheels path - # - # For dev purposes (where we might not have the wheels) warn where - # the wheels are not already present and download them - # - wheel_filepaths = glob.glob(os.path.join(dirpath, "*.whl")) - if not wheel_filepaths: - raise VirtualEnvironmentCreateError( - "No wheels in %s; try `python -mmu.wheels`" % wheels_dirpath - ) - self.reset_pip() - for wheel in wheel_filepaths: - logger.info( - "About to install from wheel: {}".format( - os.path.basename(wheel) + def install_from_zipped_wheels(self, zipped_wheels_filepath): + with tempfile.TemporaryDirectory() as unpacked_wheels_dirpath: + # + # The wheel files are shipped in Mu-version-specific zip files to avoid + # cross-contamination when a Mu version change happens and we still have + # wheels from the previous installation. + # + with zipfile.ZipFile(zipped_wheels_filepath) as zip: + zip.extractall(unpacked_wheels_dirpath) + + self.reset_pip() + for wheel in unpacked_wheels_dirpath: + logger.info( + "About to install from wheel: {}".format( + os.path.basename(wheel) + ) ) - ) - self.pip.install(wheel, deps=False, index=False) + self.pip.install(wheel, deps=False, index=False) def install_baseline_packages(self): """ @@ -861,23 +858,19 @@ def install_baseline_packages(self): --upgrade is currently used with a thought to upgrade-releases of Mu. """ logger.info("Installing baseline packages.") + # + # TODO: Add semver check to ensure filepath is safe + # zipped_wheels_filepath = os.path.join( wheels_dirpath, "%s.zip" % mu_version ) + print("Zipped wheels filepath:", zipped_wheels_filepath) logger.info("Expecting zipped wheels at %s", zipped_wheels_filepath) if not os.path.exists(zipped_wheels_filepath): logger.warning("No zipped wheels found; downloading...") wheels.download(zipped_wheels_filepath) - with tempfile.TemporaryDirectory() as unpacked_wheels_dirpath: - # - # The wheel files are shipped in Mu-version-specific zip files to avoid - # cross-contamination when a Mu version change happens and we still have - # wheels from the previous installation. - # - with zipfile.ZipFile(zipped_wheels_filepath) as zip: - zip.extractall(unpacked_wheels_dirpath) - self.install_from_unpacked_wheels(unpacked_wheels_dirpath) + self.install_from_zipped_wheels(zipped_wheels_filepath) def register_baseline_packages(self): """ diff --git a/mu/wheels/__init__.py b/mu/wheels/__init__.py index d941770a0..6903a5b71 100644 --- a/mu/wheels/__init__.py +++ b/mu/wheels/__init__.py @@ -6,7 +6,6 @@ import glob import logging import platform -import shutil import subprocess import tempfile import zipfile @@ -42,7 +41,7 @@ class WheelsBuildError(WheelsError): ("esptool", "esptool==3.*"), ] WHEELS_DIRPATH = os.path.dirname(__file__) -zip_filepath = os.path.join(WHEELS_DIRPATH, mu_version + ".zip") +ZIP_FILEPATH = os.path.join(WHEELS_DIRPATH, mu_version + ".zip") # TODO: Temp app signing workaround https://github.com/mu-editor/mu/issues/1290 if sys.version_info[:2] == (3, 8) and platform.system() == "Darwin": @@ -150,7 +149,7 @@ def convert_sdists_to_wheels(dirpath, logger): def zip_wheels(zip_filepath, dirpath, logger=logger): """Zip all the wheels into an archive""" - logger.info("Building zip %s of wheels", zip_filepath) + logger.info("Building zip %s from wheels in %s", zip_filepath, dirpath) with zipfile.ZipFile(zip_filepath, "w") as z: for filepath in glob.glob(os.path.join(dirpath, "*.whl")): filename = os.path.basename(filepath) @@ -158,20 +157,24 @@ def zip_wheels(zip_filepath, dirpath, logger=logger): z.write(filepath, filename) -def download(zip_filepath=zip_filepath, logger=logger): - # - # We allow the logger to be overridden because output from the virtual_environment - # module logger goes to the splash screen, while output from this module's - # logger doesn't - # +def download(zip_filepath=ZIP_FILEPATH, logger=logger): + """Download from PyPI, convert to wheels, and zip up - # - # Download the wheels needed for modes - # + To make all the libraries available for Mu modes (eg pygame zero, Flask etc.) + we `pip download` them to a temporary location and then pack them into a + zip file which is tagged with the current Mu version number + + We allow `logger` to be overridden because output from the + virtual_environment module logger goes to the splash screen, while + output from this module's logger doesn't + """ logger.info("Downloading wheels to %s", zip_filepath) - dirpath = os.path.dirname(zip_filepath) - remove_dist_files(dirpath, logger) + # + # Remove any leftover files from the place where the zip file + # will be -- usually the `wheels` package folder + # + remove_dist_files(os.path.dirname(zip_filepath), logger) with tempfile.TemporaryDirectory() as temp_dirpath: pip_download(temp_dirpath, logger) diff --git a/tests/virtual_environment/test_virtual_environment.py b/tests/virtual_environment/test_virtual_environment.py index 81add1eec..1fe1d18e4 100644 --- a/tests/virtual_environment/test_virtual_environment.py +++ b/tests/virtual_environment/test_virtual_environment.py @@ -263,13 +263,13 @@ def test_download_wheels_if_not_present(venv, test_wheels): ensure we try to download them """ wheels_dirpath = test_wheels - for filepath in glob.glob(os.path.join(wheels_dirpath, "*.whl")): + for filepath in glob.glob(os.path.join(wheels_dirpath, "*.zip")): os.unlink(filepath) - assert not glob.glob(os.path.join(wheels_dirpath, "*.whl")) + assert not glob.glob(os.path.join(wheels_dirpath, "*.zip")) with mock.patch.object( mu.virtual_environment, "wheels_dirpath", wheels_dirpath - ), mock.patch.object(mu.wheels, "download") as mock_download: + ), mock.patch.object(mu.wheels, "download") as mock_download, mock.patch.object(venv, "install_from_zipped_wheels"): try: venv.install_baseline_packages() # From 4fde70682a9cfc623ebe3600c53b7acf45e4f3ad Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Sun, 20 Jun 2021 08:51:06 +0100 Subject: [PATCH 03/19] Test the right exception if the download fails --- tests/virtual_environment/test_virtual_environment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/virtual_environment/test_virtual_environment.py b/tests/virtual_environment/test_virtual_environment.py index 1fe1d18e4..b26543e9f 100644 --- a/tests/virtual_environment/test_virtual_environment.py +++ b/tests/virtual_environment/test_virtual_environment.py @@ -287,9 +287,9 @@ def test_download_wheels_failure(venv, test_wheels): with the same message""" message = uuid.uuid1().hex wheels_dirpath = test_wheels - for filepath in glob.glob(os.path.join(wheels_dirpath, "*.whl")): + for filepath in glob.glob(os.path.join(wheels_dirpath, "*.zip")): os.unlink(filepath) - assert not glob.glob(os.path.join(wheels_dirpath, "*.whl")) + assert not glob.glob(os.path.join(wheels_dirpath, "*.zip")) with mock.patch.object( mu.wheels, "download", @@ -297,7 +297,7 @@ def test_download_wheels_failure(venv, test_wheels): ): try: venv.install_baseline_packages() - except VEError as exc: + except mu.wheels.WheelsDownloadError as exc: assert message in exc.message From a3bfae7483be223be9eded0360b1a7bb84f849b1 Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Sun, 20 Jun 2021 09:10:02 +0100 Subject: [PATCH 04/19] Pushing latest changes although incomplete -- Still tests failing and some coverage missing after refactor --- mu/virtual_environment.py | 1 - .../test_virtual_environment.py | 12 +++++++++--- tests/virtual_environment/wheels/wheels.zip | Bin 0 -> 6244 bytes 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 tests/virtual_environment/wheels/wheels.zip diff --git a/mu/virtual_environment.py b/mu/virtual_environment.py index d3184e7f9..bccbe103d 100644 --- a/mu/virtual_environment.py +++ b/mu/virtual_environment.py @@ -2,7 +2,6 @@ import sys from collections import namedtuple import functools -import glob import logging import subprocess import tempfile diff --git a/tests/virtual_environment/test_virtual_environment.py b/tests/virtual_environment/test_virtual_environment.py index b26543e9f..b38242be4 100644 --- a/tests/virtual_environment/test_virtual_environment.py +++ b/tests/virtual_environment/test_virtual_environment.py @@ -37,6 +37,7 @@ PIP = mu.virtual_environment.Pip HERE = os.path.dirname(__file__) +ZIP_FILENAME = "wheels.zip" WHEEL_FILENAME = "arrr-1.0.2-py3-none-any.whl" @@ -124,10 +125,11 @@ def workspace_dirpath(tmp_path): @pytest.fixture def test_wheels(tmp_path): wheels_dirpath = str(tmp_path / "wheels") + zip_filename = "%s.zip" % mu.__version__ os.mkdir(wheels_dirpath) shutil.copyfile( - os.path.join(HERE, "wheels", WHEEL_FILENAME), - os.path.join(wheels_dirpath, WHEEL_FILENAME), + os.path.join(HERE, "wheels", ZIP_FILENAME), + os.path.join(wheels_dirpath, zip_filename), ) with mock.patch.object( mu.virtual_environment, "wheels_dirpath", wheels_dirpath @@ -269,7 +271,11 @@ def test_download_wheels_if_not_present(venv, test_wheels): with mock.patch.object( mu.virtual_environment, "wheels_dirpath", wheels_dirpath - ), mock.patch.object(mu.wheels, "download") as mock_download, mock.patch.object(venv, "install_from_zipped_wheels"): + ), mock.patch.object( + mu.wheels, "download" + ) as mock_download, mock.patch.object( + venv, "install_from_zipped_wheels" + ): try: venv.install_baseline_packages() # diff --git a/tests/virtual_environment/wheels/wheels.zip b/tests/virtual_environment/wheels/wheels.zip new file mode 100644 index 0000000000000000000000000000000000000000..329a49059a465e01796eb44fa53177a61f61bf7b GIT binary patch literal 6244 zcmaKxRZtuZkgaid8{C2h2@D=IxVsMS?hxGF-7UB$z~C}yAh=s_g1a;L{rA4__NnSq zr>oz(-#%3Zcmx6%7#I|o(BgS@7NgPo2>=X?6e$c0*?&`0H#avnPIeAy}lMlO>jYC4VrE0!?S|qZy3nmt7KQa|J0;Wj2AgXEd_5~4ns00d~4Ka0l7b4ZV zAwxe~JMv0?y>l;^vFq>*qV-m~RRei`x?Weg->}0W(8Pm*2}Tv;+R~7q)-E>)ff0k_ zgbBVry=1YrwpNz;!Y51?b$OF?Aw5Qx4$l!4jplzdJYG|R5$JL0T>Fukh|$Isw}^zzaX1Id9(oa2#|Wbs zV2@{NtLV1rkYIM?FKVi)9h9H>YCn3WIeej$P&LZhD1J*liPfX6{Hp9@N9faIX2@z( zv3G9kH&ZQrlPtmftEB}LxI1a<`?I|k50}Mx=L}RU#Z8$xWftp0DH9K?^2Mrs%Kxmt zu4g}Qm(N>XG}4HcjrBy|lYeAtv71DDBgf1=3dKPN=RO{jRAvlAI&7}5%o@|VT0YX- za&Z^nMMTgl{ke6v%1%DLE5K{lO8{7x5~FY^ie5GZOmLqDm<_dJ+YNq$K-xnEnSOiX z8+G`q1Y|z?9z;{uSAVXC4@<~b+2o*&vv6)B=AW>aGw73dnO)-`S(#?ua1m?8p-KA% zD28kFpv4CLoRb$t9ro`z9&%o!hXgbci@IOa>8l%0g6s*WH+COBsF(1?^QkFUMZRY~ z;VXatN3$SX9@spR*m9i8zUI3}7aXOIm)q5#E-#zo}c~5oyu@wC@KtQC3DNgQ7UxbW_wCo&P#q1g+ zOHw47DUh|XWk%`MZ#tmIe$kurFwK%sVSc@I6d(^x8yOA}c9q4b+-+ocOc87Yn z{nX#()ALoreZoso$Usuk-wCPSjIQSa)LhA#G^WW)^A*Re6FaBx3}t>g?|N@nHm&h& z^w^s)r6%G@&gqK;{E?>ZE82M0#G7U4U42Tg$$Ib@0J1w}OgtT=kHiaAT8^QxVBoQ}kD?3JY%ZnBzU*}JBw z=o@+eqQ2c9KE!_G>M_AYLX|VoIzUj!VwWl|SdL^H5ZXQq5tV+3H`tHGajYLy_dIH8 z4|*G3u_^`kUaIz&fjtovD*-$`_E`PsVq&!o#*cm_o{cosxd2e&DM z`o)kWb*Fbz>O^_`toorq&+S31oj1%jrT6nQf>l5xddIDS^^9)ByHGuV;O@FfyFka& z_FE)d`l1W$0Xwp?3BPW#5({TCq;;2h=Qq5~0vF$R@|>Qp|1jn**!^e}*{oC0pdhG8F{h7l)e&a)LdCx%h7iUH!A;g-O^W~DQwzgKyk{A?;C z!ei>7hIdGgQK1PxzQ&9>PC#X3m7GR*w^@iCW}{QXs`#%{$WHsxwN4{}K0Kva4@gVb zuwsV#2&gih31d7g1_fcctc;D%x&=wv32QJD(b) zd_=}HS4UcU)YsyMXP8&w%SVSiAz0}OEfSnx9@EW>jDDmGtH1p{@SKb{*+cLc92@)a z3HK{1`HHyXN=^+K8+$7my&vNk@H?=;Xnu5 zg|Uc05!hJ=)&;(tnpY|&om!KjVIZBUoZob{FB8j4SVzm3^(yaD&_2xMCatw>ZKk5H z4RF-}{Yol0zL8uuYebq+=&PfpQKu+>p1LpdBW0ZXmP)PRtuB{Y#U1yjQepl#wjlwA zxxleDsWA(iwmqv@;b-N@$Qw+5tN7ZVws-t-{6VjtSn=Jwk_v-5_`sTQTQ@2Y}kY}i;x!)XkE5>sJh;ur~U(0^_6*@_=Wv17kY`s^7vOh>e zUjT`w*s>OMNCKyfoh3AsLkRqPe=T^&B>Ysf79z^iswi7>Qlq%9#QACewam#DxyMe& zz#x#X;MT21aoyH2t(=3+Z7u+``A5LgYTDA;lGcJWn2?C34^&1wd@h8t=~vRTK>*U_ zm{g;Hpv(usVGfR0X0@028DX;v;a6Y%qAxkWC1h~UyV+2e?nqPo9qT5h)5_PSbBo%@ z%gF?QHep!S5jEOYIO6*Fqz2zeA(@7Z%R8=l)=JE6n?TkKfFBkTHU?Kukp*n0o5gL#BUWb>zKV&-m6U(KoIc;BG+AuqQll1pd6-dC&8# zVXUaj{TPLAxOo!RdVo=@cpWk-y#QXEBJ15v?*N(>x;;tl~ z^En2|XOh+ESpF6~sXLh>-W`3}N?aj?R4KAjcgP9I%<&rdAqUgT1m)AQmWvz=v=sSG z7#Wars0ts6c;~05CBFndQdp~*nvXpHxI22Vpy`;SyRgG0bH*<@uK&{n``hT(FY0k- z^QLvNiyiNqJx>!QW}*U_zCywT`Q+yn;40->>?>(N=W?jn!mg4@cPo^@TG9oap?{t~ zVzoNPr(DW7&3Q`Ybm44*4?m2@@2TX6s}J6i$B?-?DL?aFBMbHj7PFG$X2;Jp%5hI`gqEfR`Gy zVr*ME+6Z_RBwGo8F+H%Q?lOXAfIC{TL10-Lj{P5re>L&1MrG6r1d!r5M}ou6tzb-k~aw z99#NUwXbPx3wI9hi$y!7sOy~6=Y{-f(Bi$Egq$oN7lasb)5UFP_I|}seR(v|M%mE# zCwMVo3C-aAKsyP-v?3SEU(fH(KUiY_Nxt+zgk_NW7j#oYnp+tr|LLt1?d+?J`O1}A zI~+1unWlXmRk)_qe2Vq%ugxJzBq)oa${g7!JPC}6#2a#F>Hs@!c$ERLe+gjmc^UC(;!Zk7}NW@ z@VIjTBN2$Zyh#*+;5H)|xCqe$3|!Gm`xpM$Ge`dVB-pS2*-jKUXB2LvbW;o$wLjO7 zB>|;7g2H2*hoBotbPfJCw35eM0OL+hJSl@>Q1~1~PjP+DFbXiG zNTsHzI^CWr>k0ee7pL3S8X!=^f#>{Zfvs$fb!{S*xhL)Kod$G#6T$5vZ{no&AW7)J zOMZnjs|+1KeTN$8?;)dJUn^t>)!_S+O)TEf0s8ecbYI^xqDFT0m9*t%Ip+OIq_y=ZBIZiDxujMk z@_Q7s4iuyyxzO|y=tPdJnW8wYAYywpWbtkmOW114^1V$0{}A{0Js<0cn@smIaZpb~ z$bl94Q8o+=CoYWGtrtLBPg+48q^^KhVSm=dboIx%w!7F{#4(Zgc@jbBYY@a@cy!qD z%Z0)igGzRHcQYPPP>fS_n;lyx+bpsRst^b<5SBah`cgXn&BI zmwsDdGydd-c3($wVR&SDE8unezA`^`?%YR$ZT7V@6#O)-^J%eB~PM$Dn`^Zcz+7MhWxul#-p;a>YE|mUE-qO zy+>`Zgv=ND$mc9FYk$|L6ED`0G)(P@F1K2D?)5;w$BDrHd_G%$u>Xv15A- ziPCs>LYXq-Y-|N@Ku+~1eEuqj72h%FhJ1W%3mRG04jlbxY`UZ^^_iMo>O!aXUZwoI z0O+Cp^!a?U8?sN*%753^A6VW9k)@);dtKf=*sARM3m*j=3*Z)?Ya*`$OZP#eur$q}y$0czUMJY`P9!`F%ZLi6 zi&gIN(Co_(`r%g$y$^B(5?SWNS?;EVn=j+Z=)U!2e#b>N@A7P$DQ=D7uAakBIY70; zDkz}fb_9Xd04>e`iGh?{OT@JH>lPFNGl4nn{Kr_aWUWN*@qWZ>`=?)qP$*dRA;Bg4 zQ`BL>lH|n;^DZPn3w`XgIly$KYnP|UweE2?CdMm9Xv*5R`&Xxff?`>JSg=gZq?s3A z$A@)O)t6W@CIY{$sm!M84k+UC$;~(NO+*^ZM2<4n_iNi%bR9o>@6g~!^+ZBVAhS!l zOq2N8)SlxN?z z|9r9}iaB{WWYZjv-jFR!H!|+m1#fW1Rqcxq*b~$pU?2zlz7L;P^J4;RiPzq2;y zPdJsqu5^M?);w5vjA6NMIh(8?POA8c!B$snE!x=I4%Gk60T*b}wcH+^`$x)I&HbI5;(_|@PR)V9I zw$EFHc-cDg#Han$3*`NcB2cA&tk)}!g94+I;r);$nn6Kr@=EDt*8ri?KoCaE1M_#) zk?w&0=)<*J`(PrV#X(yVr!<*#vJNgmG@rcL&uuoHB|RxJQzRftZdf)xUPCSe>B4r% z`054FfPP@-xYy`%(FHeJMZu0jy4?Vi<B@VS9lX&}X>@ z-gw_HeUd`aPJ$POC%rTIBLJB(IdgXo4SWnUk#SnHGvMb~+^w@EDc(DM*l+5>ap*%i zgS{RuFnZO0o=73{L61{dTio=FjWvz?^qk2z7W9w8(G4R%!<9jJHGza@iFpbAwrAA6 z=^&z6*I}^CJM#;hM8YBe$nfrds6_mid+#Te-vI7rDEE7o#1{2Pdjf78ZH%-NmeQB5 z)L#OpT|2`!E2PUL;VqqTbFK4g!~@Jk-*vidBWks$dyz*vpa~$2q^PNSZxr5NkNp>> z0fr}qC8mPWx&hLIzb0^%Gf>3kx5!%?FC1B6rGs5AK9~4@HfY*NT3Y;^%zHMf5L3^n zlyp_A&kTRvoo53rspaR2yUJ~43JWIk3DM-zsh-21w%~-n^87i?uvQgFn?Uu1z!PUQ z|K=Io?!&m}e8=+giRr^BcqSt&MQwmGLx+-bB*LX30V6@%-bEz@cdBL(2TSTZzfMME zH*J0aS1S6yxM$)=VqFt*p~gfWjxnCVDUq=8fnt6PotillQ%jA_K6TZaWfSJTJQ7Os z2t^ib!e2o$?~Z83rf}4{ftHHBq1TsF2rXWo4?s_khE^}n9glmD1Z@%$#Dw_ILoU@W zSPkHYVW) z{t5AgikQZdI~1nOc)zzBesRl`+rp^IPPPlPf2&vf6EgLyWFuhVIia#FF~`%@`v%=y zgoT+HCtX?9cCc zD&r@}zgLGw;9y5@dYR*x?zQ})ZgSv(w_Dp;i{e5zJC9nO?1pcH7%`tmpUxG}rW7FDS(NWi zS!A#tPK&viS*N->)Fs41OTcF3lXfq^ei9ozllQX(KJZnq0xR?)c-^?#d8yvAs9ER@d7ZFAZrX%&v=jF>w`4T zyUR4xQ3KPvqiHJfS{yV7!nOlFlaJeGV+hQI5jBfVNHJPRIs4L&g*-(1RmG!^xx1$w~+rJ}jMV;f%y zo;>pHs4QTOiuU02x(6;_7mSMN`&-qF&>CgAU)=*Mn}8+bQ2l;}@xI zj)FgiOyKr{2%S^j)ZVC66=2~AVE%Uu@E-~Iuc-fj|39ICssbX?|7^kjC%OL|k@$bo F{{y=O?iK(5 literal 0 HcmV?d00001 From 11bf40b5ad86f59b70ca1c1b3334cf0334f1ce34 Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Mon, 21 Jun 2021 09:08:58 +0100 Subject: [PATCH 05/19] Iterate over the files in the folder, not the name of the folder(!) --- mu/virtual_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mu/virtual_environment.py b/mu/virtual_environment.py index bccbe103d..2c4447362 100644 --- a/mu/virtual_environment.py +++ b/mu/virtual_environment.py @@ -835,7 +835,7 @@ def install_from_zipped_wheels(self, zipped_wheels_filepath): zip.extractall(unpacked_wheels_dirpath) self.reset_pip() - for wheel in unpacked_wheels_dirpath: + for wheel in glob.glob(os.path.join(unpacked_wheels_dirpath, "*.whl")): logger.info( "About to install from wheel: {}".format( os.path.basename(wheel) From 1967a59c51fb63dc1cb5f9bdb0477f0cb1db209c Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Mon, 21 Jun 2021 09:15:41 +0100 Subject: [PATCH 06/19] Ensure we can use glob --- mu/virtual_environment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mu/virtual_environment.py b/mu/virtual_environment.py index 2c4447362..0219b0285 100644 --- a/mu/virtual_environment.py +++ b/mu/virtual_environment.py @@ -2,6 +2,7 @@ import sys from collections import namedtuple import functools +import glob import logging import subprocess import tempfile From 1aa4b033de0dd0cda7b833cd694f9462dbf932b5 Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Mon, 21 Jun 2021 09:16:57 +0100 Subject: [PATCH 07/19] Tidy --- mu/virtual_environment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mu/virtual_environment.py b/mu/virtual_environment.py index 0219b0285..1d1c30d94 100644 --- a/mu/virtual_environment.py +++ b/mu/virtual_environment.py @@ -836,7 +836,9 @@ def install_from_zipped_wheels(self, zipped_wheels_filepath): zip.extractall(unpacked_wheels_dirpath) self.reset_pip() - for wheel in glob.glob(os.path.join(unpacked_wheels_dirpath, "*.whl")): + for wheel in glob.glob( + os.path.join(unpacked_wheels_dirpath, "*.whl") + ): logger.info( "About to install from wheel: {}".format( os.path.basename(wheel) From 5685d83f2de6f7de3ccc007468d3f8afade31d4b Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Mon, 21 Jun 2021 20:06:32 +0100 Subject: [PATCH 08/19] Ignore generated zip files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index aefc54fa6..1d13dd682 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,4 @@ target/ /baseline_packages.json venv-pup/ /local-scripts +/mu/wheels/*.zip From 4448691eade8828ff0150b514a002c79a187201c Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Mon, 21 Jun 2021 20:24:00 +0100 Subject: [PATCH 09/19] Refactor baseline install test to allow for the fact that we now install from zips, not from files --- .../test_virtual_environment.py | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/tests/virtual_environment/test_virtual_environment.py b/tests/virtual_environment/test_virtual_environment.py index b38242be4..0b03aecb0 100644 --- a/tests/virtual_environment/test_virtual_environment.py +++ b/tests/virtual_environment/test_virtual_environment.py @@ -24,6 +24,7 @@ import uuid import logging from unittest import mock +import zipfile import pytest @@ -39,6 +40,7 @@ HERE = os.path.dirname(__file__) ZIP_FILENAME = "wheels.zip" WHEEL_FILENAME = "arrr-1.0.2-py3-none-any.whl" +WHEEL_FILEPATH = os.path.join(HERE, "wheels", WHEEL_FILENAME) @pytest.fixture @@ -127,10 +129,10 @@ def test_wheels(tmp_path): wheels_dirpath = str(tmp_path / "wheels") zip_filename = "%s.zip" % mu.__version__ os.mkdir(wheels_dirpath) - shutil.copyfile( - os.path.join(HERE, "wheels", ZIP_FILENAME), - os.path.join(wheels_dirpath, zip_filename), - ) + + with zipfile.ZipFile(os.path.join(wheels_dirpath, zip_filename), "w") as z: + z.write(WHEEL_FILEPATH, WHEEL_FILENAME) + with mock.patch.object( mu.virtual_environment, "wheels_dirpath", wheels_dirpath ): @@ -315,20 +317,19 @@ def test_base_packages_installed(patched, venv, test_wheels): # Check that we're calling `pip install` with all the # wheels in the wheelhouse # - expected_args = glob.glob( - os.path.join(mu.virtual_environment.wheels_dirpath, "*.whl") - ) - # - # Make sure the juypter kernel install doesn't interfere - # - with mock.patch.object(VE, "install_jupyter_kernel"): - with mock.patch.object(VE, "register_baseline_packages"): - with mock.patch.object(PIP, "install") as mock_pip_install: - venv.create() - for mock_call_args in mock_pip_install.call_args_list: - assert len(mock_call_args[0]) == 1 - assert mock_call_args[0][0] in expected_args - assert mock_call_args[1] == {"deps": False, "index": False} + expected_args = [WHEEL_FILENAME] + + with mock.patch.object(venv, "create_venv"), mock.patch.object( + venv, "register_baseline_packages" + ), mock.patch.object(venv, "install_jupyter_kernel"), mock.patch.object( + PIP, "install" + ) as mock_pip_install: + venv.create() + + for (mock_args, mock_kwargs) in mock_pip_install.call_args_list: + assert len(mock_args) == 1 + assert os.path.basename(mock_args[0]) in expected_args + assert mock_kwargs == {"deps": False, "index": False} def test_jupyter_kernel_installed(patched, venv): From 27182d2626252988b47f2c947d9a595af934ea80 Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Mon, 21 Jun 2021 20:29:20 +0100 Subject: [PATCH 10/19] Tweak tests for logging setup to allow for MU_LOG_TO_STDOUT env var --- tests/test_app.py | 47 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/tests/test_app.py b/tests/test_app.py index c132e21c9..b68d0f2ed 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -72,7 +72,7 @@ def test_animated_splash_init(): asplash.animation.start.assert_called_once_with() -def test_animated_splash_set_frame(): # run successfully +def test_animated_splash_set_frame(): """ Ensure the splash screen's pixmap is updated with the animation's current pixmap. @@ -91,7 +91,7 @@ def test_animated_splash_set_frame(): # run successfully asplash.setMask.assert_called_once_with(pixmap.mask()) -def test_animated_splash_draw_log(): # run successfully +def test_animated_splash_draw_log(): """ Ensure the scrolling updates from the log handler are sliced properly and the expected text is shown in the right place on the splash screen. @@ -116,7 +116,7 @@ def test_animated_splash_draw_log(): # run successfully ) -def test_animated_splash_failed(): # run successfully +def test_animated_splash_failed(): """ When instructed to transition to a failed state, ensure the correct image is displayed along with the correct message. @@ -161,7 +161,7 @@ def test_worker_run(): assert not isinstance(handler, SplashLogHandler) -def test_worker_fail(): # run successfully +def test_worker_fail(): """ Ensure that exceptions encountered during Mu's start-up are handled in the expected manner. @@ -183,10 +183,14 @@ def test_worker_fail(): # run successfully w.finished.emit.assert_called_once_with() -def test_setup_logging(): # run successfully +def test_setup_logging_without_envvar(): """ Ensure that logging is set up in some way. + + Resetting the MU_LOG_TO_STDOUT env var ensures that the crash handler + will be enabled and stdout logging not """ + os.environ.pop("MU_LOG_TO_STDOUT", "") with mock.patch("mu.app.TimedRotatingFileHandler") as log_conf, mock.patch( "mu.app.os.path.exists", return_value=False ), mock.patch("mu.app.logging") as logging, mock.patch( @@ -205,7 +209,32 @@ def test_setup_logging(): # run successfully assert sys.excepthook == excepthook -def test_run(): # run successfully +def test_setup_logging_with_envvar(): + """ + Ensure that logging is set up in some way. + + Setting the MU_LOG_TO_STDOUT env var ensures that the crash handler + will not be enabled and stdout logging will + """ + os.environ["MU_LOG_TO_STDOUT"] = "1" + with mock.patch("mu.app.TimedRotatingFileHandler") as log_conf, mock.patch( + "mu.app.os.path.exists", return_value=False + ), mock.patch("mu.app.logging") as logging, mock.patch( + "mu.app.os.makedirs", return_value=None + ) as mkdir: + setup_logging() + mkdir.assert_called_once_with(LOG_DIR) + log_conf.assert_called_once_with( + LOG_FILE, + when="midnight", + backupCount=5, + delay=0, + encoding=ENCODING, + ) + logging.getLogger.assert_called_once_with() + #~ assert sys.excepthook == excepthook + +def test_run(): """ Ensure the run function sets things up in the expected way. @@ -333,7 +362,7 @@ def test_excepthook(): assert browser.open.call_count == 1 -def test_excepthook_alamo(): # run successfully +def test_excepthook_alamo(): """ If the crash reporting code itself encounters an error, then ensure this is logged before exiting. @@ -352,7 +381,7 @@ def test_excepthook_alamo(): # run successfully exit.assert_called_once_with(1) -def test_debug(): # run successfully +def test_debug(): """ Ensure the debugger is run with the expected arguments given the filename and other arguments passed in via sys.argv. @@ -368,7 +397,7 @@ def test_debug(): # run successfully ) -def test_debug_no_args(): # run successfully +def test_debug_no_args(): """ If the debugger is accidentally started with no filename and/or associated args, then emit a friendly message to indicate the problem. From dd222d1bbc6e11924ad788fda94c8abbf93d140a Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Mon, 21 Jun 2021 20:34:42 +0100 Subject: [PATCH 11/19] Tidy --- tests/test_app.py | 3 ++- tests/virtual_environment/test_virtual_environment.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_app.py b/tests/test_app.py index b68d0f2ed..c9685526b 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -232,7 +232,8 @@ def test_setup_logging_with_envvar(): encoding=ENCODING, ) logging.getLogger.assert_called_once_with() - #~ assert sys.excepthook == excepthook + # ~ assert sys.excepthook == excepthook + def test_run(): """ diff --git a/tests/virtual_environment/test_virtual_environment.py b/tests/virtual_environment/test_virtual_environment.py index 0b03aecb0..770d3b36c 100644 --- a/tests/virtual_environment/test_virtual_environment.py +++ b/tests/virtual_environment/test_virtual_environment.py @@ -19,7 +19,6 @@ import os import glob import random -import shutil import subprocess import uuid import logging From d6ffa3a8680ad33722aa7c5339be93ffdac5ce5e Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Tue, 22 Jun 2021 08:00:56 +0100 Subject: [PATCH 12/19] Disable the pip version check when downloading wheels Always add the temp download area to the search patch for downloading wheels. This should work for the MacOS shim where we have to download a signed wheel for pygame and then be sure to use that in preference to the PyPI version --- mu/virtual_environment.py | 4 ---- mu/wheels/__init__.py | 9 ++++++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/mu/virtual_environment.py b/mu/virtual_environment.py index 1d1c30d94..f114e2eb3 100644 --- a/mu/virtual_environment.py +++ b/mu/virtual_environment.py @@ -628,10 +628,6 @@ def ensure_and_create(self, emitter=None): else: raise finally: - logger.debug( - "Emitter: %s; Splash Handler; %s" - % (emitter, splash_handler) - ) if emitter and splash_handler: logger.removeHandler(splash_handler) diff --git a/mu/wheels/__init__.py b/mu/wheels/__init__.py index 6903a5b71..a56f04a5a 100644 --- a/mu/wheels/__init__.py +++ b/mu/wheels/__init__.py @@ -28,6 +28,9 @@ class WheelsBuildError(WheelsError): logger = logging.getLogger(__name__) +WHEELS_DIRPATH = os.path.dirname(__file__) +ZIP_FILEPATH = os.path.join(WHEELS_DIRPATH, mu_version + ".zip") + # # List of base packages to support modes # The first element should be the importable name (so "serial" rather than "pyserial") @@ -40,8 +43,6 @@ class WheelsBuildError(WheelsError): ("qtconsole", "qtconsole==4.7.4"), ("esptool", "esptool==3.*"), ] -WHEELS_DIRPATH = os.path.dirname(__file__) -ZIP_FILEPATH = os.path.join(WHEELS_DIRPATH, mu_version + ".zip") # TODO: Temp app signing workaround https://github.com/mu-editor/mu/issues/1290 if sys.version_info[:2] == (3, 8) and platform.system() == "Darwin": @@ -57,7 +58,6 @@ class WheelsBuildError(WheelsError): "https://github.com/mu-editor/pgzero/releases/download/mu-wheel/" "pgzero-1.2-py3-none-any.whl", "--no-index", - "--find-links=" + WHEELS_DIRPATH, ), ] + mode_packages[1:] @@ -97,9 +97,12 @@ def pip_download(dirpath, logger): sys.executable, "-m", "pip", + "--disable-pip-version-check", "download", "--destination-directory", dirpath, + "--find-links", + dirpath, pip_identifier, ] + extra_flags, From 05daba5107de0151d5d964facd9a84b54122157b Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Wed, 23 Jun 2021 20:28:56 +0100 Subject: [PATCH 13/19] Add more useful function comments Remove (again) `upgrade_pip` + Tidy --- mu/virtual_environment.py | 57 +++++++++++-------- .../test_virtual_environment.py | 22 ------- 2 files changed, 34 insertions(+), 45 deletions(-) diff --git a/mu/virtual_environment.py b/mu/virtual_environment.py index f114e2eb3..da93dd002 100644 --- a/mu/virtual_environment.py +++ b/mu/virtual_environment.py @@ -95,7 +95,7 @@ def __init__(self): self.environment.insert("PYTHONIOENCODING", "utf-8") def _set_up_run(self, **envvars): - """Run the process with the command and args""" + """Set up common elements of a QProcess run""" self.process = QProcess() environment = QProcessEnvironment(self.environment) for k, v in envvars.items(): @@ -104,6 +104,11 @@ def _set_up_run(self, **envvars): self.process.setProcessChannelMode(QProcess.MergedChannels) def run_blocking(self, command, args, wait_for_s=30.0, **envvars): + """Run `command` with `args` via QProcess, passing `envvars` as + environment variables for the process. + + Wait `wait_for_s` seconds for completion and return any stdout/stderr + """ logger.info( "About to run blocking %s with args %s and envvars %s", command, @@ -115,6 +120,8 @@ def run_blocking(self, command, args, wait_for_s=30.0, **envvars): return self.wait(wait_for_s=wait_for_s) def run(self, command, args, **envvars): + """Run `command` asynchronously with `args` via QProcess, passing `envvars` + as environment variables for the process.""" logger.info( "About to run %s with args %s and envvars %s", command, @@ -129,6 +136,12 @@ def run(self, command, args, **envvars): QTimer.singleShot(1, partial) def wait(self, wait_for_s=30.0): + """Wait for the process to complete, optionally timing out. + Return any stdout/stderr. + + If the process fails to complete in time or returns an error, raise a + VirtualEnvironmentError + """ finished = self.process.waitForFinished(1000 * wait_for_s) exit_status = self.process.exitStatus() exit_code = self.process.exitCode() @@ -176,6 +189,7 @@ def wait(self, wait_for_s=30.0): return output def data(self): + """Return all the data from the running process, converted to unicode""" output = self.process.readAll().data() return output.decode(sys.stdout.encoding, errors="replace") @@ -394,7 +408,7 @@ def __str__(self): @staticmethod def _generate_dirpath(): """ - Construct a unique virtual environment folder + Construct a unique virtual environment folder name To avoid clashing with previously-created virtual environments, construct one which includes the Python version and a timestamp @@ -434,6 +448,9 @@ def run_subprocess(self, *args, **kwargs): return process.returncode == 0, output def reset_pip(self): + """To avoid a problem where the same Pip process is executed by different + threads, recreate the Pip process on demand + """ self.pip = Pip(self.pip_executable) def relocate(self, dirpath): @@ -508,6 +525,12 @@ def _directory_is_venv(self): return False def quarantine_venv(self, reason="FAILED"): + """Rename an existing virtual environment folder out of the way to make + it clearer which is the current one. + + NB if this fails (eg because of file locking) it won't matter: the folder + will not be used and will simply be a little distracting + """ error_dirpath = self.path + "." + reason try: os.rename(self.path, error_dirpath) @@ -751,7 +774,7 @@ def create(self): def create_venv(self): """ - Create a new virtualenv at the referenced path. + Create a new virtualenv """ logger.info("Creating virtualenv: {}".format(self.path)) logger.info("Virtualenv name: {}".format(self.name)) @@ -779,26 +802,10 @@ def create_venv(self): % (sys.executable, self.path, compact(output)) ) - def upgrade_pip(self): - logger.debug( - "About to upgrade pip; interpreter %s %s", - self.interpreter, - "exists" if os.path.exists(self.interpreter) else "doesn't exist", - ) - ok, output = self.run_subprocess( - self.interpreter, "-m", "pip", "install", "--upgrade", "pip" - ) - if ok: - logger.info("Upgraded pip") - else: - raise VirtualEnvironmentCreateError( - "Unable to upgrade pip\n%s" % compact(output) - ) - def install_jupyter_kernel(self): """ Install a Jupyter kernel for Mu (the name of the kernel indicates this - is a Mu related kernel). + is a Mu-related kernel). """ kernel_name = self.name.replace(" ", "-") display_name = '"Python/Mu ({})"'.format(kernel_name) @@ -822,6 +829,9 @@ def install_jupyter_kernel(self): ) def install_from_zipped_wheels(self, zipped_wheels_filepath): + """Take a zip containing wheels, unzip it and install the wheels into + the current virtualenv + """ with tempfile.TemporaryDirectory() as unpacked_wheels_dirpath: # # The wheel files are shipped in Mu-version-specific zip files to avoid @@ -832,6 +842,10 @@ def install_from_zipped_wheels(self, zipped_wheels_filepath): zip.extractall(unpacked_wheels_dirpath) self.reset_pip() + # + # The wheels are installed one at a time as they reduces the possibility + # of the process installing them breaching its timeout + # for wheel in glob.glob( os.path.join(unpacked_wheels_dirpath, "*.whl") ): @@ -852,8 +866,6 @@ def install_baseline_packages(self): no network access is needed. But if the wheels aren't found, because we're not running from an installer, then just pip install in the usual way. - - --upgrade is currently used with a thought to upgrade-releases of Mu. """ logger.info("Installing baseline packages.") # @@ -862,7 +874,6 @@ def install_baseline_packages(self): zipped_wheels_filepath = os.path.join( wheels_dirpath, "%s.zip" % mu_version ) - print("Zipped wheels filepath:", zipped_wheels_filepath) logger.info("Expecting zipped wheels at %s", zipped_wheels_filepath) if not os.path.exists(zipped_wheels_filepath): logger.warning("No zipped wheels found; downloading...") diff --git a/tests/virtual_environment/test_virtual_environment.py b/tests/virtual_environment/test_virtual_environment.py index 770d3b36c..8712565b7 100644 --- a/tests/virtual_environment/test_virtual_environment.py +++ b/tests/virtual_environment/test_virtual_environment.py @@ -372,28 +372,6 @@ def test_jupyter_kernel_failure(patched, venv): assert output in exc.message -def test_upgrade_pip_failure(venv): - """Ensure that we raise an error with output when pip can't be upgraded""" - output = uuid.uuid1().hex - with mock.patch.object( - venv, "run_subprocess", return_value=(True, output) - ): - venv.upgrade_pip() - - -def test_upgrade_pip_success(venv): - """Ensure that we raise an error with output when pip can't be upgraded""" - output = uuid.uuid1().hex - with mock.patch.object( - venv, "run_subprocess", return_value=(False, output) - ): - try: - venv.upgrade_pip() - except VEError as exc: - assert "nable to upgrade pip" in exc.message - assert output in exc.message - - def test_install_user_packages(patched, venv): """Ensure that, given a list of packages, we pip install them From 6a3ee308e14893db6766a4d895ee9e2f96f98158 Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Sat, 1 Jul 2023 20:40:30 +0100 Subject: [PATCH 14/19] Housekeeping --- Makefile | 5 +++-- mu/app.py | 4 ++++ run-mu.sh | 7 +++++++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100755 run-mu.sh diff --git a/Makefile b/Makefile index d11e6ac21..5f9ec4c85 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ coverage: clean export LANG=en_GB.utf8 pytest -v --random-order --cov-config setup.cfg --cov-report term-missing --cov=mu tests/ -tidy: +tidy: python make.py tidy black: @@ -119,7 +119,8 @@ macos: check ls -la ./build/pup/ ls -la ./dist/ -linux: check +linux: + #check @echo "\nFetching wheels." python -m mu.wheels --package @echo "\nPackaging Mu into a Linux AppImage." diff --git a/mu/app.py b/mu/app.py index 3a46a13af..3082ec92e 100644 --- a/mu/app.py +++ b/mu/app.py @@ -325,6 +325,7 @@ def is_linux_wayland(): def check_only_running_once(): """If the application is already running log the error and exit""" + return try: with _shared_memory: _shared_memory.acquire() @@ -354,6 +355,9 @@ def run(): logging.info("Platform: {}".format(platform.platform())) logging.info("Python path: {}".format(sys.path)) logging.info("Language code: {}".format(i18n.language_code)) + for k, v in os.environ.items(): + if k.startswith(("XDG", "WAYLAND", "WESTON", "XCURSOR", "DISPLAY")): + logging.info("Env: %s => %s", k, v) setup_exception_handler() check_only_running_once() diff --git a/run-mu.sh b/run-mu.sh new file mode 100755 index 000000000..2ae29c26b --- /dev/null +++ b/run-mu.sh @@ -0,0 +1,7 @@ +#!/bin/sh +export WAYLAND_DISPLAY=wayland-0 +#~ export XDG_RUNTIME_DIR=/tmp/1000-runtime-dir +#export QT_QPA_PLATFORM=wayland +export WAYLAND_DEBUG=1 +export PYTHONDEVMODE=1 +dist/Mu_Editor-1.2.1-x86_64.AppImage From 46cd0650c3fa579ec4fad99b5861bc5a74627e9e Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Fri, 8 Sep 2023 07:24:58 +0100 Subject: [PATCH 15/19] Add a shim to release to acquire and release the shared memory object --- mu/app.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mu/app.py b/mu/app.py index 3082ec92e..5317a96bb 100644 --- a/mu/app.py +++ b/mu/app.py @@ -297,6 +297,16 @@ def __exit__(self, *args, **kwargs): self._shared_memory.unlock() def acquire(self): + # + # The attach-detach dance is a shim from + # https://stackoverflow.com/questions/42549904/qsharedmemory-is-not-getting-deleted-on-application-crash + # If the existing shared memory is not held by any active application + # (eg because an appimage has hard-crashed) then it will be released + # If the memory is held by an active application it will have no effect + # + self._shared_memory.attach() + self._shared_memory.detach() + if self._shared_memory.attach(): pid = struct.unpack("q", self._shared_memory.data()[:8]) raise MutexError("MUTEX: Mu is already running with pid %d" % pid) @@ -325,7 +335,6 @@ def is_linux_wayland(): def check_only_running_once(): """If the application is already running log the error and exit""" - return try: with _shared_memory: _shared_memory.acquire() From 6c3b78231ee077ffcc4faf511c298df2fc5dee8b Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Fri, 8 Sep 2023 07:25:27 +0100 Subject: [PATCH 16/19] Test acquiring and release mutex --- reset-mutex.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 reset-mutex.py diff --git a/reset-mutex.py b/reset-mutex.py new file mode 100644 index 000000000..226f8cf9f --- /dev/null +++ b/reset-mutex.py @@ -0,0 +1,27 @@ +import struct +from PyQt5.QtCore import ( + Qt, + QEventLoop, + QThread, + QObject, + pyqtSignal, + QSharedMemory, +) + +shm = QSharedMemory("mu-tex") +shm.lock() +try: + if shm.attach(): + print("Able to attach") + else: + print("Unable to attach") + data = shm.data() + print(data) + if data: + print(list(data)) + print(struct.unpack("q", data[:8])) + else: + print("No data") + shm.detach() +finally: + shm.unlock() From 739fb24251ba47a287ecde8830a32a2d4bba937e Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Fri, 8 Sep 2023 07:36:30 +0100 Subject: [PATCH 17/19] Move short-term scripts out of the committed filespace --- reset-mutex.py | 27 --------------------------- run-mu.sh | 7 ------- 2 files changed, 34 deletions(-) delete mode 100644 reset-mutex.py delete mode 100755 run-mu.sh diff --git a/reset-mutex.py b/reset-mutex.py deleted file mode 100644 index 226f8cf9f..000000000 --- a/reset-mutex.py +++ /dev/null @@ -1,27 +0,0 @@ -import struct -from PyQt5.QtCore import ( - Qt, - QEventLoop, - QThread, - QObject, - pyqtSignal, - QSharedMemory, -) - -shm = QSharedMemory("mu-tex") -shm.lock() -try: - if shm.attach(): - print("Able to attach") - else: - print("Unable to attach") - data = shm.data() - print(data) - if data: - print(list(data)) - print(struct.unpack("q", data[:8])) - else: - print("No data") - shm.detach() -finally: - shm.unlock() diff --git a/run-mu.sh b/run-mu.sh deleted file mode 100755 index 2ae29c26b..000000000 --- a/run-mu.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -export WAYLAND_DISPLAY=wayland-0 -#~ export XDG_RUNTIME_DIR=/tmp/1000-runtime-dir -#export QT_QPA_PLATFORM=wayland -export WAYLAND_DEBUG=1 -export PYTHONDEVMODE=1 -dist/Mu_Editor-1.2.1-x86_64.AppImage From 51fccc7c65687cc9ead283f0bd00c0656bcd9b5f Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Sat, 9 Sep 2023 07:33:34 +0100 Subject: [PATCH 18/19] Remove debugging aids --- Makefile | 2 +- mu/app.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 5f9ec4c85..186292d5f 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,7 @@ macos: check ls -la ./dist/ linux: - #check + check @echo "\nFetching wheels." python -m mu.wheels --package @echo "\nPackaging Mu into a Linux AppImage." diff --git a/mu/app.py b/mu/app.py index 5317a96bb..36a245177 100644 --- a/mu/app.py +++ b/mu/app.py @@ -364,10 +364,6 @@ def run(): logging.info("Platform: {}".format(platform.platform())) logging.info("Python path: {}".format(sys.path)) logging.info("Language code: {}".format(i18n.language_code)) - for k, v in os.environ.items(): - if k.startswith(("XDG", "WAYLAND", "WESTON", "XCURSOR", "DISPLAY")): - logging.info("Env: %s => %s", k, v) - setup_exception_handler() check_only_running_once() From 1d663179e08400672cdc33be099c1e5fa8149ca6 Mon Sep 17 00:00:00 2001 From: Tim Golden Date: Sat, 9 Sep 2023 07:36:42 +0100 Subject: [PATCH 19/19] Tidy up --- Makefile | 3 +-- setup.cfg | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 186292d5f..f3cde37ac 100644 --- a/Makefile +++ b/Makefile @@ -119,8 +119,7 @@ macos: check ls -la ./build/pup/ ls -la ./dist/ -linux: - check +linux: check @echo "\nFetching wheels." python -m mu.wheels --package @echo "\nPackaging Mu into a Linux AppImage." diff --git a/setup.cfg b/setup.cfg index 432044306..790b813f0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,6 +23,7 @@ exclude = ./.venv*/ ./env/ ./.env/ + ./local-scripts/ max-line-length = 88 [coverage:run]