diff --git a/bin/lib/rust_library_builder.py b/bin/lib/rust_library_builder.py index 8d5953e6e..47ec4ea20 100644 --- a/bin/lib/rust_library_builder.py +++ b/bin/lib/rust_library_builder.py @@ -122,6 +122,7 @@ def writebuildscript( flagscombination, ldPath, build_method, + logfolder, ): rustbinpath = os.path.dirname(compilerexe) rustpath = os.path.dirname(rustbinpath) @@ -144,7 +145,7 @@ def writebuildscript( f.write(f"{line}\n") if self.buildconfig.build_type == "cargo": - cargoline = f"$CARGO build {methodflags} --target-dir {buildfolder} >> buildlog.txt 2>&1\n" + cargoline = f"$CARGO build {methodflags} --target-dir {buildfolder} > {logfolder}/buildlog.txt 2>&1\n" f.write(cargoline) else: raise RuntimeError("Unknown build_type {self.buildconfig.build_type}") @@ -302,7 +303,7 @@ def conanproxy_login(self): response = json.loads(request.content) self.conanserverproxy_token = response["token"] - def save_build_logging(self, builtok, build_folder, source_folder): + def save_build_logging(self, builtok, logfolder, source_folder, build_method): if builtok == BuildStatus.Failed: url = f"{conanserver_url}/buildfailed" elif builtok == BuildStatus.Ok: @@ -313,7 +314,7 @@ def save_build_logging(self, builtok, build_folder, source_folder): return loggingfiles = [] - loggingfiles += glob.glob(source_folder + "/buildlog.txt") + loggingfiles += glob.glob(logfolder + "/buildlog.txt") logging_data = "" for logfile in loggingfiles: @@ -325,6 +326,9 @@ def save_build_logging(self, builtok, build_folder, source_folder): buildparameters_copy = self.current_buildparameters_obj.copy() buildparameters_copy["logging"] = logging_data + if builtok != BuildStatus.Ok: + buildparameters_copy["flagcollection"] = build_method["build_method"] + headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token} req_data = json.dumps(buildparameters_copy) @@ -352,9 +356,13 @@ def get_build_annotations(self, buildfolder): def get_commit_hash(self) -> str: return self.target_name - def has_failed_before(self): + def has_failed_before(self, build_method): url = f"{conanserver_url}/hasfailedbefore" - req_data = json.dumps(self.current_buildparameters_obj) + + data = self.current_buildparameters_obj.copy() + data["flagcollection"] = build_method["build_method"] + + req_data = json.dumps(data) request = self.resil_post(url, req_data) if not request.ok: @@ -420,8 +428,8 @@ def download_library(self, build_folder, source_folder, staging: StagingDir): staging, url, command=tar_cmd, subdir=source_folder, agent=get_builder_user_agent_id() ) - def get_source_folder(self, staging: StagingDir): - source_folder = os.path.join(staging.path, f"source_{self.libname}_{self.target_name}") + def get_source_folder(self, source_staging: StagingDir): + source_folder = os.path.join(source_staging.path, f"crate_{self.libname}_{self.target_name}") if not source_folder in self.cached_source_folders: if not os.path.exists(source_folder): os.mkdir(source_folder) @@ -442,6 +450,7 @@ def makebuildfor( stdlib, flagscombination, ld_path, + source_staging, ): with self.install_context.new_staging_dir() as staging: build_method = dict({"build_method": "--all-features", "linker": "/opt/compiler-explorer/gcc-11.1.0"}) @@ -460,8 +469,9 @@ def makebuildfor( ld_path, build_method, staging, + source_staging, ) - if build_status == BuildStatus.Failed: + if build_status == BuildStatus.Failed or build_status == BuildStatus.Skipped: build_method = dict({"build_method": "", "linker": "/opt/compiler-explorer/gcc-11.1.0"}) build_status = self.makebuildfor_by_method( compiler, @@ -478,6 +488,7 @@ def makebuildfor( ld_path, build_method, staging, + source_staging, ) return build_status @@ -498,6 +509,7 @@ def makebuildfor_by_method( ld_path, build_method, staging: StagingDir, + source_staging: StagingDir ): combined_hash = self.makebuildhash( compiler, options, toolchain, buildos, buildtype, arch, stdver, stdlib, flagscombination @@ -511,8 +523,10 @@ def makebuildfor_by_method( self.logger.debug(f"Buildfolder: {build_folder}") real_build_folder = os.path.join(build_folder, "build") + log_folder = os.path.join(build_folder, "log") + os.makedirs(log_folder, exist_ok=True) - source_folder = self.get_source_folder(staging) + source_folder = self.get_source_folder(source_staging) self.writeconanfile(build_folder) @@ -532,9 +546,10 @@ def makebuildfor_by_method( flagscombination, ld_path, build_method, + log_folder, ) - if not self.forcebuild and self.has_failed_before(): + if not self.forcebuild and self.has_failed_before(build_method): self.logger.info("Build has failed before, not re-attempting") return BuildStatus.Skipped @@ -561,7 +576,7 @@ def makebuildfor_by_method( self.logger.debug(f"Number of valid library binaries {filesfound}") if not self.install_context.dry_run: - self.save_build_logging(build_status, build_folder, source_folder) + self.save_build_logging(build_status, log_folder, source_folder, build_method) if build_status == BuildStatus.Ok: self.build_cleanup(build_folder) @@ -583,6 +598,8 @@ def cache_cleanup(self): if not self.install_context.dry_run: for folder in self.cached_source_folders: shutil.rmtree(folder, ignore_errors=True) + else: + self.logger.info(f"Would clean crate cache, but in dry-run mode") def upload_builds(self): if self.needs_uploading > 0: @@ -611,51 +628,52 @@ def makebuild(self, buildfor): if checkcompiler not in self.compilerprops: self.logger.error(f"Unknown compiler {checkcompiler}") - for compiler in self.compilerprops: - if checkcompiler != "" and compiler != checkcompiler: - continue + with self.install_context.new_staging_dir() as source_staging: + for compiler in self.compilerprops: + if checkcompiler != "" and compiler != checkcompiler: + continue - if compiler in self.buildconfig.skip_compilers: - self.logger.debug(f"Skipping {compiler}") - continue + if compiler in self.buildconfig.skip_compilers: + self.logger.debug(f"Skipping {compiler}") + continue - if compiler in skip_compilers: - self.logger.debug(f"Skipping {compiler}") - continue + if compiler in skip_compilers: + self.logger.debug(f"Skipping {compiler}") + continue - # compiler_semver = version.parse(self.compilerprops[compiler]['semver']) - # if compiler_semver < min_compiler_version: - # self.logger.debug(f'Skipping {compiler} (too old)') - # continue + # compiler_semver = version.parse(self.compilerprops[compiler]['semver']) + # if compiler_semver < min_compiler_version: + # self.logger.debug(f'Skipping {compiler} (too old)') + # continue - if "compilerType" in self.compilerprops[compiler]: - compilerType = self.compilerprops[compiler]["compilerType"] - else: - raise RuntimeError(f"Something is wrong with {compiler}") - - exe = self.compilerprops[compiler]["exe"] - options = self.compilerprops[compiler]["options"] - toolchain = "" - - stdlibs = [""] - archs = build_supported_arch - stdvers = build_supported_stdver - ldPath = "" - - for args in itertools.product( - build_supported_os, build_supported_buildtype, archs, stdvers, stdlibs, build_supported_flagscollection - ): - buildstatus = self.makebuildfor(compiler, options, exe, compilerType, toolchain, *args, ldPath) - if buildstatus == BuildStatus.Ok: - builds_succeeded = builds_succeeded + 1 - elif buildstatus == BuildStatus.Skipped: - builds_skipped = builds_skipped + 1 + if "compilerType" in self.compilerprops[compiler]: + compilerType = self.compilerprops[compiler]["compilerType"] else: - builds_failed = builds_failed + 1 - - if builds_succeeded > 0: - self.upload_builds() - - self.cache_cleanup() + raise RuntimeError(f"Something is wrong with {compiler}") + + exe = self.compilerprops[compiler]["exe"] + options = self.compilerprops[compiler]["options"] + toolchain = "" + + stdlibs = [""] + archs = build_supported_arch + stdvers = build_supported_stdver + ldPath = "" + + for args in itertools.product( + build_supported_os, build_supported_buildtype, archs, stdvers, stdlibs, build_supported_flagscollection + ): + buildstatus = self.makebuildfor(compiler, options, exe, compilerType, toolchain, *args, ldPath, source_staging) + if buildstatus == BuildStatus.Ok: + builds_succeeded = builds_succeeded + 1 + elif buildstatus == BuildStatus.Skipped: + builds_skipped = builds_skipped + 1 + else: + builds_failed = builds_failed + 1 + + if builds_succeeded > 0: + self.upload_builds() + + self.cache_cleanup() return [builds_succeeded, builds_skipped, builds_failed]