Skip to content

Commit

Permalink
fix cache bug, separate some paths, attempt to fix build_method issues
Browse files Browse the repository at this point in the history
  • Loading branch information
@partouf
partouf committed Aug 24, 2023
1 parent 3fa79be commit fb52c74
Showing 1 changed file with 70 additions and 52 deletions.
122 changes: 70 additions & 52 deletions bin/lib/rust_library_builder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ def writebuildscript(
flagscombination,
ldPath,
build_method,
logfolder,
):
rustbinpath = os.path.dirname(compilerexe)
rustpath = os.path.dirname(rustbinpath)
Expand All @@ -144,7 +145,7 @@ def writebuildscript(
f.write(f"{line}\n")

if self.buildconfig.build_type == "cargo":
cargoline = f"$CARGO build {methodflags} --target-dir {buildfolder} >> buildlog.txt 2>&1\n"
cargoline = f"$CARGO build {methodflags} --target-dir {buildfolder} > {logfolder}/buildlog.txt 2>&1\n"
f.write(cargoline)
else:
raise RuntimeError("Unknown build_type {self.buildconfig.build_type}")
Expand Down Expand Up @@ -302,7 +303,7 @@ def conanproxy_login(self):
response = json.loads(request.content)
self.conanserverproxy_token = response["token"]

def save_build_logging(self, builtok, build_folder, source_folder):
def save_build_logging(self, builtok, logfolder, source_folder, build_method):
if builtok == BuildStatus.Failed:
url = f"{conanserver_url}/buildfailed"
elif builtok == BuildStatus.Ok:
Expand All @@ -313,7 +314,7 @@ def save_build_logging(self, builtok, build_folder, source_folder):
return

loggingfiles = []
loggingfiles += glob.glob(source_folder + "/buildlog.txt")
loggingfiles += glob.glob(logfolder + "/buildlog.txt")

logging_data = ""
for logfile in loggingfiles:
Expand All @@ -325,6 +326,9 @@ def save_build_logging(self, builtok, build_folder, source_folder):
buildparameters_copy = self.current_buildparameters_obj.copy()
buildparameters_copy["logging"] = logging_data

if builtok != BuildStatus.Ok:
buildparameters_copy["flagcollection"] = build_method["build_method"]

headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token}

req_data = json.dumps(buildparameters_copy)
Expand Down Expand Up @@ -352,9 +356,13 @@ def get_build_annotations(self, buildfolder):
def get_commit_hash(self) -> str:
return self.target_name

def has_failed_before(self):
def has_failed_before(self, build_method):
url = f"{conanserver_url}/hasfailedbefore"
req_data = json.dumps(self.current_buildparameters_obj)

data = self.current_buildparameters_obj.copy()
data["flagcollection"] = build_method["build_method"]

req_data = json.dumps(data)

request = self.resil_post(url, req_data)
if not request.ok:
Expand Down Expand Up @@ -420,8 +428,8 @@ def download_library(self, build_folder, source_folder, staging: StagingDir):
staging, url, command=tar_cmd, subdir=source_folder, agent=get_builder_user_agent_id()
)

def get_source_folder(self, staging: StagingDir):
source_folder = os.path.join(staging.path, f"source_{self.libname}_{self.target_name}")
def get_source_folder(self, source_staging: StagingDir):
source_folder = os.path.join(source_staging.path, f"crate_{self.libname}_{self.target_name}")
if not source_folder in self.cached_source_folders:
if not os.path.exists(source_folder):
os.mkdir(source_folder)
Expand All @@ -442,6 +450,7 @@ def makebuildfor(
stdlib,
flagscombination,
ld_path,
source_staging,
):
with self.install_context.new_staging_dir() as staging:
build_method = dict({"build_method": "--all-features", "linker": "/opt/compiler-explorer/gcc-11.1.0"})
Expand All @@ -460,8 +469,9 @@ def makebuildfor(
ld_path,
build_method,
staging,
source_staging,
)
if build_status == BuildStatus.Failed:
if build_status == BuildStatus.Failed or build_status == BuildStatus.Skipped:
build_method = dict({"build_method": "", "linker": "/opt/compiler-explorer/gcc-11.1.0"})
build_status = self.makebuildfor_by_method(
compiler,
Expand All @@ -478,6 +488,7 @@ def makebuildfor(
ld_path,
build_method,
staging,
source_staging,
)

return build_status
Expand All @@ -498,6 +509,7 @@ def makebuildfor_by_method(
ld_path,
build_method,
staging: StagingDir,
source_staging: StagingDir
):
combined_hash = self.makebuildhash(
compiler, options, toolchain, buildos, buildtype, arch, stdver, stdlib, flagscombination
Expand All @@ -511,8 +523,10 @@ def makebuildfor_by_method(
self.logger.debug(f"Buildfolder: {build_folder}")

real_build_folder = os.path.join(build_folder, "build")
log_folder = os.path.join(build_folder, "log")
os.makedirs(log_folder, exist_ok=True)

source_folder = self.get_source_folder(staging)
source_folder = self.get_source_folder(source_staging)

self.writeconanfile(build_folder)

Expand All @@ -532,9 +546,10 @@ def makebuildfor_by_method(
flagscombination,
ld_path,
build_method,
log_folder,
)

if not self.forcebuild and self.has_failed_before():
if not self.forcebuild and self.has_failed_before(build_method):
self.logger.info("Build has failed before, not re-attempting")
return BuildStatus.Skipped

Expand All @@ -561,7 +576,7 @@ def makebuildfor_by_method(
self.logger.debug(f"Number of valid library binaries {filesfound}")

if not self.install_context.dry_run:
self.save_build_logging(build_status, build_folder, source_folder)
self.save_build_logging(build_status, log_folder, source_folder, build_method)

if build_status == BuildStatus.Ok:
self.build_cleanup(build_folder)
Expand All @@ -583,6 +598,8 @@ def cache_cleanup(self):
if not self.install_context.dry_run:
for folder in self.cached_source_folders:
shutil.rmtree(folder, ignore_errors=True)
else:
self.logger.info(f"Would clean crate cache, but in dry-run mode")

def upload_builds(self):
if self.needs_uploading > 0:
Expand Down Expand Up @@ -611,51 +628,52 @@ def makebuild(self, buildfor):
if checkcompiler not in self.compilerprops:
self.logger.error(f"Unknown compiler {checkcompiler}")

for compiler in self.compilerprops:
if checkcompiler != "" and compiler != checkcompiler:
continue
with self.install_context.new_staging_dir() as source_staging:
for compiler in self.compilerprops:
if checkcompiler != "" and compiler != checkcompiler:
continue

if compiler in self.buildconfig.skip_compilers:
self.logger.debug(f"Skipping {compiler}")
continue
if compiler in self.buildconfig.skip_compilers:
self.logger.debug(f"Skipping {compiler}")
continue

if compiler in skip_compilers:
self.logger.debug(f"Skipping {compiler}")
continue
if compiler in skip_compilers:
self.logger.debug(f"Skipping {compiler}")
continue

# compiler_semver = version.parse(self.compilerprops[compiler]['semver'])
# if compiler_semver < min_compiler_version:
# self.logger.debug(f'Skipping {compiler} (too old)')
# continue
# compiler_semver = version.parse(self.compilerprops[compiler]['semver'])
# if compiler_semver < min_compiler_version:
# self.logger.debug(f'Skipping {compiler} (too old)')
# continue

if "compilerType" in self.compilerprops[compiler]:
compilerType = self.compilerprops[compiler]["compilerType"]
else:
raise RuntimeError(f"Something is wrong with {compiler}")

exe = self.compilerprops[compiler]["exe"]
options = self.compilerprops[compiler]["options"]
toolchain = ""

stdlibs = [""]
archs = build_supported_arch
stdvers = build_supported_stdver
ldPath = ""

for args in itertools.product(
build_supported_os, build_supported_buildtype, archs, stdvers, stdlibs, build_supported_flagscollection
):
buildstatus = self.makebuildfor(compiler, options, exe, compilerType, toolchain, *args, ldPath)
if buildstatus == BuildStatus.Ok:
builds_succeeded = builds_succeeded + 1
elif buildstatus == BuildStatus.Skipped:
builds_skipped = builds_skipped + 1
if "compilerType" in self.compilerprops[compiler]:
compilerType = self.compilerprops[compiler]["compilerType"]
else:
builds_failed = builds_failed + 1

if builds_succeeded > 0:
self.upload_builds()

self.cache_cleanup()
raise RuntimeError(f"Something is wrong with {compiler}")

exe = self.compilerprops[compiler]["exe"]
options = self.compilerprops[compiler]["options"]
toolchain = ""

stdlibs = [""]
archs = build_supported_arch
stdvers = build_supported_stdver
ldPath = ""

for args in itertools.product(
build_supported_os, build_supported_buildtype, archs, stdvers, stdlibs, build_supported_flagscollection
):
buildstatus = self.makebuildfor(compiler, options, exe, compilerType, toolchain, *args, ldPath, source_staging)
if buildstatus == BuildStatus.Ok:
builds_succeeded = builds_succeeded + 1
elif buildstatus == BuildStatus.Skipped:
builds_skipped = builds_skipped + 1
else:
builds_failed = builds_failed + 1

if builds_succeeded > 0:
self.upload_builds()

self.cache_cleanup()

return [builds_succeeded, builds_skipped, builds_failed]

0 comments on commit fb52c74

Please sign in to comment.