Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Generate etc/profile/init.sh early #800

Closed
wants to merge 1 commit into from
Closed

Generate etc/profile/init.sh early #800

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 77 additions & 69 deletions alibuild_helpers/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from os.path import abspath, exists, basename, dirname, join, realpath
from os import makedirs, unlink, readlink, rmdir
from textwrap import dedent
from alibuild_helpers import __version__
from alibuild_helpers.analytics import report_event
from alibuild_helpers.log import debug, error, info, banner, warning
Expand Down Expand Up @@ -32,6 +33,8 @@

import concurrent.futures
import importlib
import itertools
import json
import socket
import os
import re
Expand Down Expand Up @@ -894,76 +897,81 @@ def doBuild(args, parser):
# Notice that we guarantee that a dependency is always sourced before the
# parts depending on it, but we do not guaranteed anything for the order in
# which unrelated components are activated.
dependencies = "ALIBUILD_ARCH_PREFIX=\"${ALIBUILD_ARCH_PREFIX:-%s}\"\n" % args.architecture
dependenciesInit = "echo ALIBUILD_ARCH_PREFIX=\"\${ALIBUILD_ARCH_PREFIX:-%s}\" >> $INSTALLROOT/etc/profile.d/init.sh\n" % args.architecture
for dep in spec.get("requires", []):
depSpec = specs[dep]
depInfo = {
source_dependencies = "".join(dedent("""\
[ -n "${bigpackage}_REVISION" ] ||
. "$WORK_DIR/${{ALIBUILD_ARCH_PREFIX:-{architecture}}}"/{package}/{version}-{revision}/etc/profile.d/init.sh
""").format(
architecture=quote(args.architecture),
package=quote(dep),
version=quote(specs[dep]["version"]),
revision=quote(specs[dep]["revision"]),
bigpackage=dep.upper().replace("-", "_"),
) for dep in spec.get("requires", ()))

package_vars = dedent("""\
export {bigpackage}_ROOT="$WORK_DIR/$ALIBUILD_ARCH_PREFIX"/{package}/{version}-{revision}
export {bigpackage}_VERSION={version}
export {bigpackage}_REVISION={revision}
export {bigpackage}_HASH={pkghash}
export {bigpackage}_COMMIT={commit_hash}
""").format(
bigpackage=spec["package"].upper().replace("-", "_"),
package=quote(spec["package"]),
version=quote(spec["version"]),
revision=quote(spec["revision"]),
pkghash=quote(spec["hash"]),
commit_hash=quote(spec["commit_hash"]),
)

dependencies_json = json.dumps({
dep: {
"architecture": args.architecture,
"package": dep,
"version": depSpec["version"],
"revision": depSpec["revision"],
"bigpackage": dep.upper().replace("-", "_")
"version": specs[dep]["version"],
"revision": specs[dep]["revision"],
"hash": specs[dep]["hash"],
}
dependencies += format("[ -z ${%(bigpackage)s_REVISION+x} ] && source \"$WORK_DIR/$ALIBUILD_ARCH_PREFIX/%(package)s/%(version)s-%(revision)s/etc/profile.d/init.sh\"\n",
**depInfo)
dependenciesInit += format('echo [ -z \${%(bigpackage)s_REVISION+x} ] \&\& source \${WORK_DIR}/\${ALIBUILD_ARCH_PREFIX}/%(package)s/%(version)s-%(revision)s/etc/profile.d/init.sh >> \"$INSTALLROOT/etc/profile.d/init.sh\"\n',
**depInfo)
dependenciesDict = {}
for dep in spec.get("full_requires", []):
depSpec = specs[dep]
depInfo = {
"architecture": args.architecture,
"package": dep,
"version": depSpec["version"],
"revision": depSpec["revision"],
"hash": depSpec["hash"]
}
dependenciesDict[dep] = depInfo
dependenciesJSON = str(dependenciesDict)

# Generate the part which creates the environment for the package.
for dep in spec.get("full_requires", ())
})

# Generate the part of init.sh which creates the manually-specified build
# environment for subsequent packages.
# This can be either variable set via the "env" keyword in the metadata
# or paths which get appended via the "append_path" one.
# By default we append LD_LIBRARY_PATH, PATH
environment = ""
dieOnError(not isinstance(spec.get("env", {}), dict),
"Tag `env' in %s should be a dict." % p)
for key,value in spec.get("env", {}).items():
if key == "DYLD_LIBRARY_PATH":
continue
environment += format("echo 'export %(key)s=\"%(value)s\"' >> $INSTALLROOT/etc/profile.d/init.sh\n",
key=key,
value=value)
basePath = "%s_ROOT" % p.upper().replace("-", "_")

pathDict = spec.get("append_path", {})
dieOnError(not isinstance(pathDict, dict),
"Tag `append_path' in %s should be a dict." % p)
for pathName,pathVal in pathDict.items():
pathVal = isinstance(pathVal, list) and pathVal or [ pathVal ]
if pathName == "DYLD_LIBRARY_PATH":
continue
environment += format("\ncat << \EOF >> \"$INSTALLROOT/etc/profile.d/init.sh\"\nexport %(key)s=$%(key)s:%(value)s\nEOF",
key=pathName,
value=":".join(pathVal))

# Same thing, but prepending the results so that they win against system ones.
defaultPrependPaths = { "LD_LIBRARY_PATH": "$%s/lib" % basePath,
"PATH": "$%s/bin" % basePath }
pathDict = spec.get("prepend_path", {})
dieOnError(not isinstance(pathDict, dict),
"Tag `prepend_path' in %s should be a dict." % p)
for pathName,pathVal in pathDict.items():
pathDict[pathName] = isinstance(pathVal, list) and pathVal or [ pathVal ]
for pathName,pathVal in defaultPrependPaths.items():
pathDict[pathName] = [ pathVal ] + pathDict.get(pathName, [])
for pathName,pathVal in pathDict.items():
if pathName == "DYLD_LIBRARY_PATH":
continue
environment += format("\ncat << \EOF >> \"$INSTALLROOT/etc/profile.d/init.sh\"\nexport %(key)s=%(value)s${%(key)s+:$%(key)s}\nEOF",
key=pathName,
value=":".join(pathVal))
# or paths which get appended via the "{append,prepend}_path" keywords.
# By default we prepend to LD_LIBRARY_PATH, PATH.
for key in ("env", "append_path", "prepend_path"):
dieOnError(not isinstance(spec.get(key, {}), dict),
"Value for key `%s' in %s should be a dict." % (key, p))
pkgroot = "$%s_ROOT" % p.upper().replace("-", "_")
default_prepend_paths = {
"LD_LIBRARY_PATH": "%s/lib" % pkgroot,
"PATH": "%s/bin" % pkgroot,
}
environment = "".join(itertools.chain((
# Explicitly-set variables using the env: key.
'export {var}="{value}"\n'.format(var=var, value=value)
for var, value in spec.get("env", {}).items()
if var != "DYLD_LIBRARY_PATH"
), (
# Append to path variables using append_path: key.
'export {var}="${{{var}:+${var}:}}{value}"\n'.format(
var=var,
value=":".join(value) if isinstance(value, list) else value,
)
for var, value in spec.get("append_path", {}).items()
if var != "DYLD_LIBRARY_PATH"
), (
# Prepend to path variables using prepend_path: key. Handle default
# prepend_path entries first so that any recipe-specified entries for
# the same path come before them in the final path value.
'export {var}="{value}${{{var}+:${var}}}"\n'.format(
var=var,
value=":".join(value) if isinstance(value, list) else value,
)
for prepend_path in (default_prepend_paths, spec.get("prepend_path", {}))
for var, value in prepend_path.items()
if var != "DYLD_LIBRARY_PATH"
)))

# The actual build script.
referenceStatement = ""
Expand Down Expand Up @@ -999,9 +1007,9 @@ def doBuild(args, parser):


cmd = format(cmd_raw,
dependencies=dependencies,
dependenciesInit=dependenciesInit,
dependenciesJSON=dependenciesJSON,
source_dependencies=source_dependencies,
package_vars=package_vars,
dependencies_json=dependencies_json,
develPrefix=develPrefix,
environment=environment,
workDir=workDir,
Expand Down
42 changes: 19 additions & 23 deletions alibuild_helpers/build_template.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ function hash() { true; }
export WORK_DIR="${WORK_DIR_OVERRIDE:-%(workDir)s}"

# From our dependencies
%(dependencies)s
%(source_dependencies)s

# Insert our own wrapper scripts into $PATH, patched to use the system OpenSSL,
# instead of the one we build ourselves.
Expand Down Expand Up @@ -84,6 +84,24 @@ if [[ $DEVEL_PREFIX ]]; then
ln -snf $PKGHASH $WORK_DIR/BUILD/$PKGNAME-latest-$DEVEL_PREFIX
fi

# Create etc/profile/init.sh early, to help with debugging failed builds.
# If we're unpacking a tarball, instead of building a package, the init.sh
# from the tarball will replace the one we create here, but that should be
# safe.
mkdir -p "$INSTALLROOT/etc/profile.d"
cat <<\EOF > "$INSTALLROOT/etc/profile.d/init.sh"
# Source dependencies' init.sh.
%(source_dependencies)s
# Set up variables pointing to this package.
%(package_vars)s
# Environment variables set by this alidist recipe.
%(environment)s
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This might actually be the reason why we did not do this before. Environment variables created by a package should be effective only after the build. Does that change the behaviour?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, it doesn't -- this block is inside the heredoc, so it only creates init.sh. It doesn't source it. Further up, only %(source_dependencies)s is executed.

The problem is that the order used to be "unpack tarball" -> "overwrite init.sh" (so the newly-generated init.sh wins); this changes it to "create init.sh" -> "unpack tarball, including init.sh, over the top" (i.e. tarball wins). But as I say in the comment, the files ought to be the same anyway I think...

EOF
echo "$PKGHASH" > "$INSTALLROOT/.build-hash"
cat <<\EOF > "$INSTALLROOT/.full-dependencies"
%(dependencies_json)s
EOF

# Reference statements
%(referenceStatement)s
%(gitOptionsStatement)s
Expand Down Expand Up @@ -155,25 +173,6 @@ else
fi

cd "$WORK_DIR/INSTALLROOT/$PKGHASH"
echo "$PKGHASH" > "$INSTALLROOT/.build-hash"
cat <<\EOF | tr \' \" >"$INSTALLROOT/.full-dependencies"
%(dependenciesJSON)s
EOF

mkdir -p "$INSTALLROOT/etc/profile.d"
BIGPKGNAME=`echo "$PKGNAME" | tr [:lower:] [:upper:] | tr - _`
rm -f "$INSTALLROOT/etc/profile.d/init.sh"

# Init our dependencies
%(dependenciesInit)s

cat << EOF >> $INSTALLROOT/etc/profile.d/init.sh
export ${BIGPKGNAME}_ROOT=\${WORK_DIR}/\${ALIBUILD_ARCH_PREFIX}/$PKGNAME/$PKGVERSION-$PKGREVISION
export ${BIGPKGNAME}_VERSION=$PKGVERSION
export ${BIGPKGNAME}_REVISION=$PKGREVISION
export ${BIGPKGNAME}_HASH=$PKGHASH
export ${BIGPKGNAME}_COMMIT=${COMMIT_HASH}
EOF

# Add support for direnv https://github.com/direnv/direnv/
#
Expand All @@ -192,9 +191,6 @@ source_up
unset DYLD_LIBRARY_PATH
EOF

# Environment
%(environment)s

cd "$WORK_DIR/INSTALLROOT/$PKGHASH/$PKGPATH"
# Find which files need relocation.
{ grep -I -H -l -R "\(INSTALLROOT/$PKGHASH\|[@][@]PKGREVISION[@]$PKGHASH[@][@]\)" . || true; } | sed -e 's|^\./||' > "$INSTALLROOT/.original-unrelocated"
Expand Down
Loading