Script improvements and fixes

- termux_step_get_dependencies.sh: don't use the `-s` flag to compile dependencies as need to check their dependencies and detect circular dependencies
- termux_step_setup_variables.sh: setting new values:
  - `TERMUX_PKG_ONLY_INSTALLING` - when set to true you indicate that the package can only be installed (this value may no longer be useful due to a changed dependency compilation method)
  - `TERMUX_PKG_SEPARATE_SUB_DEPENDS` - when set to true, subpackages will not be added to the dependency of the parent package
  - `TERMUX_PKG_ACCEPT_PKG_IN_DEP` - when set to true, the package can be in its own dependency list if necessary (useful for cyclic dependencies)
- buildorder.py:
  - Adding a new function `get_list_cyclic_dependencies` and with it a new flag `-l` which allows you to find cyclic dependencies
  - Rewrote the `generate_full_buildorder` function so that it can generate a build order with circular dependencies
  - Adding a new parameter `depend_on_parent` has been added for subpackages, which allows you to disable dependency on the parent package or have the same dependency as the parent
  - Retrieving dependencies from a dependency is disabled if `fast_build_mode` is not enabled
- build-all.sh:
  - Adding a new `-f` flag that allows you to control the packet format
  - The log creation method has been changed
  - Don't use the `-s` flag by default to allow `build-package.sh` to detect circular dependencies
  - Added checking for compiled packages so that if there are cyclic dependencies, they can be added to the list of ready packages.

build-all.sh

@@ -18,26 +18,28 @@ fi
 test -f "$HOME"/.termuxrc && . "$HOME"/.termuxrc
 : ${TERMUX_TOPDIR:="$HOME/.termux-build"}
 : ${TERMUX_ARCH:="aarch64"}
+: ${TERMUX_FORMAT:="debian"}
 : ${TERMUX_DEBUG_BUILD:=""}
-: ${TERMUX_INSTALL_DEPS:="-s"}
-# Set TERMUX_INSTALL_DEPS to -s unless set to -i
+: ${TERMUX_INSTALL_DEPS:=""}
 
 _show_usage() {
-	echo "Usage: ./build-all.sh [-a ARCH] [-d] [-i] [-o DIR]"
+	echo "Usage: ./build-all.sh [-a ARCH] [-d] [-i] [-o DIR] [-f FORMAT]"
 	echo "Build all packages."
 	echo "  -a The architecture to build for: aarch64(default), arm, i686, x86_64 or all."
 	echo "  -d Build with debug symbols."
 	echo "  -i Build dependencies."
 	echo "  -o Specify deb directory. Default: debs/."
+	echo "  -f Specify format pkg. Default: debian."
 	exit 1
 }
 
-while getopts :a:hdio: option; do
+while getopts :a:hdio:f: option; do
 case "$option" in
 	a) TERMUX_ARCH="$OPTARG";;
 	d) TERMUX_DEBUG_BUILD='-d';;
 	i) TERMUX_INSTALL_DEPS='-i';;
 	o) TERMUX_OUTPUT_DIR="$(realpath -m "$OPTARG")";;
+	f) TERMUX_FORMAT="$OPTARG";;
 	h) _show_usage;;
 	*) _show_usage >&2 ;;
 esac
@@ -50,6 +52,11 @@ if [[ ! "$TERMUX_ARCH" =~ ^(all|aarch64|arm|i686|x86_64)$ ]]; then
 	exit 1
 fi
 
+if [[ ! "$TERMUX_FORMAT" =~ ^(debian|pacman)$ ]]; then
+	echo "ERROR: Invalid format '$TERMUX_FORMAT'" 1>&2
+	exit 1
+fi
+
 BUILDSCRIPT=$(dirname "$0")/build-package.sh
 BUILDALL_DIR=$TERMUX_TOPDIR/_buildall-$TERMUX_ARCH
 BUILDORDER_FILE=$BUILDALL_DIR/buildorder.txt
@@ -65,28 +72,42 @@ if [ -e "$BUILDSTATUS_FILE" ]; then
 	echo "Continuing build-all from: $BUILDSTATUS_FILE"
 fi
 
-exec >	>(tee -a "$BUILDALL_DIR"/ALL.out)
-exec 2> >(tee -a "$BUILDALL_DIR"/ALL.err >&2)
-trap 'echo ERROR: See $BUILDALL_DIR/${PKG}.err' ERR
+exec &>	>(tee -a "$BUILDALL_DIR"/ALL.out)
+trap 'echo ERROR: See $BUILDALL_DIR/${PKG}.out' ERR
 
 while read -r PKG PKG_DIR; do
 	# Check build status (grepping is a bit crude, but it works)
-	if [ -e "$BUILDSTATUS_FILE" ] && grep "^$PKG\$" "$BUILDSTATUS_FILE" >/dev/null; then
+	if [ -e "$BUILDSTATUS_FILE" ] && grep -q "^$PKG\$" "$BUILDSTATUS_FILE"; then
 		echo "Skipping $PKG"
 		continue
 	fi
 
+	# Start building
+	if [ -n "${TERMUX_DEBUG_BUILD}" ]; then
+		echo "\"$BUILDSCRIPT\" -a \"$TERMUX_ARCH\" $TERMUX_DEBUG_BUILD --format \"$TERMUX_FORMAT\" ${TERMUX_OUTPUT_DIR+-o $TERMUX_OUTPUT_DIR} $TERMUX_INSTALL_DEPS \"$PKG_DIR\""
+	fi
 	echo -n "Building $PKG... "
 	BUILD_START=$(date "+%s")
-	bash -x "$BUILDSCRIPT" -a "$TERMUX_ARCH" $TERMUX_DEBUG_BUILD \
+	"$BUILDSCRIPT" -a "$TERMUX_ARCH" $TERMUX_DEBUG_BUILD --format "$TERMUX_FORMAT" \
 		${TERMUX_OUTPUT_DIR+-o $TERMUX_OUTPUT_DIR} $TERMUX_INSTALL_DEPS "$PKG_DIR" \
-		> "$BUILDALL_DIR"/"${PKG}".out 2> "$BUILDALL_DIR"/"${PKG}".err
+		&> "$BUILDALL_DIR"/"${PKG}".out
 	BUILD_END=$(date "+%s")
 	BUILD_SECONDS=$(( BUILD_END - BUILD_START ))
-	echo "done in $BUILD_SECONDS"
+	echo "done in $BUILD_SECONDS sec"
 
 	# Update build status
 	echo "$PKG" >> "$BUILDSTATUS_FILE"
+
+	# Check which packages were also compiled
+	if [ -z "$TERMUX_INSTALL_DEPS" ]; then
+		for build_pkg in $(ls -1 ~/.termux-build/ | grep -v -e '^_cache' -e '^_buildall'); do
+			if grep -q "^${build_pkg}\$" "$BUILDSTATUS_FILE"; then
+				continue
+			fi
+			echo "The \"${build_pkg}\" package was also compiled"
+			echo "$build_pkg" >> "$BUILDSTATUS_FILE"
+		done
+	fi
 done<"${BUILDORDER_FILE}"
 
 # Update build status

scripts/build/termux_step_get_dependencies.sh

@@ -132,7 +132,7 @@ termux_run_build-package() {
 		fi
 	fi
 	TERMUX_BUILD_IGNORE_LOCK=true ./build-package.sh \
- 		$(test "${TERMUX_INSTALL_DEPS}" = "true" && echo "-I" || echo "-s") \
+ 		$(test "${TERMUX_INSTALL_DEPS}" = "true" && echo "-I") \
  		$(test "${TERMUX_FORCE_BUILD_DEPENDENCIES}" = "true" && echo "-F") \
    		$(test "${TERMUX_WITHOUT_DEPVERSION_BINDING}" = "true" && echo "-w") \
      		--format $TERMUX_PACKAGE_FORMAT --library $set_library "${PKG_DIR}"

scripts/build/termux_step_setup_variables.sh

@@ -187,6 +187,9 @@ termux_step_setup_variables() {
 	TERMUX_PYTHON_HOME=$TERMUX_PREFIX/lib/python${TERMUX_PYTHON_VERSION} # location of python libraries
 	TERMUX_PKG_MESON_NATIVE=false
 	TERMUX_PKG_CMAKE_CROSSCOMPILING=true
+	TERMUX_PKG_ONLY_INSTALLING=false # when set to true you indicate that the package can only be installed
+	TERMUX_PKG_SEPARATE_SUB_DEPENDS=false # when set to true, subpackages will not be added to the dependency of the parent package
+	TERMUX_PKG_ACCEPT_PKG_IN_DEP=false # when set to true, the package can be in its own dependency list if necessary (useful for cyclic dependencies)
 
 	unset CFLAGS CPPFLAGS LDFLAGS CXXFLAGS
 	unset TERMUX_MESON_ENABLE_SOVERSION # setenv to enable SOVERSION suffix for shared libs built with Meson

scripts/buildorder.py

@@ -81,16 +81,19 @@ def parse_build_file_excluded_arches(path):
 
     return set(arches)
 
-def parse_build_file_variable_bool(path, var):
-    value = 'false'
+def parse_build_file_variable(path, var):
+    value = None
 
     with open(path, encoding="utf-8") as build_script:
         for line in build_script:
             if line.startswith(var):
                 value = line.split('=')[-1].replace('\n', '')
                 break
 
-    return value == 'true'
+    return value
+
+def parse_build_file_variable_bool(path, var):
+    return parse_build_file_variable(path, var) == 'true'
 
 def add_prefix_glibc_to_pkgname(name):
 	return name.replace("-static", "-glibc-static") if "static" == name.split("-")[-1] else name+"-glibc"
@@ -167,7 +170,8 @@ def recursive_dependencies(self, pkgs_map, dir_root=None):
                 dependency_package = pkgs_map[dependency_name]
                 if dependency_package.dir != dir_root and dependency_package.only_installing and not self.fast_build_mode:
                     continue
-                result += dependency_package.recursive_dependencies(pkgs_map, dir_root)
+                if self.fast_build_mode:
+                    result += dependency_package.recursive_dependencies(pkgs_map, dir_root)
                 if dependency_package.accept_dep_scr or dependency_package.dir != dir_root:
                     result += [dependency_package]
         return unique_everseen(result)
@@ -182,13 +186,18 @@ def __init__(self, subpackage_file_path, parent, virtual=False):
         if "gpkg" in subpackage_file_path.split("/")[-3].split("-") and "glibc" not in self.name.split("-"):
             self.name = add_prefix_glibc_to_pkgname(self.name)
         self.parent = parent
-        self.deps = set([parent.name])
         self.only_installing = parent.only_installing
         self.accept_dep_scr = parent.accept_dep_scr
+        self.depend_on_parent = parse_build_file_variable(subpackage_file_path, "TERMUX_SUBPKG_DEPEND_ON_PARENT") if not virtual else None
         self.excluded_arches = set()
+        self.deps = set()
         if not virtual:
             self.deps |= parse_build_file_dependencies(subpackage_file_path)
             self.excluded_arches |= parse_build_file_excluded_arches(subpackage_file_path)
+        if not self.depend_on_parent or self.depend_on_parent == "unversioned" or self.depend_on_parent == "true":
+            self.deps |= set([parent.name])
+        elif self.depend_on_parent == "deps":
+            self.deps |= parent.deps
         self.dir = parent.dir
 
         self.needed_by = set()  # Populated outside constructor, reverse of deps.
@@ -200,7 +209,7 @@ def recursive_dependencies(self, pkgs_map, dir_root=None):
         """All the dependencies of the subpackage, both direct and indirect.
         Only relevant when building in fast-build mode"""
         result = []
-        if dir_root == None:
+        if not dir_root:
             dir_root = self.dir
         for dependency_name in sorted(self.deps):
             if dependency_name == self.parent.name:
@@ -247,10 +256,7 @@ def read_packages_from_directories(directories, fast_build_mode, full_buildmode)
                         continue
                     if subpkg.name in pkgs_map:
                         die('Duplicated package: ' + subpkg.name)
-                    elif fast_build_mode:
-                        pkgs_map[subpkg.name] = subpkg
-                    else:
-                        pkgs_map[subpkg.name] = new_package
+                    pkgs_map[subpkg.name] = subpkg
                     all_packages.append(subpkg)
 
     for pkg in all_packages:
@@ -264,73 +270,49 @@ def read_packages_from_directories(directories, fast_build_mode, full_buildmode)
 
 def generate_full_buildorder(pkgs_map):
     "Generate a build order for building all packages."
-    build_order = []
-
-    # List of all TermuxPackages without dependencies
-    leaf_pkgs = [pkg for pkg in pkgs_map.values() if not pkg.deps]
+    # Identify packages that have no dependencies and are not subpackages.
+    pkgs_sort = [pkg.name for pkg in pkgs_map.values() if not pkg.deps and isinstance(pkg, TermuxPackage)]
 
-    if not leaf_pkgs:
+    if not pkgs_sort:
         die('No package without dependencies - where to start?')
 
-    # Sort alphabetically:
-    pkg_queue = sorted(leaf_pkgs, key=lambda p: p.name)
+    # Create a list `build_order` with packages from the list `pkgs_sort`.
+    build_order = [pkgs_map[pkg] for pkg in pkgs_sort]
 
-    # Topological sorting
-    visited = set()
+    # Get a list of cyclic dependencies for proper operation of package sorting.
+    cyclic_deps_list = get_list_cyclic_dependencies(pkgs_map, return_list=True)
 
-    # Tracks non-visited deps for each package
-    remaining_deps = {}
-    for name, pkg in pkgs_map.items():
-        remaining_deps[name] = set(pkg.deps)
-        for subpkg in pkg.subpkgs:
-            remaining_deps[subpkg.name] = set(subpkg.deps)
-
-    while pkg_queue:
-        pkg = pkg_queue.pop(0)
-        if pkg.name in visited:
-            continue
-
-        # print("Processing {}:".format(pkg.name), pkg.needed_by)
-        visited.add(pkg.name)
-        build_order.append(pkg)
-
-        for other_pkg in sorted(pkg.needed_by, key=lambda p: p.name):
-            # Remove this pkg from deps
-            remaining_deps[other_pkg.name].discard(pkg.name)
-            # ... and all its subpackages
-            remaining_deps[other_pkg.name].difference_update(
-                [subpkg.name for subpkg in pkg.subpkgs]
-            )
-
-            if not remaining_deps[other_pkg.name]:  # all deps were already appended?
-                pkg_queue.append(other_pkg)  # should be processed
-
-    if set(pkgs_map.values()) != set(build_order):
-        print("ERROR: Cycle exists. Remaining: ", file=sys.stderr)
-        for name, pkg in pkgs_map.items():
-            if pkg not in build_order:
-                print(name, remaining_deps[name], file=sys.stderr)
-
-        # Print cycles so we have some idea where to start fixing this.
-        def find_cycles(deps, pkg, path):
-            """Yield every dependency path containing a cycle."""
-            if pkg in path:
-                yield path + [pkg]
+    # Start sorting packages by dependencies.
+    while len(pkgs_sort) < len(pkgs_map):
+        for pkg in pkgs_map.values():
+            if pkg.name in pkgs_sort:
+                continue
+            is_subpackage = isinstance(pkg, TermuxSubPackage)
+            if is_subpackage and pkg.parent.name not in pkgs_sort:
+                # Since these packages will be compiled, the sorting should
+                # start with the package and then with its subpackages.
+                pkg = pkg.parent
+                is_subpackage = False
+            subpkgs = [subpkg.name for subpkg in (pkg.parent.subpkgs if is_subpackage else pkg.subpkgs)]
+            for dep in pkg.deps:
+                # First, package dependencies are checked for circular dependencies.
+                # If a package does not have a cyclic dependency, then the condition
+                # is triggered that the dependency is not in the `subpkgs` list and
+                # in the `pkgs_sort` list.  If the condition returns `True`, then the
+                # package is added to the `build_order` list (except for subpackages)
+                # and to the `pkgs_sort` list.
+                for cyclic in cyclic_deps_list:
+                    if pkg.name in cyclic and dep == cyclic[cyclic.index(pkg.name)+1]:
+                        break
+                else:
+                    if dep not in subpkgs and dep not in pkgs_sort:
+                        break
             else:
-                for dep in deps[pkg]:
-                    yield from find_cycles(deps, dep, path + [pkg])
-
-        cycles = set()
-        for pkg in remaining_deps:
-            for path_with_cycle in find_cycles(remaining_deps, pkg, []):
-                # Cut the path down to just the cycle.
-                cycle_start = path_with_cycle.index(path_with_cycle[-1])
-                cycles.add(tuple(path_with_cycle[cycle_start:]))
-        for cycle in sorted(cycles):
-            print(f"cycle: {' -> '.join(cycle)}", file=sys.stderr)
-
-        sys.exit(1)
+                if not is_subpackage:
+                    build_order.append(pkg)
+                pkgs_sort.append(pkg.name)
 
+    # Return sort result
     return build_order
 
 def generate_target_buildorder(target_path, pkgs_map, fast_build_mode):
@@ -347,19 +329,63 @@ def generate_target_buildorder(target_path, pkgs_map, fast_build_mode):
         package.deps.difference_update([subpkg.name for subpkg in package.subpkgs])
     return package.recursive_dependencies(pkgs_map)
 
+def get_list_cyclic_dependencies(pkgs_map, index=None, checked=None, pkgname=None, return_list=False):
+    "Find and print (or return a list of) circular dependencies for all packages or for one specified package."
+    # is_root - indicates that the function was run for the first time (called root function)
+    is_root = index == None
+    if is_root:
+        checked = []
+        index = []
+    result = []
+
+    # Start search
+    if is_root:
+        if pkgname:
+            range_pkgs = [pkgname]
+        else:
+            range_pkgs = pkgs_map.keys()
+    else:
+        range_pkgs = pkgs_map[index[-1]].deps
+    for pkg in range_pkgs:
+        if pkg in checked:
+            continue
+        index.append(pkg)
+        if index.count(pkg) == 2:
+            result.append(index.copy() if pkgname else index[index.index(pkg)::])
+        else:
+            result += get_list_cyclic_dependencies(pkgs_map, index, checked, pkgname)
+        del index[-1]
+        checked.append(pkg)
+
+    # return result
+    if is_root and not return_list:
+        # If launched in a root function and the `return_list` variable is set to False,
+        # then the result will be printed
+        if len(result) == 0:
+            print("No cyclic dependencies were found")
+        else:
+            print(f"Found {len(result)} cyclic dependencies:")
+            for cycle in result:
+                print("- "+" -> ".join(cycle))
+        sys.exit(0)
+    return result
+
 def main():
     "Generate the build order either for all packages or a specific one."
     import argparse
 
     parser = argparse.ArgumentParser(description='Generate order in which to build dependencies for a package. Generates')
     parser.add_argument('-i', default=False, action='store_true',
                         help='Generate dependency list for fast-build mode. This includes subpackages in output since these can be downloaded.')
+    parser.add_argument('-l', default=False, action='store_true',
+			help='Return a list of packages (including subpackages) that have a circular dependency.')
     parser.add_argument('package', nargs='?',
                         help='Package to generate dependency list for.')
     parser.add_argument('package_dirs', nargs='*',
                         help='Directories with packages. Can for example point to "../community-packages/packages". Note that the packages suffix is no longer added automatically if not present.')
     args = parser.parse_args()
     fast_build_mode = args.i
+    get_list_cirdep = args.l
     package = args.package
     packages_directories = args.package_dirs
 
@@ -368,6 +394,9 @@ def main():
     else:
         full_buildorder = False
 
+    if fast_build_mode and get_list_cirdep:
+        die('-i mode does not work for getting a list of circular dependencies')
+
     if fast_build_mode and full_buildorder:
         die('-i mode does not work when building all packages')
 
@@ -385,6 +414,12 @@ def main():
             packages_directories.insert(0, os.path.dirname(package))
     pkgs_map = read_packages_from_directories(packages_directories, fast_build_mode, full_buildorder)
 
+    if get_list_cirdep:
+        if full_buildorder:
+            get_list_cyclic_dependencies(pkgs_map)
+        else:
+            get_list_cyclic_dependencies(pkgs_map, pkgname=package.split("/")[-1])
+
     if full_buildorder:
         build_order = generate_full_buildorder(pkgs_map)
     else: