[AutoBuild] Split log files into a separate tarball (#1229)

* [AutoBuild] Split log files into a separate tarball Long term plan is still to do #778, but that's a lot of work, and we have little time at the moment. Simply splitting the logs into a separate tarball, instead, is a simpler short-term solution which has some very important bonuses: - the git tree hash of the main tarball should be reproducible (as long as you use the same toolchain) - which means we can start testing reproducibility of builds - rebuilding a package without changing its content won't create a new artifact: less strain on the Package Storage Server. * [AutoBuild] Filter out logs tarball when rebuilding the package
JuliaPackaging · Sep 19, 2022 · c1a0d6a · c1a0d6a
1 parent c3ab910
commit c1a0d6a
Show file tree

Hide file tree

Showing 4 changed files with 86 additions and 22 deletions.
diff --git a/Manifest.toml b/Manifest.toml
@@ -1,7 +1,8 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.7.3"
+julia_version = "1.8.1"
 manifest_format = "2.0"
+project_hash = "f35810f1518b3c855e1d8ed4f1522241fc6dcb7a"
 
 [[deps.ArgParse]]
 deps = ["Logging", "TextWrap"]
@@ -11,6 +12,7 @@ version = "1.1.4"
 
 [[deps.ArgTools]]
 uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
+version = "1.1.1"
 
 [[deps.Artifacts]]
 uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
@@ -31,11 +33,11 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[deps.BinaryBuilderBase]]
 deps = ["CodecZlib", "Downloads", "InteractiveUtils", "JSON", "LibGit2", "LibGit2_jll", "Libdl", "Logging", "OrderedCollections", "OutputCollectors", "Pkg", "ProgressMeter", "Random", "SHA", "Scratch", "SimpleBufferStream", "TOML", "Tar", "UUIDs", "p7zip_jll", "pigz_jll"]
-git-tree-sha1 = "4f53be732387e0b07975c968cc114f0824479fe9"
+git-tree-sha1 = "9b1c2e9866fdf353f925693d69aa440dec5b4c60"
 repo-rev = "master"
 repo-url = "https://github.com/JuliaPackaging/BinaryBuilderBase.jl.git"
 uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
-version = "1.13.0"
+version = "1.15.0"
 
 [[deps.CodecZlib]]
 deps = ["TranscodingStreams", "Zlib_jll"]
@@ -52,6 +54,7 @@ version = "3.46.0"
 [[deps.CompilerSupportLibraries_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
+version = "0.5.2+0"
 
 [[deps.DataAPI]]
 git-tree-sha1 = "fb5f5316dd3fd4c5e7c30a24d50643b73e37cd40"
@@ -76,8 +79,9 @@ deps = ["Random", "Serialization", "Sockets"]
 uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 
 [[deps.Downloads]]
-deps = ["ArgTools", "LibCURL", "NetworkOptions"]
+deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
 uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+version = "1.6.0"
 
 [[deps.ExprTools]]
 git-tree-sha1 = "56559bbef6ca5ea0c0818fa5c90320398a6fbf8d"
@@ -107,9 +111,9 @@ version = "5.7.3"
 
 [[deps.HTTP]]
 deps = ["Base64", "CodecZlib", "Dates", "IniFile", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"]
-git-tree-sha1 = "f0956f8d42a92816d2bf062f8a6a6a0ad7f9b937"
+git-tree-sha1 = "59ba44e0aa49b87a8c7a8920ec76f8afe87ed502"
 uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-version = "1.2.1"
+version = "1.3.3"
 
 [[deps.Hiccup]]
 deps = ["MacroTools", "Test"]
@@ -133,9 +137,9 @@ version = "1.0.0"
 
 [[deps.JLD2]]
 deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "Pkg", "Printf", "Reexport", "TranscodingStreams", "UUIDs"]
-git-tree-sha1 = "81b9477b49402b47fbe7f7ae0b252077f53e4a08"
+git-tree-sha1 = "6c38bbe47948f74d63434abed68bdfc8d2c46b99"
 uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
-version = "0.4.22"
+version = "0.4.23"
 
 [[deps.JLLWrappers]]
 deps = ["Preferences"]
@@ -158,10 +162,12 @@ version = "0.3.4"
 [[deps.LibCURL]]
 deps = ["LibCURL_jll", "MozillaCACerts_jll"]
 uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
+version = "0.6.3"
 
 [[deps.LibCURL_jll]]
 deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
 uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
+version = "7.84.0+0"
 
 [[deps.LibGit2]]
 deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
@@ -170,10 +176,12 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 [[deps.LibGit2_jll]]
 deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"]
 uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
+version = "1.3.0+0"
 
 [[deps.LibSSH2_jll]]
 deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
 uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
+version = "1.10.2+0"
 
 [[deps.Libdl]]
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
@@ -203,13 +211,14 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 
 [[deps.MbedTLS]]
 deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "Random", "Sockets"]
-git-tree-sha1 = "d9ab10da9de748859a7780338e1d6566993d1f25"
+git-tree-sha1 = "6872f9594ff273da6d13c7c1a1545d5a8c7d0c1c"
 uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
-version = "1.1.3"
+version = "1.1.6"
 
 [[deps.MbedTLS_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
+version = "2.28.0+0"
 
 [[deps.Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
@@ -222,6 +231,7 @@ version = "0.7.3"
 
 [[deps.MozillaCACerts_jll]]
 uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
+version = "2022.2.1"
 
 [[deps.Mustache]]
 deps = ["Printf", "Tables"]
@@ -237,6 +247,7 @@ version = "1.0.0"
 
 [[deps.NetworkOptions]]
 uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
+version = "1.2.0"
 
 [[deps.ObjectFile]]
 deps = ["Reexport", "StructIO"]
@@ -247,6 +258,7 @@ version = "0.3.7"
 [[deps.OpenBLAS_jll]]
 deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
 uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
+version = "0.3.20+0"
 
 [[deps.OrderedCollections]]
 git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c"
@@ -260,9 +272,9 @@ version = "0.1.1"
 
 [[deps.Parsers]]
 deps = ["Dates"]
-git-tree-sha1 = "0044b23da09b5608b4ecacb4e5e6c6332f833a7e"
+git-tree-sha1 = "3d5bf43e3e8b412656404ed9466f1dcbf7c50269"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "2.3.2"
+version = "2.4.0"
 
 [[deps.Pidfile]]
 deps = ["FileWatching", "Test"]
@@ -273,6 +285,7 @@ version = "1.3.0"
 [[deps.Pkg]]
 deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+version = "1.8.0"
 
 [[deps.PkgLicenses]]
 deps = ["Test"]
@@ -329,6 +342,7 @@ version = "1.3.0"
 
 [[deps.SHA]]
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+version = "0.7.0"
 
 [[deps.Scratch]]
 deps = ["Dates"]
@@ -374,6 +388,7 @@ version = "0.3.0"
 [[deps.TOML]]
 deps = ["Dates"]
 uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+version = "1.0.0"
 
 [[deps.TableTraits]]
 deps = ["IteratorInterfaceExtensions"]
@@ -383,13 +398,14 @@ version = "1.0.1"
 
 [[deps.Tables]]
 deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"]
-git-tree-sha1 = "5ce79ce186cc678bbb5c5681ca3379d1ddae11a1"
+git-tree-sha1 = "7149a60b01bf58787a1b83dad93f90d4b9afbe5d"
 uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
-version = "1.7.0"
+version = "1.8.1"
 
 [[deps.Tar]]
 deps = ["ArgTools", "SHA"]
 uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+version = "1.10.0"
 
 [[deps.Test]]
 deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
@@ -408,9 +424,9 @@ version = "0.3.0"
 
 [[deps.TranscodingStreams]]
 deps = ["Random", "Test"]
-git-tree-sha1 = "ed5d390c7addb70e90fd1eb783dcb9897922cbfa"
+git-tree-sha1 = "8a75929dcd3c38611db2f8d08546decb514fcadf"
 uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
-version = "0.9.8"
+version = "0.9.9"
 
 [[deps.URIs]]
 git-tree-sha1 = "e59ecc5a41b000fa94423a578d29290c7266fc10"
@@ -439,6 +455,7 @@ version = "4.3.4+0"
 [[deps.Zlib_jll]]
 deps = ["Libdl"]
 uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
+version = "1.2.12+3"
 
 [[deps.ghr_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -449,6 +466,7 @@ version = "0.14.0+0"
 [[deps.libblastrampoline_jll]]
 deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
 uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
+version = "5.1.1+0"
 
 [[deps.libsodium_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -459,10 +477,12 @@ version = "1.0.20+0"
 [[deps.nghttp2_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
+version = "1.48.0+0"
 
 [[deps.p7zip_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
+version = "17.4.0+0"
 
 [[deps.pigz_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]

diff --git a/Project.toml b/Project.toml
@@ -33,7 +33,7 @@ ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533"
 
 [compat]
 ArgParse = "1.1"
-BinaryBuilderBase = "1.12"
+BinaryBuilderBase = "1.15"
 GitHub = "5.1"
 HTTP = "0.8, 0.9, 1"
 JLD2 = "0.1.6, 0.2, 0.3, 0.4"

diff --git a/src/AutoBuild.jl b/src/AutoBuild.jl
@@ -43,6 +43,9 @@ function Base.show(io::IO, t::BuildTimer)
     end
 end
 
+exclude_logs(_, f) = f != "logs"
+only_logs(_, f)    = f == "logs"
+
 # Helper function to get the minimum version supported by the given compat
 # specification, given as a string.
 minimum_compat(compat::String) =
@@ -953,6 +956,18 @@ function autobuild(dir::AbstractString,
             platform=platform,
             verbose=verbose,
             force=true,
+            # Do not include logs into the main tarball
+            filter=exclude_logs,
+        )
+        # Create another tarball only for the logs
+        package(
+            dest_prefix,
+            joinpath(out_path, src_name * "-logs"),
+            src_version;
+            platform=platform,
+            verbose=verbose,
+            force=true,
+            filter=only_logs,
         )
         timer.end_package = time()
 
@@ -1112,11 +1127,15 @@ function rebuild_jll_package(name::String, build_version::VersionNumber, sources
     # We're going to recreate "build_output_meta"
     build_output_meta = Dict()
 
+    # For each platform, we have two tarballs: the main with the full product,
+    # and the logs-only one.  This function filters out the logs one.
+    filter_main_tarball(f, platform) = occursin(".$(triplet(platform)).tar", f) && !occursin("-logs.", f)
+
     # Then generate a JLL package for each platform
     downloaded_files = readdir(download_dir)
     for platform in sort(collect(platforms), by = triplet)
         # Find the corresponding tarball:
-        tarball_idx = findfirst([occursin(".$(triplet(platform)).tar", f) for f in downloaded_files])
+        tarball_idx = findfirst(f -> filter_main_tarball(f, platform), downloaded_files)
 
         # No tarball matching the given platform...
         if tarball_idx === nothing
@@ -1126,7 +1145,7 @@ function rebuild_jll_package(name::String, build_version::VersionNumber, sources
                 if isos(platform) && os_version(platform) === nothing
                     tmp_platform = deepcopy(platform)
                     tmp_platform["os_version"] = try_os_version
-                    tarball_idx = findfirst([occursin(".$(triplet(tmp_platform)).tar", f) for f in downloaded_files])
+                    tarball_idx = findfirst(f -> filter_main_tarball(f, tmp_platform), downloaded_files)
                 end
             end
         end

diff --git a/test/building.jl b/test/building.jl
@@ -180,6 +180,21 @@ end
 
 @testset "gfortran linking specialty flags" begin
     # We test things like linking against libgfortran with `$FC` on a couple of troublesome platforms
+    expected_git_shas = Dict(
+        v"4" => Dict(
+            Platform("x86_64", "linux"; libgfortran_version=v"3")  => Base.SHA1("6b705c0ff5760e236ab414fb9a366193f36d3e45"),
+            Platform("aarch64", "linux"; libgfortran_version=v"3") => Base.SHA1("fb501d0fb03bbf8f034cff50d8d8a1b2956cdf1a"),
+        ),
+        v"5" => Dict(
+            Platform("x86_64", "linux"; libgfortran_version=v"3")  => Base.SHA1("ef9653f47abba2895b14c1086ba40ad8c64ac098"),
+            Platform("aarch64", "linux"; libgfortran_version=v"3") => Base.SHA1("ed873d488f47fb8bf5b4b89bb31ed1cad27ee27c"),
+        ),
+        v"6" => Dict(
+            Platform("x86_64", "linux"; libgfortran_version=v"3")  => Base.SHA1("b3829adf90f6521e4a05f20924506c8358247e06"),
+            Platform("aarch64", "linux"; libgfortran_version=v"3") => Base.SHA1("641031469413f4fa776c74310098b1b181796720"),
+        ),
+    )
+
     for gcc_version in (v"4", v"5", v"6")
         mktempdir() do build_path
             build_output_meta = autobuild(
@@ -210,6 +225,11 @@ end
                 preferred_gcc_version=gcc_version,
             )
 
+            for p in (Platform("x86_64", "linux"; libgfortran_version=v"3"), Platform("aarch64", "linux"; libgfortran_version=v"3"))
+                # Test build reproducibility
+                @test build_output_meta[p][3] == expected_git_shas[gcc_version][p]
+            end
+
             # Just a simple test to ensure that it worked.
             @test length(keys(build_output_meta)) == 6
         end
@@ -218,6 +238,7 @@ end
     # Test that building something that links against gfortran suggests depending on CSL
     @test_logs (:warn, r"CompilerSupportLibraries_jll") match_mode=:any begin
         mktempdir() do build_path
+            p = Platform("x86_64", "linux"; libgfortran_version=v"3")
             build_output_meta = autobuild(
                 build_path,
                 "csl_dependency",
@@ -231,10 +252,12 @@ end
                 # Install fake license just to silence the warning
                 install_license /usr/share/licenses/libuv/LICENSE
                 """,
-                [Platform("x86_64", "linux"; libgfortran_version=v"3")],
+                [p],
                 [ExecutableProduct("hello_world_fortran", :hello_world_fortran)],
                 Dependency[],
             )
+            # Test build reproducibility
+            @test build_output_meta[p][3] == Base.SHA1("95e005d9b057b3a28af61189b9af5613127416a6")
         end
     end
 end
@@ -319,6 +342,7 @@ end
 
 @testset "AnyPlatform" begin
     mktempdir() do build_path
+        p = AnyPlatform()
         build_output_meta = autobuild(
             build_path,
             "header",
@@ -330,12 +354,13 @@ end
             touch ${includedir}/libqux.h
             install_license /usr/share/licenses/MIT
             """,
-            [AnyPlatform()],
+            [p],
             [FileProduct("include/libqux.h", :libqux_h)],
             # No dependencies
             Dependency[]
         )
-        @test haskey(build_output_meta, AnyPlatform())
+        @test haskey(build_output_meta, p)
+        @test build_output_meta[p][3] == Base.SHA1("45c55bfed92bd890d6487c58c4c03e07f5fb8829")
 
         # Test that having a LibraryProduct for AnyPlatform raises an error
         @test_throws ErrorException autobuild(