From a714518fbd7741093821ef30d76581922cd73dfe Mon Sep 17 00:00:00 2001 From: Timothy Middelkoop Date: Wed, 23 Oct 2024 15:05:00 -0500 Subject: [PATCH] Fixes for generating recipe.sh for Warewulf 4 * Contains a bugfix for HEREDOC handling Signed-off-by: Timothy Middelkoop --- .../install/common/add_ww4_hosts_intro.tex | 4 +-- .../install/common/add_ww4_hosts_slurm.tex | 5 ++++ .../common/import_ww4_files_ib_centos.tex | 5 ++-- .../warewulf4_add_to_compute_chroot_intro.tex | 4 +-- .../common/warewulf4_mkchroot_rocky.tex | 6 ++--- .../install/common/warewulf4_setup_centos.tex | 2 +- .../common/warewulf4_slurm_test_job.tex | 25 ++++--------------- docs/recipes/install/parse_doc.pl | 18 ++++++++----- .../rocky9/x86_64/warewulf4/slurm/steps.tex | 13 +++++----- 9 files changed, 39 insertions(+), 43 deletions(-) diff --git a/docs/recipes/install/common/add_ww4_hosts_intro.tex b/docs/recipes/install/common/add_ww4_hosts_intro.tex index 170b859ce1..1ce3a191f3 100644 --- a/docs/recipes/install/common/add_ww4_hosts_intro.tex +++ b/docs/recipes/install/common/add_ww4_hosts_intro.tex @@ -4,8 +4,8 @@ \begin{lstlisting}[language=bash,keywords={},upquote=true,basicstyle=\footnotesize\ttfamily,literate={BOSVER}{\baseos{}}1] # add the nodes with --discoverable so warewulf will identify new mac addresses [sms](*\#*) for ((i=0; i<$num_computes; i++)) ; do - wwctl node add --discoverable=yes --container=BOSVER \ - --ipaddr=${c_ip[$i]} --netmask=${internal_netmask} ${compute_prefix}$i +wwctl node add --discoverable=yes --container=rocky9.4 \ + --ipaddr=${c_ip[$i]} --netmask=${internal_netmask} ${compute_prefix}$i done \end{lstlisting} % end_ohpc_run diff --git a/docs/recipes/install/common/add_ww4_hosts_slurm.tex b/docs/recipes/install/common/add_ww4_hosts_slurm.tex index 2ba546f778..575b798853 100644 --- a/docs/recipes/install/common/add_ww4_hosts_slurm.tex +++ b/docs/recipes/install/common/add_ww4_hosts_slurm.tex @@ -9,3 +9,8 @@ [sms](*\#*) systemctl enable --now slurmctld \end{lstlisting} % end_ohpc_run + +% begin_ohpc_run +% ohpc_comment_header Allow for optional sleep to wait for provisioning to complete +% ohpc_command sleep ${provision_wait} +% end_ohpc_run diff --git a/docs/recipes/install/common/import_ww4_files_ib_centos.tex b/docs/recipes/install/common/import_ww4_files_ib_centos.tex index 7fe8f005a3..7a092e9b71 100644 --- a/docs/recipes/install/common/import_ww4_files_ib_centos.tex +++ b/docs/recipes/install/common/import_ww4_files_ib_centos.tex @@ -10,8 +10,9 @@ % ohpc_command if [[ ${enable_ipoib} -eq 1 ]];then % ohpc_indent 5 \begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true] -[sms](*\#*) wwsh file import /opt/ohpc/pub/examples/network/centos/ifcfg-ib0.ww -[sms](*\#*) wwsh -y file set ifcfg-ib0.ww --path=/etc/sysconfig/network-scripts/ifcfg-ib0 +[sms](*\#*) wwctl overlay mkdir generic /etc/sysconfig/network-scripts/ +[sms](*\#*) wwctl overlay import generic /opt/ohpc/pub/examples/network/centos/ifcfg-ib0.ww \ + /etc/sysconfig/network-scripts/ifcfg-ib0.ww \end{lstlisting} % ohpc_indent 0 % ohpc_command fi diff --git a/docs/recipes/install/common/warewulf4_add_to_compute_chroot_intro.tex b/docs/recipes/install/common/warewulf4_add_to_compute_chroot_intro.tex index eaa4b8a434..5c5fe5bbba 100644 --- a/docs/recipes/install/common/warewulf4_add_to_compute_chroot_intro.tex +++ b/docs/recipes/install/common/warewulf4_add_to_compute_chroot_intro.tex @@ -10,8 +10,8 @@ % ohpc_comment_header Add OpenHPC base components to compute image \ref{sec:add_components} \begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true,literate={BOSVER}{\baseos{}}1] # Install compute node base meta-package -[sms](*\#*) (*\containerinstall*) - (*\install*) ohpc-base-compute +[sms](*\#*) wwctl container exec rocky-9.4 /bin/bash <<- EOF + dnf -y install ohpc-base-compute /bin/false EOF \end{lstlisting} diff --git a/docs/recipes/install/common/warewulf4_mkchroot_rocky.tex b/docs/recipes/install/common/warewulf4_mkchroot_rocky.tex index 7b0f3a954c..7ee9134bd6 100644 --- a/docs/recipes/install/common/warewulf4_mkchroot_rocky.tex +++ b/docs/recipes/install/common/warewulf4_mkchroot_rocky.tex @@ -22,9 +22,9 @@ \subsubsection{Build initial BOS image} \label{sec:assemble_bos} [sms](*\#*) wwctl container import docker://ghcr.io/warewulf/warewulf-rockylinux:BOSTAG BOSVER --syncuser # Enable OpenHPC inside image and update container -[sms](*\#*) (*\containerinstall*) - (*\install*) http://repos.openhpc.community/OpenHPC/3/EL_9/x86_64/ohpc-release-3-1.el9.x86_64.rpm - (*\pkgmgr*) -y update +[sms](*\#*) wwctl container exec rocky-9.4 /bin/bash <<- EOF + dnf -y install http://repos.openhpc.community/OpenHPC/3/EL_9/x86_64/ohpc-release-3-1.el9.x86_64.rpm + dnf -y update /bin/false EOF diff --git a/docs/recipes/install/common/warewulf4_setup_centos.tex b/docs/recipes/install/common/warewulf4_setup_centos.tex index 6c3025fecf..608ecbd229 100644 --- a/docs/recipes/install/common/warewulf4_setup_centos.tex +++ b/docs/recipes/install/common/warewulf4_setup_centos.tex @@ -10,7 +10,7 @@ [sms](*\#*) perl -pi -e "s/netmask:.*/netmask: ${internal_netmask}/" /etc/warewulf/warewulf.conf [sms](*\#*) perl -pi -e "s/network:.*/network: ${internal_network}/" /etc/warewulf/warewulf.conf [sms](*\#*) perl -pi -e "s/range start:.*/range start: ${c_ip[0]}/" /etc/warewulf/warewulf.conf -[sms](*\#*) perl -pi -e "s/range start:.*/range start: ${c_ip[$((num_computes-1))]}/" /etc/warewulf/warewulf.conf +[sms](*\#*) perl -pi -e "s/range end:.*/range end: ${c_ip[$((num_computes-1))]}/" /etc/warewulf/warewulf.conf [sms](*\#*) perl -pi -e "s/mount: false/mount: true/" /etc/warewulf/warewulf.conf # Configure /etc/hostname on master and compute nodes diff --git a/docs/recipes/install/common/warewulf4_slurm_test_job.tex b/docs/recipes/install/common/warewulf4_slurm_test_job.tex index d2c16035d3..f08ba19434 100644 --- a/docs/recipes/install/common/warewulf4_slurm_test_job.tex +++ b/docs/recipes/install/common/warewulf4_slurm_test_job.tex @@ -9,16 +9,12 @@ % end_ohpc_run \Warewulf{} installs a utility on the compute nodes to automatically -synchronize known files from the provisioning server at five minute intervals. In this -recipe, recall that we previously registered credential files with Warewulf (e.g. passwd, -group, and shadow) so that these files would be propagated during compute node -imaging. However, with the addition of a new ``test'' user above, the files -have been outdated and we need to update the Warewulf database to incorporate -the additions. This re-sync process can be accomplished as follows: +synchronize overlay files from the provisioning server at one minute intervals. +To rebuild the overlay, run the following: % begin_ohpc_run \begin{lstlisting}[language=bash,keywords={}] -[sms](*\#*) wwsh file resync passwd shadow group +[sms](*\#*) wwctl overlay build \end{lstlisting} % end_ohpc_run @@ -26,20 +22,9 @@ % ohpc_command sleep 2 % end_ohpc_run -\begin{center} -\begin{tcolorbox}[] -\small After re-syncing to notify Warewulf of file modifications made on the {\em -master} host, it should take approximately 5 minutes for the changes to -propagate. However, you can also manually pull the changes from compute nodes -via the following: -% begin_ohpc_run -\begin{lstlisting}[language=bash,keywords={}] -[sms](*\#*) pdsh -w ${compute_prefix}[1-${num_computes}] /warewulf/bin/wwgetfiles -\end{lstlisting} -% end_ohpc_run -\end{tcolorbox} -\end{center} +master} host, it should take approximately one minute for the changes to +propagate. \input{common/prun} diff --git a/docs/recipes/install/parse_doc.pl b/docs/recipes/install/parse_doc.pl index a057b31785..c955e4f16f 100755 --- a/docs/recipes/install/parse_doc.pl +++ b/docs/recipes/install/parse_doc.pl @@ -36,6 +36,7 @@ # Determine BaseOS, arch, and define package manager commands my $BaseOS = ""; +my $BaseOSTag = ""; my $BaseOSshort = ""; my $OSimage = ""; my $arch = ""; @@ -81,6 +82,8 @@ $groupChrootInstall = $1; } elsif( $line =~ /\\newcommand\{\\baseos\}\{(.+)\}/ ) { $BaseOS = $1; + } elsif( $line =~ /\\newcommand\{\\baseostag\}\{(.+)\}/ ) { + $BaseOSTag = $1; } elsif( $line =~ /\\newcommand\{\\baseosshort\}\{(.+)\}/ ) { $BaseOSshort = $1; } elsif( $line =~ /\\newcommand\{\\osimage\}\{(.+)\}/ ) { @@ -105,6 +108,7 @@ # print "groupInstall = $groupInstall\n"; # print "groupChrootInstall = $groupChrootInstall\n"; # print "BaseOS = $BaseOS\n"; +# print "BaseOSTa = $BaseOSTag\n"; # print "VERLONG = $verlong\n"; # print "OSTree = $OSTree\n"; # print "IMAGE = $image\n"; @@ -169,20 +173,22 @@ # <<- indicates the HERE document will ignore leadings tabs (not spaces) } elsif( $_ =~ /$prompt (.+ <<-[ ]*([^ ]+).*)$/ ) { my $cmd = update_cmd($1); + chomp $cmd; my $here = $2; + chomp $here; # commands that begin with a % are for CI only next if( $_ =~ /^%/ && !$ci_run ); print $fh ' ' x $indent . "$cmd\n"; my $next_line; - do { - $next_line = ; - # trim leading and trailing space - $next_line =~ s/^\s+|\s+$//g; + do { + $next_line = ; + # trim leading and trailing space + $next_line =~ s/^\s+|\s+$//g; - print $fh "$next_line\n"; - } while( $next_line !~ /^$here/ ); + print $fh "$next_line\n"; + } while( $next_line !~ /^$here/ ); # handle commands line line continuation: prompt$ command \ } elsif( $_ =~ /$prompt (.+) \\$/ ) { diff --git a/docs/recipes/install/rocky9/x86_64/warewulf4/slurm/steps.tex b/docs/recipes/install/rocky9/x86_64/warewulf4/slurm/steps.tex index 0c3c8d93d6..4d103a99d4 100644 --- a/docs/recipes/install/rocky9/x86_64/warewulf4/slurm/steps.tex +++ b/docs/recipes/install/rocky9/x86_64/warewulf4/slurm/steps.tex @@ -28,7 +28,6 @@ \newcommand{\chrootclean}{dnf --installroot=\$CHROOT clean expire-cache} \newcommand{\install}{dnf -y install} \newcommand{\chrootinstall}{dnf -y --installroot=\$CHROOT install} -\newcommand{\containerinstall}{wwctl container exec \baseos{} /bin/bash <