From 2233d22ed2d199869a98d50b9fd9bd1f54fa3fab Mon Sep 17 00:00:00 2001 From: Joe Pappano Date: Wed, 26 Jul 2023 16:13:59 -0400 Subject: [PATCH] t3c to use package metadata if rpm db is unhealthy (#7652) * adding check to verify rpmdb * added check to verify rpm db * rebase and fix conflicts * if rpmdb is unhealthy use package data in metadata * requested changes addressed. * added changelog entry * fixed spelling error and added GoDoc commnet * fixed formatting error --- CHANGELOG.md | 1 + cache-config/t3c-apply/config/config.go | 69 +++++++++++++++---- cache-config/t3c-apply/t3c-apply.go | 10 +-- cache-config/t3c-apply/torequest/torequest.go | 47 ++++++++++++- 4 files changed, 110 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55f97ce1e2..134e3eabe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - [#7619](https://github.com/apache/trafficcontrol/pull/7619) Traffic Ops* added optional field `oauth_user_attribute` for OAuth login credentials - [#7641](https://github.com/apache/trafficcontrol/pull/7641) *Traffic Router* Added further optimization to TR's algorithm of figuring out the zone for an incoming request. - [#7646](https://github.com/apache/trafficcontrol/pull/7646) *Traffic Portal* Add the ability to delete a cert. +- [#7652](https://github.com/apache/trafficcontrol/pull/7652) *t3c* added rpmdb checks and use package data from t3c-apply-metadata.json if rpmdb is corrupt ### Changed - [#7584](https://github.com/apache/trafficcontrol/pull/7584) *Documentation* Upgrade Traffic Control Sphinx documentation Makefile OS intelligent. diff --git a/cache-config/t3c-apply/config/config.go b/cache-config/t3c-apply/config/config.go index 64a62a0160..e7956e1eba 100644 --- a/cache-config/t3c-apply/config/config.go +++ b/cache-config/t3c-apply/config/config.go @@ -27,6 +27,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "time" @@ -81,6 +82,7 @@ type Cfg struct { SvcManagement SvcManagement Retries int ReverseProxyDisable bool + RpmDBOk bool SkipOSCheck bool UseStrategies t3cutil.UseStrategiesFlag TOInsecure bool @@ -188,6 +190,29 @@ func directoryExists(dir string) (bool, os.FileInfo) { return info.IsDir(), info } +const rpmDir = "/var/lib/rpm" + +// verifies the rpm database files. if there is any database corruption +// it will return false +func verifyRpmDB() bool { + exclude := regexp.MustCompile(`(^\.|^__)`) + dbFiles, err := os.ReadDir(rpmDir) + if err != nil { + return false + } + for _, file := range dbFiles { + if exclude.Match([]byte(file.Name())) { + continue + } + cmd := exec.Command("/usr/lib/rpm/rpmdb_verify", rpmDir+"/"+file.Name()) + err := cmd.Run() + if err != nil || cmd.ProcessState.ExitCode() > 0 { + return false + } + } + return true +} + // derives the ATS Installation directory from // the rpm config file list. func GetTSPackageHome() string { @@ -322,10 +347,11 @@ If any of the related flags are also set, they override the mode's default behav // so we want to log what flags the mode set here, to aid debugging. // But we can't do that until the loggers are initialized. modeLogStrs := []string{} + fatalLogStrs := []string{} if getopt.IsSet(runModeFlagName) { runMode := t3cutil.StrToMode(*runModePtr) if runMode == t3cutil.ModeInvalid { - return Cfg{}, errors.New(*runModePtr + " is an invalid mode.") + fatalLogStrs = append(fatalLogStrs, *runModePtr+" is an invalid mode.") } modeLogStrs = append(modeLogStrs, "t3c-apply is running in "+runMode.String()+" mode") switch runMode { @@ -411,7 +437,7 @@ If any of the related flags are also set, they override the mode's default behav } if *verbosePtr > 2 { - return Cfg{}, errors.New("Too many verbose options. The maximum log verbosity level is 2 (-vv or --verbose=2) for errors (0), warnings (1), and info (2)") + fatalLogStrs = append(fatalLogStrs, "Too many verbose options. The maximum log verbosity level is 2 (-vv or --verbose=2) for errors (0), warnings (1), and info (2)") } var cacheHostName string @@ -420,7 +446,7 @@ If any of the related flags are also set, they override the mode's default behav } else { cacheHostName, err = os.Hostname() if err != nil { - return Cfg{}, errors.New("Could not get the hostname from the O.S., please supply a hostname: " + err.Error()) + fatalLogStrs = append(fatalLogStrs, "Could not get the hostname from the O.S., please supply a hostname: "+err.Error()) } // strings.Split always returns a slice with at least 1 element, so we don't need a len check cacheHostName = strings.Split(cacheHostName, ".")[0] @@ -429,7 +455,7 @@ If any of the related flags are also set, they override the mode's default behav useGit := StrToUseGitFlag(*useGitStr) if useGit == UseGitInvalid { - return Cfg{}, errors.New("Invalid git flag '" + *useGitStr + "'. Valid options are yes, no, auto.") + fatalLogStrs = append(fatalLogStrs, "Invalid git flag '"+*useGitStr+"'. Valid options are yes, no, auto.") } retries := *retriesPtr @@ -471,6 +497,17 @@ If any of the related flags are also set, they override the mode's default behav os.Setenv("TO_PASS", toPass) } + rpmDBisOk := verifyRpmDB() + + if *installPackagesPtr && !rpmDBisOk { + if t3cutil.StrToMode(*runModePtr) == t3cutil.ModeBadAss { + fatalLogStrs = append(fatalLogStrs, "RPM database check failed unable to install packages cannot continue in badass mode") + } else { + fatalLogStrs = append(fatalLogStrs, "RPM database check failed unable to install packages cannot continue") + } + } + + toInfoLog = append(toInfoLog, fmt.Sprintf("rpm database is ok: %t", rpmDBisOk)) // set TSHome var tsHome = "" if *tsHomePtr != "" { @@ -481,13 +518,13 @@ If any of the related flags are also set, they override the mode's default behav tsHome = os.Getenv("TS_HOME") // check for the environment variable. if tsHome != "" { toInfoLog = append(toInfoLog, fmt.Sprintf("set TSHome from TS_HOME environment variable '%s'\n", TSHome)) - } else { // finally check using the config file listing from the rpm package. + } else if rpmDBisOk { // check using the config file listing from the rpm package if rpmdb is ok. tsHome = GetTSPackageHome() if tsHome != "" { toInfoLog = append(toInfoLog, fmt.Sprintf("set TSHome from the RPM config file list '%s'\n", TSHome)) - } else { - toInfoLog = append(toInfoLog, fmt.Sprintf("no override for TSHome was found, using the configured default: '%s'\n", TSHome)) } + } else if tsHome == "" { + toInfoLog = append(toInfoLog, fmt.Sprintf("no override for TSHome was found, using the configured default: '%s'\n", TSHome)) } } @@ -503,23 +540,23 @@ If any of the related flags are also set, they override the mode's default behav if *useLocalATSVersionPtr { atsVersionStr, err = GetATSVersionStr(tsHome) if err != nil { - return Cfg{}, errors.New("getting local ATS version: " + err.Error()) + fatalLogStrs = append(fatalLogStrs, "getting local ATS version: "+err.Error()) } } toInfoLog = append(toInfoLog, fmt.Sprintf("ATSVersionStr: '%s'\n", atsVersionStr)) usageStr := "basic usage: t3c-apply --traffic-ops-url=myurl --traffic-ops-user=myuser --traffic-ops-password=mypass --cache-host-name=my-cache" if strings.TrimSpace(toURL) == "" { - return Cfg{}, errors.New("Missing required argument --traffic-ops-url or TO_URL environment variable. " + usageStr) + fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-url or TO_URL environment variable. "+usageStr) } if strings.TrimSpace(toUser) == "" { - return Cfg{}, errors.New("Missing required argument --traffic-ops-user or TO_USER environment variable. " + usageStr) + fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-user or TO_USER environment variable. "+usageStr) } if strings.TrimSpace(toPass) == "" { - return Cfg{}, errors.New("Missing required argument --traffic-ops-password or TO_PASS environment variable. " + usageStr) + fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-password or TO_PASS environment variable. "+usageStr) } if strings.TrimSpace(cacheHostName) == "" { - return Cfg{}, errors.New("Missing required argument --cache-host-name. " + usageStr) + fatalLogStrs = append(fatalLogStrs, "Missing required argument --cache-host-name. "+usageStr) } toURLParsed, err := url.Parse(toURL) @@ -540,6 +577,7 @@ If any of the related flags are also set, they override the mode's default behav CacheHostName: cacheHostName, SvcManagement: svcManagement, Retries: retries, + RpmDBOk: rpmDBisOk, ReverseProxyDisable: reverseProxyDisable, SkipOSCheck: skipOsCheck, UseStrategies: useStrategies, @@ -580,6 +618,13 @@ If any of the related flags are also set, they override the mode's default behav return Cfg{}, errors.New("Initializing loggers: " + err.Error() + "\n") } + if len(fatalLogStrs) > 0 { + for _, str := range fatalLogStrs { + str = strings.TrimSpace(str) + log.Errorln(str) + } + return Cfg{}, errors.New("fatal error has occurred") + } for _, str := range modeLogStrs { str = strings.TrimSpace(str) if str == "" { diff --git a/cache-config/t3c-apply/t3c-apply.go b/cache-config/t3c-apply/t3c-apply.go index bc98fcf23a..24e1e8e91a 100644 --- a/cache-config/t3c-apply/t3c-apply.go +++ b/cache-config/t3c-apply/t3c-apply.go @@ -22,7 +22,6 @@ package main import ( "encoding/json" "errors" - "fmt" "io/ioutil" "os" "path/filepath" @@ -94,8 +93,8 @@ func Main() int { var lock util.FileLock cfg, err := config.GetCfg(Version, GitRevision) if err != nil { - fmt.Println(err) - fmt.Println(FailureExitMsg) + log.Infoln(err) + log.Errorln(FailureExitMsg) return ExitCodeConfigError } else if cfg == (config.Cfg{}) { // user used the --help option return ExitCodeSuccess @@ -261,7 +260,7 @@ func Main() int { // make sure we got the data necessary to check packages log.Infoln("======== Didn't get all files, no package processing needed or possible ========") metaData.InstalledPackages = oldMetaData.InstalledPackages - } else { + } else if cfg.RpmDBOk { log.Infoln("======== Start processing packages ========") err = trops.ProcessPackages() if err != nil { @@ -276,6 +275,9 @@ func Main() int { log.Errorf("Error verifying system services: %s\n", err.Error()) return GitCommitAndExit(ExitCodeServicesError, FailureExitMsg, cfg, metaData, oldMetaData) } + } else { + log.Warnln("======== RPM DB checks failed, package processing not possible, using installed packages from metadata if available========") + trops.ProcessPackagesWithMetaData(oldMetaData.InstalledPackages) } log.Debugf("Preparing to fetch the config files for %s, files: %s, syncdsUpdate: %s\n", cfg.CacheHostName, cfg.Files, syncdsUpdate) diff --git a/cache-config/t3c-apply/torequest/torequest.go b/cache-config/t3c-apply/torequest/torequest.go index 1b6016abf1..5298091f6a 100644 --- a/cache-config/t3c-apply/torequest/torequest.go +++ b/cache-config/t3c-apply/torequest/torequest.go @@ -587,10 +587,14 @@ func (r *TrafficOpsReq) CheckSystemServices() error { func (r *TrafficOpsReq) IsPackageInstalled(name string) bool { for k, v := range r.Pkgs { if strings.HasPrefix(k, name) { + log.Infof("Found in cache for '%s'", k) return v } } - + if !r.Cfg.RpmDBOk { + log.Warnf("RPM DB is corrupted cannot run IsPackageInstalled for '%s' and package metadata is unavailable", name) + return false + } log.Infof("IsPackageInstalled '%v' not found in cache, querying rpm", name) pkgArr, err := util.PackageInfo("pkg-query", name) if err != nil { @@ -1030,6 +1034,47 @@ func (r *TrafficOpsReq) ProcessPackages() error { return nil } +func pkgMetaDataToMap(pmd []string) map[string]bool { + pkgMap := map[string]bool{} + for _, pkg := range pmd { + pkgMap[pkg] = true + } + return pkgMap +} + +func pkgMatch(pkgMetaData []string, pk string) bool { + for _, pkg := range pkgMetaData { + if strings.Contains(pk, pkg) { + return true + } + } + return false + +} + +// ProcessPackagesWithMetaData will attempt to get installed package data from +// t3c-apply-metadata.json and log the results. +func (r *TrafficOpsReq) ProcessPackagesWithMetaData(packageMetaData []string) error { + pkgs, err := getPackages(r.Cfg) + pkgMdataMap := pkgMetaDataToMap(packageMetaData) + if err != nil { + return fmt.Errorf("getting packages: %w", err) + } + for _, pkg := range pkgs { + fullPackage := pkg.Name + "-" + pkg.Version + if pkgMdataMap[fullPackage] { + log.Infof("package %s is assumed to be installed according to metadata file", fullPackage) + r.Pkgs[fullPackage] = true + } else if pkgMatch(packageMetaData, pkg.Name) { + log.Infof("package %s is assumed to be installed according to metadata, but doesn't match traffic ops pkg", fullPackage) + r.Pkgs[fullPackage] = true + } else { + log.Infof("package %s does not appear to be installed.", pkg.Name+"-"+pkg.Version) + } + } + return nil +} + func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData) (UpdateStatus, error) { updateStatus, err := r.CheckRevalidateState(true) if err != nil {