Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(disk): templates catalog #5914

Open
wants to merge 5 commits into
base: dev
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion internal/runner/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -571,7 +571,6 @@ func (r *Runner) RunEnumeration() error {
}
store.Load()
// TODO: remove below functions after v3 or update warning messages
disk.PrintDeprecatedPathsMsgIfApplicable(r.options.Silent)
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)

// add the hosts from the metadata queries of loaded templates into input provider
Expand Down
8 changes: 1 addition & 7 deletions pkg/catalog/disk/catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,7 @@ func NewFSCatalog(fs fs.FS, directory string) *DiskCatalog {
// It is used to read template and payload files based on catalog responses.
func (d *DiskCatalog) OpenFile(filename string) (io.ReadCloser, error) {
if d.templatesFS == nil {
file, err := os.Open(filename)
if err != nil {
if file, errx := os.Open(BackwardsCompatiblePaths(d.templatesDirectory, filename)); errx == nil {
return file, nil
}
}
return file, err
return os.Open(filename)
}

return d.templatesFS.Open(filename)
Expand Down
7 changes: 7 additions & 0 deletions pkg/catalog/disk/errors.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package disk

import "errors"

var (
ErrNoTemplatesFound = errors.New("no templates found in path")
)
126 changes: 40 additions & 86 deletions pkg/catalog/disk/find.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package disk

import (
"fmt"
"io/fs"
"os"
"path/filepath"
Expand Down Expand Up @@ -66,41 +67,34 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri
// or folders provided as in.
func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
processed := make(map[string]struct{})
// Template input includes a wildcard
if strings.Contains(target, "*") {
matches, findErr := c.findGlobPathMatches(target, processed)
if findErr != nil {
return nil, errors.Wrap(findErr, "could not find glob matches")
}
if len(matches) == 0 {
return nil, errors.Errorf("no templates found for path")
}
return matches, nil
}

// try to handle deprecated template paths
absPath := target
if c.templatesFS == nil {
absPath = BackwardsCompatiblePaths(c.templatesDirectory, target)
if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target {
if config.DefaultConfig.LogAllEvents {
gologger.DefaultLogger.Print().Msgf("[%v] requested Template path %s is deprecated, please update to %s\n", aurora.Yellow("WRN").String(), target, absPath)
}
deprecatedPathsCounter++
}

var err error
absPath, err = c.convertPathToAbsolute(absPath)
target, err = c.convertPathToAbsolute(target)
if err != nil {
return nil, errors.Wrapf(err, "could not find template file")
}
}

// Template input is either a file or a directory
match, file, err := c.findFileMatches(absPath, processed)
if strings.Contains(target, "*") {
globMatches, err := c.findGlobPathMatches(target, processed)
if err != nil {
return nil, errors.Wrap(err, "could not globbing path")
}

if len(globMatches) > 0 {
return globMatches, nil
} else {
return globMatches, fmt.Errorf("%w in path %q", ErrNoTemplatesFound, target)
}
}

// `target` is either a file or a directory
match, file, err := c.findFileMatches(target, processed)
if err != nil {
return nil, errors.Wrap(err, "could not find file")
}

if file {
if match != "" {
return []string{match}, nil
Expand All @@ -110,13 +104,15 @@ func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {

// Recursively walk down the Templates directory and run all
// the template file checks
matches, err := c.findDirectoryMatches(absPath, processed)
matches, err := c.findDirectoryMatches(target, processed)
if err != nil {
return nil, errors.Wrap(err, "could not find directory matches")
}

if len(matches) == 0 {
return nil, errors.Errorf("no templates found in path %s", absPath)
return nil, fmt.Errorf("%w in path %q", ErrNoTemplatesFound, target)
}

return matches, nil
}

Expand All @@ -136,87 +132,42 @@ func (c *DiskCatalog) convertPathToAbsolute(t string) (string, error) {

// findGlobPathMatches returns the matched files from a glob path
func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) {
// to support globbing on old paths we use brute force to find matches with exit on first match
// trim templateDir if any
relPath := strings.TrimPrefix(absPath, c.templatesDirectory)
// trim leading slash if any
relPath = strings.TrimPrefix(relPath, string(os.PathSeparator))

OldPathsResolver := func(inputGlob string) []string {
templateDir := c.templatesDirectory
if c.templatesDirectory == "" {
templateDir = "./"
}

if c.templatesFS == nil {
matches, _ := fs.Glob(os.DirFS(filepath.Join(templateDir, "http")), inputGlob)
if len(matches) != 0 {
return matches
}

// condition to support network cve related globs
matches, _ = fs.Glob(os.DirFS(filepath.Join(templateDir, "network")), inputGlob)
return matches
} else {
sub, err := fs.Sub(c.templatesFS, filepath.Join(templateDir, "http"))
if err != nil {
return nil
}
matches, _ := fs.Glob(sub, inputGlob)
if len(matches) != 0 {
return matches
}

// condition to support network cve related globs
sub, err = fs.Sub(c.templatesFS, filepath.Join(templateDir, "network"))
if err != nil {
return nil
}
matches, _ = fs.Glob(sub, inputGlob)
return matches
}
}

var matched []string
var err error
var matches []string
if c.templatesFS == nil {
var err error
matches, err = filepath.Glob(relPath)
if len(matches) != 0 {
matched = append(matched, matches...)
} else {
matched = append(matched, OldPathsResolver(relPath)...)
}
if err != nil && len(matched) == 0 {
return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
}
} else {
var err error

if c.templatesFS != nil {
matches, err = fs.Glob(c.templatesFS, relPath)
if len(matches) != 0 {
matched = append(matched, matches...)
} else {
matched = append(matched, OldPathsResolver(relPath)...)
if err != nil {
return matches, err
}
if err != nil && len(matched) == 0 {
return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
} else {
matches, err = filepath.Glob(absPath)
if err != nil {
return matches, err
}
}

results := make([]string, 0, len(matches))
for _, match := range matches {
if _, ok := processed[match]; !ok {
processed[match] = struct{}{}
results = append(results, match)
}
}

return results, nil
}

// findFileMatches finds if a path is an absolute file. If the path
// is a file, it returns true otherwise false with no errors.
func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
if c.templatesFS != nil {
absPath = strings.Trim(absPath, "/")
absPath = strings.Trim(absPath, string(filepath.Separator))
}
var info fs.File
if c.templatesFS == nil {
Expand Down Expand Up @@ -272,7 +223,7 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]
if absPath == "" {
absPath = "."
}
absPath = strings.TrimSuffix(absPath, "/")
absPath = strings.TrimSuffix(absPath, string(filepath.Separator))

err = fs.WalkDir(
c.templatesFS,
Expand All @@ -295,8 +246,11 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]
return results, err
}

// PrintDeprecatedPathsMsgIfApplicable prints a warning message if any deprecated paths are found
// Unless mode is silent warning message is printed
// PrintDeprecatedPathsMsgIfApplicable prints a warning message if any
// deprecated paths are found. Unless mode is silent warning message is printed.
//
// Deprecated: No longer used since the official Nuclei Templates repository
// have restructured this a long time ago.
func PrintDeprecatedPathsMsgIfApplicable(isSilent bool) {
if !updateutils.IsOutdated("v9.4.3", config.DefaultConfig.TemplateVersion) {
return
Expand Down
3 changes: 3 additions & 0 deletions pkg/catalog/disk/path.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ func (c *DiskCatalog) tryResolve(fullPath string) (string, error) {

// BackwardsCompatiblePaths returns new paths for all old/legacy template paths
// Note: this is a temporary function and will be removed in the future release
//
// Deprecated: No longer used since the official Nuclei Templates repository
// have restructured this a long time ago.
func BackwardsCompatiblePaths(templateDir string, oldPath string) string {
// TODO: remove this function in the future release
// 1. all http related paths are now moved at path /http
Expand Down
Loading