Add fields whitelist in mask plugin (#625)

* Add fields whitelist in mask plugin * Replace short field names with long field names --------- Co-authored-by: george pogosyan <[email protected]>
ozontech · Jun 5, 2024 · 8358364 · 8358364
1 parent 3485123
commit 8358364
Show file tree

Hide file tree

Showing 5 changed files with 323 additions and 48 deletions.
diff --git a/plugin/README.md b/plugin/README.md
@@ -420,8 +420,7 @@ pipelines:
       ignore_fields:
       - trace_id
       masks:
-      - mask:
-        re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b"
+      - re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b"
         groups: [1,2,3]
     ...
 ```

diff --git a/plugin/action/README.md b/plugin/action/README.md
@@ -251,8 +251,7 @@ pipelines:
       ignore_fields:
       - trace_id
       masks:
-      - mask:
-        re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b"
+      - re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b"
         groups: [1,2,3]
     ...
 ```

diff --git a/plugin/action/mask/README.md b/plugin/action/mask/README.md
@@ -13,8 +13,7 @@ pipelines:
       ignore_fields:
       - trace_id
       masks:
-      - mask:
-        re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b"
+      - re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b"
         groups: [1,2,3]
     ...
 ```
@@ -44,9 +43,22 @@ If any mask has been applied then `mask_applied_field` will be set to `mask_appl
 
 <br>
 
-**`ignore_fields`** *`[]string`* 
+**`ignore_fields`** *`[]cfg.FieldSelector`* 
 
-List of the ignored event fields (including nested fields).
+List of the ignored event fields.
+If name of some field contained in this list
+then all nested fields will be ignored (even if they are not listed).
+
+<br>
+
+**`process_fields`** *`[]cfg.FieldSelector`* 
+
+List of the processed event fields.
+If name of some field contained in this list
+then all nested fields will be processed (even if they are not listed).
+If ignored fields list is empty and processed fields list is empty
+we consider this as empty ignored fields list (all fields will be processed).
+It is wrong to set non-empty ignored fields list and non-empty processed fields list at the same time.
 
 <br>
 

diff --git a/plugin/action/mask/mask.go b/plugin/action/mask/mask.go
@@ -2,6 +2,7 @@ package mask
 
 import (
 	"regexp"
+	"slices"
 	"strings"
 	"unicode/utf8"
 
@@ -30,8 +31,7 @@ pipelines:
       ignore_fields:
       - trace_id
       masks:
-      - mask:
-        re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b"
+      - re: "\b(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\D?(\d{1,4})\b"
         groups: [1,2,3]
     ...
 ```
@@ -52,8 +52,11 @@ type Plugin struct {
 	maskBuf []byte
 
 	// common match regex
-	matchRe       *regexp.Regexp
-	ignoredFields map[string]struct{}
+	matchRe *regexp.Regexp
+
+	fieldPaths   [][]string
+	isWhitelist  bool
+	ignoredNodes []*insaneJSON.Node
 
 	valueNodes []*insaneJSON.Node
 	logger     *zap.Logger
@@ -86,8 +89,20 @@ type Config struct {
 
 	// > @3@4@5@6
 	// >
-	// > List of the ignored event fields (including nested fields).
-	IgnoreFields []string `json:"ignore_fields"` // *
+	// > List of the ignored event fields.
+	// > If name of some field contained in this list
+	// > then all nested fields will be ignored (even if they are not listed).
+	IgnoreFields []cfg.FieldSelector `json:"ignore_fields" slice:"true"` // *
+
+	// > @3@4@5@6
+	// >
+	// > List of the processed event fields.
+	// > If name of some field contained in this list
+	// > then all nested fields will be processed (even if they are not listed).
+	// > If ignored fields list is empty and processed fields list is empty
+	// > we consider this as empty ignored fields list (all fields will be processed).
+	// > It is wrong to set non-empty ignored fields list and non-empty processed fields list at the same time.
+	ProcessFields []cfg.FieldSelector `json:"process_fields" slice:"true"` // *
 
 	// > @3@4@5@6
 	// >
@@ -248,10 +263,25 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.ActionPluginP
 	p.logger = params.Logger.Desugar()
 	p.config.Masks, p.matchRe = compileMasks(p.config.Masks, p.logger)
 
-	if len(p.config.IgnoreFields) > 0 {
-		p.ignoredFields = make(map[string]struct{}, len(p.config.IgnoreFields))
-		for _, field := range p.config.IgnoreFields {
-			p.ignoredFields[field] = struct{}{}
+	isBlacklist := len(p.config.IgnoreFields) > 0
+	isWhitelist := len(p.config.ProcessFields) > 0
+	if isBlacklist && isWhitelist {
+		p.logger.Fatal("ignored fields list and processed fields list are both non-empty")
+	}
+
+	p.isWhitelist = isWhitelist
+
+	var fieldPaths []cfg.FieldSelector
+	if isBlacklist {
+		fieldPaths = p.config.IgnoreFields
+	} else if isWhitelist {
+		fieldPaths = p.config.ProcessFields
+	}
+
+	if len(fieldPaths) > 0 {
+		p.fieldPaths = make([][]string, 0, len(fieldPaths))
+		for _, fieldPath := range fieldPaths {
+			p.fieldPaths = append(p.fieldPaths, cfg.ParseFieldSelector(string(fieldPath)))
 		}
 	}
 
@@ -379,26 +409,42 @@ func (p *Plugin) maskValue(mask *Mask, value, buf []byte) ([]byte, bool) {
 	return value, true
 }
 
-func getValueNodeList(currentNode *insaneJSON.Node, valueNodes []*insaneJSON.Node, ignoredFields map[string]struct{}) []*insaneJSON.Node {
+func getNestedValueNodes(currentNode *insaneJSON.Node, ignoredNodes []*insaneJSON.Node, valueNodes []*insaneJSON.Node) []*insaneJSON.Node {
 	switch {
+	case currentNode == nil:
+		break
 	case currentNode.IsField():
-		fieldName := currentNode.AsString()
-		// check field name in list of ignored fields
-		_, fieldIsIgnored := ignoredFields[fieldName]
-		if !fieldIsIgnored {
-			valueNodes = getValueNodeList(currentNode.AsFieldValue(), valueNodes, ignoredFields)
-		}
+		valueNodes = getNestedValueNodes(currentNode.AsFieldValue(), ignoredNodes, valueNodes)
+	case slices.Contains(ignoredNodes, currentNode):
+		break
 	case currentNode.IsArray():
 		for _, n := range currentNode.AsArray() {
-			valueNodes = getValueNodeList(n, valueNodes, ignoredFields)
+			valueNodes = getNestedValueNodes(n, ignoredNodes, valueNodes)
 		}
 	case currentNode.IsObject():
 		for _, n := range currentNode.AsFields() {
-			valueNodes = getValueNodeList(n, valueNodes, ignoredFields)
+			valueNodes = getNestedValueNodes(n, ignoredNodes, valueNodes)
 		}
 	default:
 		valueNodes = append(valueNodes, currentNode)
 	}
+
+	return valueNodes
+}
+
+func (p *Plugin) getValueNodes(root *insaneJSON.Node, valueNodes []*insaneJSON.Node) []*insaneJSON.Node {
+	if p.isWhitelist {
+		for _, fieldPath := range p.fieldPaths {
+			valueNodes = getNestedValueNodes(root.Dig(fieldPath...), nil, valueNodes)
+		}
+	} else {
+		p.ignoredNodes = p.ignoredNodes[:0]
+		for _, fieldPath := range p.fieldPaths {
+			p.ignoredNodes = append(p.ignoredNodes, root.Dig(fieldPath...))
+		}
+		valueNodes = getNestedValueNodes(root, p.ignoredNodes, valueNodes)
+	}
+
 	return valueNodes
 }
 
@@ -410,7 +456,7 @@ func (p *Plugin) Do(event *pipeline.Event) pipeline.ActionResult {
 	locApplied := false
 
 	p.valueNodes = p.valueNodes[:0]
-	p.valueNodes = getValueNodeList(root, p.valueNodes, p.ignoredFields)
+	p.valueNodes = p.getValueNodes(root, p.valueNodes)
 	for _, v := range p.valueNodes {
 		value := v.AsBytes()
 		var valueIsCommonMatched bool