diff --git a/.gitignore b/.gitignore
index 2d57afcfb..2e8efd29c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,5 @@ ozon*
dist/
tests-offsets
testdata
+
+.idea/
diff --git a/fd/util.go b/fd/util.go
index 827c0ca3a..31129692b 100644
--- a/fd/util.go
+++ b/fd/util.go
@@ -205,6 +205,9 @@ var (
"suffix": struct{}{},
"regex": struct{}{},
}
+ doIfBytesLengthCmpNodes = map[string]struct{}{
+ "byte_len_cmp": struct{}{},
+ }
)
func extractFieldOpVals(jsonNode *simplejson.Json) [][]byte {
@@ -248,6 +251,52 @@ func extractFieldOpNode(opName string, jsonNode *simplejson.Json) (pipeline.DoIf
return result, nil
}
+func noRequiredFieldError(field string) error {
+ return fmt.Errorf("no required field: %s", field)
+}
+
+const (
+ fieldNameField = "field"
+ fieldNameCmpOp = "cmp_op"
+ fieldNameCmpValue = "value"
+)
+
+func extractByteLengthCmpOpNode(_ string, jsonNode *simplejson.Json) (pipeline.DoIfNode, error) {
+ fieldPathNode, has := jsonNode.CheckGet(fieldNameField)
+ if !has {
+ return nil, noRequiredFieldError(fieldNameField)
+ }
+ fieldPath, err := fieldPathNode.String()
+ if err != nil {
+ return nil, err
+ }
+
+ cmpOpNode, has := jsonNode.CheckGet(fieldNameCmpOp)
+ if !has {
+ return nil, noRequiredFieldError(fieldNameCmpOp)
+ }
+ cmpOp, err := cmpOpNode.String()
+ if err != nil {
+ return nil, err
+ }
+
+ cmpValueNode, has := jsonNode.CheckGet(fieldNameCmpValue)
+ if !has {
+ return nil, noRequiredFieldError(fieldNameCmpValue)
+ }
+ cmpValue, err := cmpValueNode.Int()
+ if err != nil {
+ return nil, err
+ }
+
+ result, err := pipeline.NewByteLengthCmpNode(fieldPath, cmpOp, cmpValue)
+ if err != nil {
+ return nil, fmt.Errorf("failed to init bytes length cmp op: %w", err)
+ }
+
+ return result, nil
+}
+
func extractLogicalOpNode(opName string, jsonNode *simplejson.Json) (pipeline.DoIfNode, error) {
var result, operand pipeline.DoIfNode
var err error
@@ -278,6 +327,8 @@ func extractDoIfNode(jsonNode *simplejson.Json) (pipeline.DoIfNode, error) {
return extractLogicalOpNode(opName, jsonNode)
} else if _, has := doIfFieldOpNodes[opName]; has {
return extractFieldOpNode(opName, jsonNode)
+ } else if _, has := doIfBytesLengthCmpNodes[opName]; has {
+ return extractByteLengthCmpOpNode(opName, jsonNode)
}
return nil, fmt.Errorf("unknown op %q", opName)
}
diff --git a/fd/util_test.go b/fd/util_test.go
index 3359f5e94..54ecd8239 100644
--- a/fd/util_test.go
+++ b/fd/util_test.go
@@ -46,18 +46,22 @@ type doIfTreeNode struct {
logicalOp string
operands []*doIfTreeNode
+
+ byteLenCmpOp string
+ cmpValue int
}
// nolint:gocritic
func buildDoIfTree(node *doIfTreeNode) (pipeline.DoIfNode, error) {
- if node.fieldOp != "" {
+ switch {
+ case node.fieldOp != "":
return pipeline.NewFieldOpNode(
node.fieldOp,
node.fieldName,
node.caseSensitive,
node.values,
)
- } else if node.logicalOp != "" {
+ case node.logicalOp != "":
operands := make([]pipeline.DoIfNode, 0)
for _, operandNode := range node.operands {
operand, err := buildDoIfTree(operandNode)
@@ -70,8 +74,11 @@ func buildDoIfTree(node *doIfTreeNode) (pipeline.DoIfNode, error) {
node.logicalOp,
operands,
)
+ case node.byteLenCmpOp != "":
+ return pipeline.NewByteLengthCmpNode(node.fieldName, node.byteLenCmpOp, node.cmpValue)
+ default:
+ return nil, errors.New("unknown type of node")
}
- return nil, errors.New("unknown type of node")
}
func Test_extractDoIfChecker(t *testing.T) {
@@ -107,6 +114,12 @@ func Test_extractDoIfChecker(t *testing.T) {
"values": ["test-1", "test-2"],
"case_sensitive": false
},
+ {
+ "op": "byte_len_cmp",
+ "field": "msg",
+ "cmp_op": "gt",
+ "value": 100
+ },
{
"op": "or",
"operands": [
@@ -152,6 +165,11 @@ func Test_extractDoIfChecker(t *testing.T) {
values: [][]byte{[]byte("test-1"), []byte("test-2")},
caseSensitive: false,
},
+ {
+ byteLenCmpOp: "gt",
+ fieldName: "msg",
+ cmpValue: 100,
+ },
{
logicalOp: "or",
operands: []*doIfTreeNode{
@@ -187,6 +205,17 @@ func Test_extractDoIfChecker(t *testing.T) {
},
wantErr: false,
},
+ {
+ name: "ok_byte_len_cmp_op",
+ args: args{
+ cfgStr: `{"op":"byte_len_cmp","field":"data","cmp_op":"lt","value":10}`,
+ },
+ want: &doIfTreeNode{
+ byteLenCmpOp: "lt",
+ fieldName: "data",
+ cmpValue: 10,
+ },
+ },
{
name: "ok_single_val",
args: args{
@@ -259,6 +288,58 @@ func Test_extractDoIfChecker(t *testing.T) {
},
wantErr: true,
},
+ {
+ name: "error_byte_len_cmp_op_no_field",
+ args: args{
+ cfgStr: `{"op":"byte_len_cmp","cmp_op":"lt","value":10}`,
+ },
+ wantErr: true,
+ },
+ {
+ name: "error_byte_len_cmp_op_field_is_not_string",
+ args: args{
+ cfgStr: `{"op":"byte_len_cmp","field":123,"cmp_op":"lt","value":10}`,
+ },
+ wantErr: true,
+ },
+ {
+ name: "error_byte_len_cmp_op_no_cmp_op",
+ args: args{
+ cfgStr: `{"op":"byte_len_cmp","field":"data","value":10}`,
+ },
+ wantErr: true,
+ },
+ {
+ name: "error_byte_len_cmp_op_cmp_op_is_not_string",
+ args: args{
+ cfgStr: `{"op":"byte_len_cmp","field":"data","cmp_op":123,"value":10}`,
+ },
+ wantErr: true,
+ },
+ {
+ name: "error_byte_len_cmp_op_no_cmp_value",
+ args: args{
+ cfgStr: `{"op":"byte_len_cmp","field":"data","cmp_op":"lt"}`,
+ },
+ wantErr: true,
+ },
+ {
+ name: "error_byte_len_cmp_op_cmp_value_is_not_integer",
+ args: args{
+ cfgStr: `{"op":"byte_len_cmp","field":"data","cmp_op":"lt","value":"abc"}`,
+ },
+ wantErr: true,
+ },
+ {
+ name: "error_byte_len_cmp_op_invalid_cmp_op",
+ args: args{cfgStr: `{"op":"byte_len_cmp","field":"data","cmp_op":"ABC","value":10}`},
+ wantErr: true,
+ },
+ {
+ name: "error_byte_len_cmp_op_negative_cmp_value",
+ args: args{cfgStr: `{"op":"byte_len_cmp","field":"data","cmp_op":"lt","value":-1}`},
+ wantErr: true,
+ },
}
for _, tt := range tests {
tt := tt
diff --git a/pipeline/README.idoc.md b/pipeline/README.idoc.md
index abb775a21..ffc9f9280 100644
--- a/pipeline/README.idoc.md
+++ b/pipeline/README.idoc.md
@@ -22,3 +22,6 @@ the chain of Match func calls are performed across the whole tree.
### Logical operations
@do-if-logical-op|description
+
+### Byte length comparison op node
+@do-if-byte-len-cmp-op-node
diff --git a/pipeline/README.md b/pipeline/README.md
index fd18bd78f..1a0e631c5 100755
--- a/pipeline/README.md
+++ b/pipeline/README.md
@@ -117,6 +117,10 @@ the chain of Match func calls are performed across the whole tree.
+**`ByteLenCmpOp`** Type of node where matching rules for byte lengths of fields are stored.
+
+
+
**`LogicalOp`** Type of node where logical rules for applying other rules are stored.
@@ -388,4 +392,47 @@ result:
+### Byte length comparison op node
+DoIf byte length comparison op node is considered to always be a leaf in the DoIf tree like DoIf field op node.
+It contains operation that compares field length in bytes with certain value.
+
+Params:
+ - `op` - must be `byte_len_cmp`. Required.
+ - `field` - name of the field to apply operation. Required.
+ - `cmp_op` - comparison operation name (see below). Required.
+ - `value` - integer value to compare length with. Required non-negative.
+
+Example:
+```yaml
+pipelines:
+ test:
+ actions:
+ - type: discard
+ do_if:
+ op: byte_len_cmp
+ field: pod_id
+ cmp_op: lt
+ value: 5
+```
+
+result:
+```
+{"pod_id":""} # discarded
+{"pod_id":123} # discarded
+{"pod_id":12345} # not discarded
+{"pod_id":123456} # not discarded
+```
+
+Possible values of field 'cmp_op': `lt`, `le`, `gt`, `ge`, `eq`, `ne`.
+They denote corresponding comparison operations.
+
+| Name | Op |
+|------|----|
+| `lt` | `<` |
+| `le` | `<=` |
+| `gt` | `>` |
+| `ge` | `>=` |
+| `eq` | `==` |
+| `ne` | `!=` |
+
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)*
\ No newline at end of file
diff --git a/pipeline/do_if.go b/pipeline/do_if.go
index e4e5b93fa..60d8fac12 100644
--- a/pipeline/do_if.go
+++ b/pipeline/do_if.go
@@ -22,6 +22,9 @@ const (
// > Type of node where matching rules for fields are stored.
DoIfNodeFieldOp // *
+ // > Type of node where matching rules for byte lengths of fields are stored.
+ DoIfNodeByteLenCmpOp // *
+
// > Type of node where logical rules for applying other rules are stored.
DoIfNodeLogicalOp // *
)
@@ -676,3 +679,131 @@ func (c *DoIfChecker) Check(eventRoot *insaneJSON.Root) bool {
}
return c.root.Check(eventRoot)
}
+
+type comparisonOperation string
+
+const (
+ cmpOpLess comparisonOperation = "lt"
+ cmpOpLessOrEqual comparisonOperation = "le"
+ cmpOpGreater comparisonOperation = "gt"
+ cmpOpGreaterOrEqual comparisonOperation = "ge"
+ cmpOpEqual comparisonOperation = "eq"
+ cmpOpNotEqual comparisonOperation = "ne"
+)
+
+/*{ do-if-byte-len-cmp-op-node
+DoIf byte length comparison op node is considered to always be a leaf in the DoIf tree like DoIf field op node.
+It contains operation that compares field length in bytes with certain value.
+
+Params:
+ - `op` - must be `byte_len_cmp`. Required.
+ - `field` - name of the field to apply operation. Required.
+ - `cmp_op` - comparison operation name (see below). Required.
+ - `value` - integer value to compare length with. Required non-negative.
+
+Example:
+```yaml
+pipelines:
+ test:
+ actions:
+ - type: discard
+ do_if:
+ op: byte_len_cmp
+ field: pod_id
+ cmp_op: lt
+ value: 5
+```
+
+result:
+```
+{"pod_id":""} # discarded
+{"pod_id":123} # discarded
+{"pod_id":12345} # not discarded
+{"pod_id":123456} # not discarded
+```
+
+Possible values of field 'cmp_op': `lt`, `le`, `gt`, `ge`, `eq`, `ne`.
+They denote corresponding comparison operations.
+
+| Name | Op |
+|------|----|
+| `lt` | `<` |
+| `le` | `<=` |
+| `gt` | `>` |
+| `ge` | `>=` |
+| `eq` | `==` |
+| `ne` | `!=` |
+}*/
+
+type doIfByteLengthCmpNode struct {
+ fieldPath []string
+ cmpOp comparisonOperation
+ cmpValue int
+}
+
+func NewByteLengthCmpNode(field string, cmpOp string, cmpValue int) (DoIfNode, error) {
+ fieldPath := cfg.ParseFieldSelector(field)
+
+ typedCmpOp := comparisonOperation(cmpOp)
+ switch typedCmpOp {
+ case cmpOpLess, cmpOpLessOrEqual, cmpOpGreater, cmpOpGreaterOrEqual, cmpOpEqual, cmpOpNotEqual:
+ default:
+ return nil, fmt.Errorf("unknown comparison operation: %s", typedCmpOp)
+ }
+
+ if cmpValue < 0 {
+ return nil, fmt.Errorf("compare length must be non-negative value: %d", cmpValue)
+ }
+
+ return &doIfByteLengthCmpNode{
+ fieldPath: fieldPath,
+ cmpOp: typedCmpOp,
+ cmpValue: cmpValue,
+ }, nil
+}
+
+func (n *doIfByteLengthCmpNode) Type() DoIfNodeType {
+ return DoIfNodeByteLenCmpOp
+}
+
+func (n *doIfByteLengthCmpNode) Check(eventRoot *insaneJSON.Root) bool {
+ data := eventRoot.Dig(n.fieldPath...).AsString()
+
+ switch n.cmpOp {
+ case cmpOpLess:
+ return len(data) < n.cmpValue
+ case cmpOpLessOrEqual:
+ return len(data) <= n.cmpValue
+ case cmpOpGreater:
+ return len(data) > n.cmpValue
+ case cmpOpGreaterOrEqual:
+ return len(data) >= n.cmpValue
+ case cmpOpEqual:
+ return len(data) == n.cmpValue
+ case cmpOpNotEqual:
+ return len(data) != n.cmpValue
+ default:
+ panic("invalid cmp op")
+ }
+}
+
+func (n *doIfByteLengthCmpNode) isEqualTo(n2 DoIfNode, _ int) error {
+ n2Explicit, ok := n2.(*doIfByteLengthCmpNode)
+ if !ok {
+ return errors.New("nodes have different types expected: bytesLengthCmpNode")
+ }
+
+ if n.cmpOp != n2Explicit.cmpOp {
+ return fmt.Errorf("nodes have different op expected: %q", n.cmpOp)
+ }
+
+ if n.cmpValue != n2Explicit.cmpValue {
+ return fmt.Errorf("nodes have different cmp values: %d", n.cmpValue)
+ }
+
+ if slices.Compare(n.fieldPath, n2Explicit.fieldPath) != 0 {
+ return fmt.Errorf("nodes have different fieldPathStr expected: fieldPath=%v", n.fieldPath)
+ }
+
+ return nil
+}
diff --git a/pipeline/do_if_test.go b/pipeline/do_if_test.go
index 0d79dcbdf..9caeec671 100644
--- a/pipeline/do_if_test.go
+++ b/pipeline/do_if_test.go
@@ -19,18 +19,22 @@ type treeNode struct {
logicalOp string
operands []treeNode
+
+ byteLenCmpOp string
+ cmpValue int
}
// nolint:gocritic
func buildTree(node treeNode) (DoIfNode, error) {
- if node.fieldOp != "" {
+ switch {
+ case node.fieldOp != "":
return NewFieldOpNode(
node.fieldOp,
node.fieldName,
node.caseSensitive,
node.values,
)
- } else if node.logicalOp != "" {
+ case node.logicalOp != "":
operands := make([]DoIfNode, 0)
for _, operandNode := range node.operands {
operand, err := buildTree(operandNode)
@@ -43,8 +47,11 @@ func buildTree(node treeNode) (DoIfNode, error) {
node.logicalOp,
operands,
)
+ case node.byteLenCmpOp != "":
+ return NewByteLengthCmpNode(node.fieldName, node.byteLenCmpOp, node.cmpValue)
+ default:
+ return nil, errors.New("unknown type of node")
}
- return nil, errors.New("unknown type of node")
}
func checkDoIfNode(t *testing.T, want, got DoIfNode) {
@@ -92,6 +99,14 @@ func checkDoIfNode(t *testing.T, want, got DoIfNode) {
for i := 0; i < len(wantNode.operands); i++ {
checkDoIfNode(t, wantNode.operands[i], gotNode.operands[i])
}
+ case DoIfNodeByteLenCmpOp:
+ wantNode := want.(*doIfByteLengthCmpNode)
+ gotNode := got.(*doIfByteLengthCmpNode)
+ assert.Equal(t, wantNode.cmpOp, gotNode.cmpOp)
+ assert.Equal(t, wantNode.cmpValue, gotNode.cmpValue)
+ assert.Equal(t, 0, slices.Compare[[]string](wantNode.fieldPath, gotNode.fieldPath))
+ default:
+ t.Error("unknown node type")
}
}
@@ -220,6 +235,32 @@ func TestBuildDoIfNodes(t *testing.T) {
},
},
},
+ {
+ name: "ok_byte_len_cmp_op_node",
+ tree: treeNode{
+ byteLenCmpOp: "lt",
+ fieldName: "pod",
+ cmpValue: 100,
+ },
+ want: &doIfByteLengthCmpNode{
+ fieldPath: []string{"pod"},
+ cmpOp: "lt",
+ cmpValue: 100,
+ },
+ },
+ {
+ name: "ok_byte_len_cmp_op_node_empty_selector",
+ tree: treeNode{
+ byteLenCmpOp: "lt",
+ fieldName: "",
+ cmpValue: 100,
+ },
+ want: &doIfByteLengthCmpNode{
+ fieldPath: []string{},
+ cmpOp: "lt",
+ cmpValue: 100,
+ },
+ },
{
name: "err_field_op_node_empty_field",
tree: treeNode{
@@ -253,6 +294,24 @@ func TestBuildDoIfNodes(t *testing.T) {
},
wantErr: true,
},
+ {
+ name: "err_byte_len_op_node_invalid_op_type",
+ tree: treeNode{
+ byteLenCmpOp: "no-op",
+ fieldName: "pod",
+ cmpValue: 100,
+ },
+ wantErr: true,
+ },
+ {
+ name: "err_byte_len_op_node_negative_cmp_value",
+ tree: treeNode{
+ byteLenCmpOp: "lt",
+ fieldName: "pod",
+ cmpValue: -1,
+ },
+ wantErr: true,
+ },
{
name: "err_logical_op_node_empty_operands",
tree: treeNode{
@@ -561,6 +620,83 @@ func TestCheck(t *testing.T) {
{`{"pod":"my-TEST-2","test-field":"non-empty"}`, false},
},
},
+ {
+ name: "ok_byte_len_cmp_lt",
+ tree: treeNode{
+ byteLenCmpOp: "lt",
+ fieldName: "msg",
+ cmpValue: 4,
+ },
+ data: []argsResp{
+ {`{"msg":""}`, true},
+ {`{"msg":1}`, true},
+ {`{"msg":12}`, true},
+ {`{"msg":123}`, true},
+ {`{"msg":1234}`, false},
+ {`{"msg":12345}`, false},
+ {`{"msg":123456}`, false},
+ },
+ },
+ {
+ name: "ok_byte_len_cmp_ge",
+ tree: treeNode{
+ byteLenCmpOp: "ge",
+ fieldName: "msg",
+ cmpValue: 4,
+ },
+ data: []argsResp{
+ {`{"msg":""}`, false},
+ {`{"msg":1}`, false},
+ {`{"msg":12}`, false},
+ {`{"msg":123}`, false},
+ {`{"msg":1234}`, true},
+ {`{"msg":12345}`, true},
+ {`{"msg":123456}`, true},
+ },
+ },
+ {
+ name: "ok_byte_len_cmp_lt_empty_selector",
+ tree: treeNode{
+ byteLenCmpOp: "lt",
+ fieldName: "",
+ cmpValue: 4,
+ },
+ data: []argsResp{
+ {`""`, true},
+ {`1`, true},
+ {`12`, true},
+ {`123`, true},
+ {`1234`, false},
+ {`12345`, false},
+ {`123456`, false},
+ },
+ },
+ {
+ name: "ok_byte_len_cmp_eq",
+ tree: treeNode{
+ byteLenCmpOp: "eq",
+ fieldName: "msg",
+ cmpValue: 2,
+ },
+ data: []argsResp{
+ {`{"msg":1}`, false},
+ {`{"msg":12}`, true},
+ {`{"msg":123}`, false},
+ },
+ },
+ {
+ name: "ok_byte_len_cmp_ne",
+ tree: treeNode{
+ byteLenCmpOp: "ne",
+ fieldName: "msg",
+ cmpValue: 2,
+ },
+ data: []argsResp{
+ {`{"msg":1}`, true},
+ {`{"msg":12}`, false},
+ {`{"msg":123}`, true},
+ },
+ },
}
for _, tt := range tests {
tt := tt
@@ -591,12 +727,17 @@ func TestCheck(t *testing.T) {
}
func TestDoIfNodeIsEqual(t *testing.T) {
- singleNode := treeNode{
+ singleNode1 := treeNode{
fieldOp: "equal",
fieldName: "service",
caseSensitive: true,
values: [][]byte{[]byte("test-1"), []byte("test-2")},
}
+ singleNode2 := treeNode{
+ byteLenCmpOp: "lt",
+ fieldName: "msg",
+ cmpValue: 100,
+ }
twoNodes := treeNode{
logicalOp: "not",
operands: []treeNode{
@@ -626,6 +767,11 @@ func TestDoIfNodeIsEqual(t *testing.T) {
caseSensitive: false,
values: [][]byte{[]byte("pod-1"), []byte("pod-2")},
},
+ {
+ byteLenCmpOp: "lt",
+ fieldName: "msg",
+ cmpValue: 100,
+ },
{
logicalOp: "and",
operands: []treeNode{
@@ -661,8 +807,14 @@ func TestDoIfNodeIsEqual(t *testing.T) {
}{
{
name: "equal_single_node",
- t1: singleNode,
- t2: singleNode,
+ t1: singleNode1,
+ t2: singleNode1,
+ wantErr: false,
+ },
+ {
+ name: "equal_byte_len_cmp_node",
+ t1: singleNode2,
+ t2: singleNode2,
wantErr: false,
},
{
@@ -678,24 +830,21 @@ func TestDoIfNodeIsEqual(t *testing.T) {
wantErr: false,
},
{
- name: "not_equal_type_mismatch",
- t1: treeNode{
- fieldOp: "equal",
- fieldName: "service",
- caseSensitive: false,
- values: [][]byte{nil},
- },
- t2: treeNode{
- logicalOp: "not",
- operands: []treeNode{
- {
- fieldOp: "equal",
- fieldName: "service",
- caseSensitive: false,
- values: [][]byte{nil},
- },
- },
- },
+ name: "not_equal_type_mismatch_1",
+ t1: singleNode1,
+ t2: singleNode2,
+ wantErr: true,
+ },
+ {
+ name: "not_equal_type_mismatch_2",
+ t1: singleNode1,
+ t2: multiNodes,
+ wantErr: true,
+ },
+ {
+ name: "not_equal_type_mismatch_3",
+ t1: singleNode2,
+ t2: multiNodes,
wantErr: true,
},
{
@@ -906,6 +1055,48 @@ func TestDoIfNodeIsEqual(t *testing.T) {
},
wantErr: true,
},
+ {
+ name: "not_equal_byte_len_cmp_op_mismatch",
+ t1: treeNode{
+ byteLenCmpOp: "lt",
+ fieldName: "msg",
+ cmpValue: 100,
+ },
+ t2: treeNode{
+ byteLenCmpOp: "gt",
+ fieldName: "msg",
+ cmpValue: 100,
+ },
+ wantErr: true,
+ },
+ {
+ name: "not_equal_byte_len_cmp_op_field_mismatch",
+ t1: treeNode{
+ byteLenCmpOp: "lt",
+ fieldName: "msg",
+ cmpValue: 100,
+ },
+ t2: treeNode{
+ byteLenCmpOp: "lt",
+ fieldName: "pod",
+ cmpValue: 100,
+ },
+ wantErr: true,
+ },
+ {
+ name: "not_equal_byte_len_cmp_op_value_mismatch",
+ t1: treeNode{
+ byteLenCmpOp: "lt",
+ fieldName: "msg",
+ cmpValue: 100,
+ },
+ t2: treeNode{
+ byteLenCmpOp: "lt",
+ fieldName: "msg",
+ cmpValue: 200,
+ },
+ wantErr: true,
+ },
{
name: "not_equal_logical_op_mismatch",
t1: treeNode{