Make 'format' optional

ozontech · Sep 4, 2024 · b878f39 · b878f39
1 parent c421c57
commit b878f39
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 61 deletions.
diff --git a/fd/util.go b/fd/util.go
@@ -331,19 +331,17 @@ const (
 	tsCmpValueStartTag = "file_d_start"
 )
 
-const defaultTSCmpValUpdateInterval = 10 * time.Second
+const (
+	defaultTsCmpValUpdateInterval = 10 * time.Second
+	defaultTsFormat               = time.RFC3339Nano
+)
 
 func extractTsCmpOpNode(_ string, jsonNode *simplejson.Json) (doif.Node, error) {
 	fieldPath, err := requiredString(jsonNode, fieldNameField)
 	if err != nil {
 		return nil, err
 	}
 
-	format, err := requiredString(jsonNode, fieldNameFormat)
-	if err != nil {
-		return nil, err
-	}
-
 	cmpOp, err := requiredString(jsonNode, fieldNameCmpOp)
 	if err != nil {
 		return nil, err
@@ -371,16 +369,22 @@ func extractTsCmpOpNode(_ string, jsonNode *simplejson.Json) (doif.Node, error)
 		}
 	}
 
+	format := defaultTsFormat
+	str := jsonNode.Get(fieldNameFormat).MustString()
+	if str != "" {
+		format = str
+	}
+
 	cmpValueShift := time.Duration(0)
-	str := jsonNode.Get(fieldNameCmpValueShift).MustString()
+	str = jsonNode.Get(fieldNameCmpValueShift).MustString()
 	if str != "" {
 		cmpValueShift, err = time.ParseDuration(str)
 		if err != nil {
 			return nil, fmt.Errorf("parse cmp value shift: %w", err)
 		}
 	}
 
-	updateInterval := defaultTSCmpValUpdateInterval
+	updateInterval := defaultTsCmpValUpdateInterval
 	str = jsonNode.Get(fieldNameUpdateInterval).MustString()
 	if str != "" {
 		updateInterval, err = time.ParseDuration(str)

diff --git a/fd/util_test.go b/fd/util_test.go
@@ -287,78 +287,38 @@ func Test_extractDoIfChecker(t *testing.T) {
 					"field": "timestamp",
 					"cmp_op": "lt",
 					"value": "2009-11-10T23:00:00Z",
-					"format": "2006-01-02T15:04:05.999999999Z07:00",
+					"value_shift": "-24h",
+					"format": "2006-01-02T15:04:05Z07:00",
 					"update_interval": "15s"}`,
 			},
 			want: &doIfTreeNode{
 				tsCmpOp:            true,
 				cmpOp:              "lt",
 				fieldName:          "timestamp",
-				tsFormat:           time.RFC3339Nano,
+				tsFormat:           time.RFC3339,
 				tsCmpValue:         time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
+				tsCmpValueShift:    -24 * time.Hour,
 				tsCmpValChangeMode: tsCmpModeConstTag,
 				tsUpdateInterval:   15 * time.Second,
 			},
 		},
 		{
-			name: "ok_ts_cmp_op_variable_cmp_value",
-			args: args{
-				cfgStr: `{
-					"op": "ts_cmp",
-					"field": "timestamp",
-					"cmp_op": "lt",
-					"value": "now",
-					"format": "2006-01-02T15:04:05.999999999Z07:00",
-					"update_interval": "15s"}`,
-			},
-			want: &doIfTreeNode{
-				tsCmpOp:            true,
-				cmpOp:              "lt",
-				fieldName:          "timestamp",
-				tsFormat:           time.RFC3339Nano,
-				tsCmpValChangeMode: tsCmpModeNowTag,
-				tsUpdateInterval:   15 * time.Second,
-			},
-		},
-		{
-			name: "ok_ts_cmp_op_value_shifted",
+			name: "ok_ts_cmp_op_default_settings",
 			args: args{
 				cfgStr: `{
 					"op": "ts_cmp",
 					"field": "timestamp",
 					"cmp_op": "lt",
-					"value": "now",
-					"value_shift": "-24h",
-					"format": "2006-01-02T15:04:05.999999999Z07:00"}`,
+					"value": "now"}`,
 			},
 			want: &doIfTreeNode{
 				tsCmpOp:            true,
 				cmpOp:              "lt",
 				fieldName:          "timestamp",
-				tsFormat:           time.RFC3339Nano,
 				tsCmpValChangeMode: tsCmpModeNowTag,
-				tsCmpValueShift:    -24 * time.Hour,
-				tsUpdateInterval:   10 * time.Second,
-			},
-		},
-		{
-			name: "ok_ts_cmp_op_default_update_interval",
-			args: args{
-				cfgStr: `{
-					"op": "ts_cmp",
-					"field": "timestamp",
-					"cmp_op": "lt",
-					"value": "2009-11-10T23:00:00Z",
-					"format": "2006-01-02T15:04:05.999999999Z07:00"}`,
-			},
-			want: &doIfTreeNode{
-				tsCmpOp:            true,
-				cmpOp:              "lt",
-				fieldName:          "timestamp",
-				tsFormat:           time.RFC3339Nano,
-				tsCmpValue:         time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
-				tsCmpValChangeMode: tsCmpModeConstTag,
-				tsUpdateInterval:   defaultTSCmpValUpdateInterval,
+				tsFormat:           defaultTsFormat,
+				tsCmpValueShift:    0,
+				tsUpdateInterval:   defaultTsCmpValUpdateInterval,
 			},
 		},
 		{

diff --git a/pipeline/doif/README.md b/pipeline/doif/README.md
@@ -14,7 +14,7 @@ the chain of Match func calls are performed across the whole tree.
 
 <br>
 
-**`TimestampCmpOp`** Type of node where mathing rules for timestamps are stored.
+**`TimestampCmpOp`** Type of node where matching rules for timestamps are stored.
 
 <br>
 
@@ -363,7 +363,7 @@ It contains operation that compares timestamps with certain value.
 Params:
   - `op` - must be `ts_cmp`. Required.
   - `field` - name of the field to apply operation. Required. Field will be parsed with `time.Parse` function.
-  - `format` - format for timestamps representation. Required.
+  - `format` - format for timestamps representation. Optional; default = `time.RFC3339Nano`.
   - `cmp_op` - comparison operation name (same as for length comparison operations). Required.
   - `value` - timestamp value to compare field timestamps with. It must have `RFC3339Nano` format. Required.
 Also, it may be `now` or `file_d_start`. If it is `now` then value to compare timestamps with is periodically updated current time.

diff --git a/pipeline/doif/do_if.go b/pipeline/doif/do_if.go
@@ -18,7 +18,7 @@ const (
 	// > Type of node where matching rules for byte length and array length are stored.
 	NodeLengthCmpOp // *
 
-	// > Type of node where mathing rules for timestamps are stored.
+	// > Type of node where matching rules for timestamps are stored.
 	NodeTimestampCmpOp // *
 
 	// > Type of node where logical rules for applying other rules are stored.

diff --git a/pipeline/doif/ts_cmp_op.go b/pipeline/doif/ts_cmp_op.go
@@ -18,7 +18,7 @@ It contains operation that compares timestamps with certain value.
 Params:
   - `op` - must be `ts_cmp`. Required.
   - `field` - name of the field to apply operation. Required. Field will be parsed with `time.Parse` function.
-  - `format` - format for timestamps representation. Required.
+  - `format` - format for timestamps representation. Optional; default = `time.RFC3339Nano`.
   - `cmp_op` - comparison operation name (same as for length comparison operations). Required.
   - `value` - timestamp value to compare field timestamps with. It must have `RFC3339Nano` format. Required.
 Also, it may be `now` or `file_d_start`. If it is `now` then value to compare timestamps with is periodically updated current time.