diff --git a/collection/collection.go b/collection/collection.go index 668608966..8558a5598 100644 --- a/collection/collection.go +++ b/collection/collection.go @@ -43,24 +43,45 @@ type Keyed interface { FindString(key string) []types.MatchData } +type Editable interface { + Keyed + + // Remove deletes the key from the CollectionMap + Remove(key string) + + // Set will replace the key's value with this slice + Set(key string, values []string) + + // TODO: in v4 this should contain setters for Map and Persistence +} + // Map are used to store VARIABLE data // for transactions, this data structured is designed // to store slices of data for keys // Important: CollectionMaps ARE NOT concurrent safe type Map interface { - Keyed + Editable // Add a value to some key Add(key string, value string) - // Set will replace the key's value with this slice - Set(key string, values []string) - // SetIndex will place the value under the index // If the index is higher than the current size of the CollectionMap // it will be appended SetIndex(key string, index int, value string) +} - // Remove deletes the key from the CollectionMap - Remove(key string) +// Persistent collections won't use arrays as values +// They are designed for collections that will be stored +type Persistent interface { + Editable + + // Initializes the input as the collection key + Init(key string) + + // Sum will add the value to the key + Sum(key string, sum int) + + // SetOne will replace the key's value with this string + SetOne(key string, value string) } diff --git a/experimental/plugins/persistence.go b/experimental/plugins/persistence.go new file mode 100644 index 000000000..d1cf67399 --- /dev/null +++ b/experimental/plugins/persistence.go @@ -0,0 +1,14 @@ +// Copyright 2023 Juan Pablo Tosso and the OWASP Coraza contributors +// SPDX-License-Identifier: Apache-2.0 + +package plugins + +import ( + "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" + "github.com/corazawaf/coraza/v3/internal/persistence" +) + +// RegisterPersistenceEngine registers a new persistence engine +func RegisterPersistenceEngine(name string, engine plugintypes.PersistenceEngine) { + persistence.Register(name, engine) +} diff --git a/experimental/plugins/plugintypes/persistence.go b/experimental/plugins/plugintypes/persistence.go new file mode 100644 index 000000000..8036819a3 --- /dev/null +++ b/experimental/plugins/plugintypes/persistence.go @@ -0,0 +1,15 @@ +// Copyright 2023 Juan Pablo Tosso and the OWASP Coraza contributors +// SPDX-License-Identifier: Apache-2.0 + +package plugintypes + +type PersistenceEngine interface { + Open(uri string, ttl int) error + Close() error + Sum(collectionName string, collectionKey string, key string, sum int) error + Get(collectionName string, collectionKey string, key string) (string, error) + + All(collectionName string, collectionKey string) (map[string]string, error) + Set(collection string, collectionKey string, key string, value string) error + Remove(collection string, collectionKey string, key string) error +} diff --git a/experimental/plugins/plugintypes/transaction.go b/experimental/plugins/plugintypes/transaction.go index c520088d0..050841e27 100644 --- a/experimental/plugins/plugintypes/transaction.go +++ b/experimental/plugins/plugintypes/transaction.go @@ -115,4 +115,10 @@ type TransactionVariables interface { ArgsNames() collection.Collection ArgsGetNames() collection.Collection ArgsPostNames() collection.Collection + // TODO(v4: Add these) + // Session() collection.Persistent + // User() collection.Persistent + // IP() collection.Persistent + // Global() collection.Persistent + // Resource() collection.Persistent } diff --git a/internal/actions/initcol.go b/internal/actions/initcol.go index 4899d7213..06d74602f 100644 --- a/internal/actions/initcol.go +++ b/internal/actions/initcol.go @@ -4,9 +4,14 @@ package actions import ( + "fmt" "strings" + "github.com/corazawaf/coraza/v3/experimental/plugins/macro" "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" + "github.com/corazawaf/coraza/v3/internal/collections" + utils "github.com/corazawaf/coraza/v3/internal/strings" + "github.com/corazawaf/coraza/v3/types/variables" ) // Action Group: Non-disruptive @@ -23,9 +28,8 @@ import ( // SecAction "phase:1,id:116,nolog,pass,initcol:ip=%{REMOTE_ADDR}" // ``` type initcolFn struct { - collection string - variable byte - key string + collection variables.RuleVariable + key macro.Macro } func (a *initcolFn) Init(_ plugintypes.RuleMetadata, data string) error { @@ -34,34 +38,34 @@ func (a *initcolFn) Init(_ plugintypes.RuleMetadata, data string) error { return ErrInvalidKVArguments } - a.collection = col - a.key = key - a.variable = 0x0 + c, err := variables.Parse(col) + if err != nil { + return fmt.Errorf("initcol: collection %s is not valid", col) + } + // we validate if this is a persistent collection + persistent := []string{"USER", "SESSION", "IP", "RESOURCE", "GLOBAL"} + if !utils.InSlice(c.Name(), persistent) { + return fmt.Errorf("initcol: collection %s is not persistent", c.Name()) + } + a.collection = c + mkey, err := macro.NewMacro(key) + if err != nil { + return err + } + a.key = mkey return nil } -func (a *initcolFn) Evaluate(_ plugintypes.RuleMetadata, _ plugintypes.TransactionState) { - // tx.DebugLogger().Error().Msg("initcol was used but it's not supported", zap.Int("rule", r.Id)) - /* - key := tx.MacroExpansion(a.key) - data := tx.WAF.Persistence.Get(a.variable, key) - if data == nil { - ts := time.Now().UnixNano() - tss := strconv.FormatInt(ts, 10) - tsstimeout := strconv.FormatInt(ts+(int64(tx.WAF.CollectionTimeout)*1000), 10) - data = map[string][]string{ - "CREATE_TIME": {tss}, - "IS_NEW": {"1"}, - "KEY": {key}, - "LAST_UPDATE_TIME": {tss}, - "TIMEOUT": {tsstimeout}, - "UPDATE_COUNTER": {"0"}, - "UPDATE_RATE": {"0"}, - } - } - tx.GetCollection(a.variable).SetData(data) - tx.PersistentCollections[a.variable] = key - */ +func (a *initcolFn) Evaluate(_ plugintypes.RuleMetadata, txs plugintypes.TransactionState) { + col := txs.Collection(a.collection) + key := a.key.Expand(txs) + txs.DebugLogger().Debug().Str("collection", a.collection.Name()).Str("key", key).Msg("initcol: initializing collection") + c, ok := col.(*collections.Persistent) + if !ok { + txs.DebugLogger().Error().Str("collection", a.collection.Name()).Msg("initcol: collection is not a persistent collection") + return + } + c.Init(key) } func (a *initcolFn) Type() plugintypes.ActionType { diff --git a/internal/actions/initcol_test.go b/internal/actions/initcol_test.go index 1098e902b..01673e4b0 100644 --- a/internal/actions/initcol_test.go +++ b/internal/actions/initcol_test.go @@ -1,24 +1,33 @@ // Copyright 2023 Juan Pablo Tosso and the OWASP Coraza contributors // SPDX-License-Identifier: Apache-2.0 -package actions +package actions_test -import "testing" +import ( + "testing" + + "github.com/corazawaf/coraza/v3" + "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" + "github.com/corazawaf/coraza/v3/internal/actions" +) func TestInitcolInit(t *testing.T) { + waf, _ := coraza.NewWAF(coraza.NewWAFConfig()) //nolint:errcheck + initcol, err := actions.Get("initcol") + if err != nil { + t.Error(err) + } t.Run("invalid argument", func(t *testing.T) { - initcol := initcol() - err := initcol.Init(nil, "foo") - if err == nil { - t.Errorf("expected error") + if err := initcol.Init(&md{}, "test"); err == nil { + t.Error("expected error") } }) - t.Run("passing argument", func(t *testing.T) { - initcol := initcol() - err := initcol.Init(nil, "foo=bar") - if err != nil { - t.Errorf("unexpected error: %s", err.Error()) + t.Run("editable variable", func(t *testing.T) { + if err := initcol.Init(&md{}, "session=abcdef"); err != nil { + t.Error(err) } + txs := waf.NewTransaction().(plugintypes.TransactionState) + initcol.Evaluate(&md{}, txs) }) } diff --git a/internal/actions/setvar.go b/internal/actions/setvar.go index a372b4410..3d7664dd9 100644 --- a/internal/actions/setvar.go +++ b/internal/actions/setvar.go @@ -8,6 +8,8 @@ import ( "strconv" "strings" + utils "github.com/corazawaf/coraza/v3/internal/strings" + "github.com/corazawaf/coraza/v3/collection" "github.com/corazawaf/coraza/v3/experimental/plugins/macro" "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" @@ -76,8 +78,10 @@ func (a *setvarFn) Init(_ plugintypes.RuleMetadata, data string) error { colKey, colVal, colOk := strings.Cut(key, ".") // Right not it only makes sense to allow setting TX // key is also required - if strings.ToUpper(colKey) != "TX" { - return errors.New("invalid arguments, expected collection TX") + available := []string{"TX", "USER", "GLOBAL", "RESOURCE", "SESSION", "IP"} + // we validate uppercase colKey is one of available + if !utils.InSlice(strings.ToUpper(colKey), available) { + return errors.New("setvar: invalid editable collection, available collections are: " + strings.Join(available, ", ")) } if strings.TrimSpace(colVal) == "" { return errors.New("invalid arguments, expected syntax TX.{key}={value}") @@ -111,7 +115,7 @@ func (a *setvarFn) Evaluate(r plugintypes.RuleMetadata, tx plugintypes.Transacti Str("var_key", key). Str("var_value", value). Int("rule_id", r.ID()). - Msg("Action evaluated") + Msg("Action SetVar evaluated") a.evaluateTxCollection(r, tx, strings.ToLower(key), value) } @@ -120,17 +124,14 @@ func (a *setvarFn) Type() plugintypes.ActionType { } func (a *setvarFn) evaluateTxCollection(r plugintypes.RuleMetadata, tx plugintypes.TransactionState, key string, value string) { - var col collection.Map - if c, ok := tx.Collection(a.collection).(collection.Map); !ok { - tx.DebugLogger().Error().Msg("collection in setvar is not a map") + // TODO for api breaking issues, we have to split this function in Map and Persistent + var col collection.Editable + if c, ok := tx.Collection(a.collection).(collection.Editable); !ok { + tx.DebugLogger().Error().Msg("collection in setvar is not editable") return } else { col = c } - if col == nil { - tx.DebugLogger().Error().Msg("collection in setvar is nil") - return - } if a.isRemove { col.Remove(key) diff --git a/internal/actions/setvar_persistence_test.go b/internal/actions/setvar_persistence_test.go new file mode 100644 index 000000000..2ee547533 --- /dev/null +++ b/internal/actions/setvar_persistence_test.go @@ -0,0 +1,37 @@ +// Copyright 2023 Juan Pablo Tosso and the OWASP Coraza contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build !tinygo +// +build !tinygo + +package actions_test + +import ( + "testing" + + "github.com/corazawaf/coraza/v3" + "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" + "github.com/corazawaf/coraza/v3/internal/actions" + "github.com/corazawaf/coraza/v3/types/variables" +) + +func TestPersistenceSetvar(t *testing.T) { + a, err := actions.Get("setvar") + if err != nil { + t.Error("failed to get setvar action") + } + waf, err := coraza.NewWAF(coraza.NewWAFConfig().WithDirectives("SecPersistenceEngine default")) + if err != nil { + t.Fatal(err) + } + t.Run("SESSION should be set", func(t *testing.T) { + if err := a.Init(&md{}, "SESSION.test=test"); err != nil { + t.Errorf("unexpected error: %v", err) + } + tx := waf.NewTransaction() + txs := tx.(plugintypes.TransactionState) + a.Evaluate(&md{}, txs) + col := txs.Collection(variables.Session) + col.FindAll() + }) +} diff --git a/internal/actions/setvar_test.go b/internal/actions/setvar_test.go index c512c83a6..140834894 100644 --- a/internal/actions/setvar_test.go +++ b/internal/actions/setvar_test.go @@ -1,10 +1,12 @@ // Copyright 2023 Juan Pablo Tosso and the OWASP Coraza contributors // SPDX-License-Identifier: Apache-2.0 -package actions +package actions_test import ( "testing" + + "github.com/corazawaf/coraza/v3/internal/actions" ) type md struct { @@ -21,27 +23,27 @@ func (_ md) Status() int { } func TestSetvarInit(t *testing.T) { + a, err := actions.Get("setvar") + if err != nil { + t.Error("failed to get setvar action") + } t.Run("no arguments", func(t *testing.T) { - a := setvar() - if err := a.Init(nil, ""); err == nil || err != ErrMissingArguments { + if err := a.Init(nil, ""); err == nil || err != actions.ErrMissingArguments { t.Error("expected error ErrMissingArguments") } }) t.Run("non-map variable", func(t *testing.T) { - a := setvar() if err := a.Init(&md{}, "PATH_INFO=test"); err == nil { t.Error("expected error") } }) t.Run("TX set ok", func(t *testing.T) { - a := setvar() if err := a.Init(&md{}, "TX.some=test"); err != nil { t.Error(err) } }) - t.Run("TX without key should fail", func(t *testing.T) { - a := setvar() - if err := a.Init(&md{}, "TX=test"); err == nil { + t.Run("SESSION without key should fail", func(t *testing.T) { + if err := a.Init(&md{}, "SESSION=test"); err == nil { t.Error("expected error") } }) diff --git a/internal/auditlog/concurrent_writer_test.go b/internal/auditlog/concurrent_writer_test.go index 1cbde6613..bee2f9abe 100644 --- a/internal/auditlog/concurrent_writer_test.go +++ b/internal/auditlog/concurrent_writer_test.go @@ -34,7 +34,7 @@ func TestConcurrentWriterNoop(t *testing.T) { func TestConcurrentWriterFailsOnInit(t *testing.T) { config := NewConfig() - config.Target = "/unexisting.log" + config.Target = "/invalid/unexisting.log" config.Dir = t.TempDir() config.FileMode = fs.FileMode(0777) config.DirMode = fs.FileMode(0777) diff --git a/internal/auditlog/serial_writer_test.go b/internal/auditlog/serial_writer_test.go index 72913e13b..4fc732963 100644 --- a/internal/auditlog/serial_writer_test.go +++ b/internal/auditlog/serial_writer_test.go @@ -57,7 +57,7 @@ func TestSerialLoggerSuccessOnInit(t *testing.T) { func TestSerialWriterFailsOnInitForUnexistingFile(t *testing.T) { config := NewConfig() - config.Target = "/unexisting.log" + config.Target = "/invalid/unexisting.log" config.Dir = t.TempDir() config.FileMode = fs.FileMode(0777) config.DirMode = fs.FileMode(0777) diff --git a/internal/bodyprocessors/multipart_test.go b/internal/bodyprocessors/multipart_test.go index 5173a5cb2..6ae4e7783 100644 --- a/internal/bodyprocessors/multipart_test.go +++ b/internal/bodyprocessors/multipart_test.go @@ -26,7 +26,7 @@ func TestProcessRequestFailsDueToIncorrectMimeType(t *testing.T) { expectedError := "not a multipart body" - if err := mp.ProcessRequest(strings.NewReader(""), corazawaf.NewTransactionVariables(), plugintypes.BodyProcessorOptions{ + if err := mp.ProcessRequest(strings.NewReader(""), corazawaf.NewTransactionVariables(nil), plugintypes.BodyProcessorOptions{ Mime: "application/json", }); err == nil || err.Error() != expectedError { t.Fatal("expected error") @@ -56,7 +56,7 @@ Content-Type: text/html mp := multipartProcessor(t) - v := corazawaf.NewTransactionVariables() + v := corazawaf.NewTransactionVariables(nil) if err := mp.ProcessRequest(strings.NewReader(payload), v, plugintypes.BodyProcessorOptions{ Mime: "multipart/form-data; boundary=---------------------------9051914041544843365972754266", }); err != nil { @@ -87,7 +87,7 @@ text default -----------------------------9051914041544843365972754266 `) mp := multipartProcessor(t) - v := corazawaf.NewTransactionVariables() + v := corazawaf.NewTransactionVariables(nil) if err := mp.ProcessRequest(strings.NewReader(payload), v, plugintypes.BodyProcessorOptions{ Mime: "multipart/form-data; boundary=---------------------------9051914041544843365972754266; a=1; a=2", }); err == nil { diff --git a/internal/bodyprocessors/urlencoded_test.go b/internal/bodyprocessors/urlencoded_test.go index c9912483a..d58c2e388 100644 --- a/internal/bodyprocessors/urlencoded_test.go +++ b/internal/bodyprocessors/urlencoded_test.go @@ -18,7 +18,7 @@ func TestURLEncode(t *testing.T) { if err != nil { t.Fatal(err) } - v := corazawaf.NewTransactionVariables() + v := corazawaf.NewTransactionVariables(nil) m := map[string]string{ "a": "1", "b": "2", diff --git a/internal/collections/persistent.go b/internal/collections/persistent.go new file mode 100644 index 000000000..9e8722705 --- /dev/null +++ b/internal/collections/persistent.go @@ -0,0 +1,98 @@ +// Copyright 2023 Juan Pablo Tosso and the OWASP Coraza contributors +// SPDX-License-Identifier: Apache-2.0 + +package collections + +import ( + "regexp" + + "github.com/corazawaf/coraza/v3/collection" + "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" + "github.com/corazawaf/coraza/v3/internal/corazarules" + "github.com/corazawaf/coraza/v3/types" + "github.com/corazawaf/coraza/v3/types/variables" +) + +// Persistent uses collection.Map. +type Persistent struct { + variable variables.RuleVariable + engine plugintypes.PersistenceEngine + collectionKey string +} + +func NewPersistent(variable variables.RuleVariable, engine plugintypes.PersistenceEngine) *Persistent { + return &Persistent{ + variable: variable, + engine: engine, + collectionKey: "", + } +} + +func (c *Persistent) Init(key string) { + c.collectionKey = key +} + +func (c *Persistent) Get(key string) []string { + res, _ := c.engine.Get(c.variable.Name(), c.collectionKey, key) //nolint:errcheck + return []string{res} +} + +func (c *Persistent) FindRegex(key *regexp.Regexp) []types.MatchData { + all, _ := c.engine.All(c.variable.Name(), c.collectionKey) //nolint:errcheck + matches := make([]types.MatchData, 0, len(all)) + for i, v := range all { + if key.MatchString(i) { + matches = append(matches, &corazarules.MatchData{ + Variable_: c.variable, + Key_: i, + Value_: v, + }) + } + } + return matches +} + +func (c *Persistent) FindString(key string) []types.MatchData { + res, _ := c.engine.Get(c.variable.Name(), c.collectionKey, key) //nolint:errcheck + return []types.MatchData{&corazarules.MatchData{ + Variable_: c.variable, + Key_: key, + Value_: res, + }, + } +} + +func (c *Persistent) FindAll() []types.MatchData { + all, _ := c.engine.All(c.variable.Name(), c.collectionKey) //nolint:errcheck + matches := make([]types.MatchData, 0, len(all)) + for i, v := range all { + matches = append(matches, &corazarules.MatchData{ + Variable_: c.variable, + Key_: i, + Value_: v, + }) + } + return matches +} + +func (c *Persistent) SetOne(key string, value string) { + c.engine.Set(c.variable.Name(), c.collectionKey, key, value) //nolint:errcheck +} + +func (c *Persistent) Set(key string, values []string) { + c.engine.Set(c.variable.Name(), c.collectionKey, key, values[0]) //nolint:errcheck +} + +func (c *Persistent) Remove(key string) { + c.engine.Remove(c.variable.Name(), c.collectionKey, key) //nolint:errcheck +} + +func (c *Persistent) Sum(key string, sum int) { + c.engine.Sum(c.variable.Name(), c.collectionKey, key, sum) //nolint:errcheck +} + +func (c *Persistent) Name() string { + return c.variable.Name() +} + +var _ collection.Persistent = &Persistent{} diff --git a/internal/corazawaf/transaction.go b/internal/corazawaf/transaction.go index 5ebfb5306..be7517436 100644 --- a/internal/corazawaf/transaction.go +++ b/internal/corazawaf/transaction.go @@ -271,6 +271,16 @@ func (tx *Transaction) Collection(idx variables.RuleVariable) collection.Collect return tx.variables.xml case variables.MultipartPartHeaders: return tx.variables.multipartPartHeaders + case variables.User: + return tx.variables.user + case variables.Session: + return tx.variables.session + case variables.Global: + return tx.variables.global + case variables.IP: + return tx.variables.ip + case variables.Resource: + return tx.variables.resource } return collections.Noop @@ -543,13 +553,15 @@ func (tx *Transaction) GetField(rv ruleVariableParams) []types.MatchData { if m, ok := col.(collection.Keyed); ok { matches = m.FindRegex(rv.KeyRx) } else { - panic("attempted to use regex with non-selectable collection: " + rv.Variable.Name()) + tx.DebugLogger().Error().Msg("attempted to use regex with non-selectable collection: " + rv.Variable.Name()) + return matches } case rv.KeyStr != "": if m, ok := col.(collection.Keyed); ok { matches = m.FindString(rv.KeyStr) } else { - panic("attempted to use string with non-selectable collection: " + rv.Variable.Name()) + tx.DebugLogger().Error().Msg("attempted to use string with non-selectable collection: " + rv.Variable.Name()) + return matches } default: matches = col.FindAll() @@ -1586,9 +1598,15 @@ type TransactionVariables struct { resBodyErrorMsg *collections.Single resBodyProcessorError *collections.Single resBodyProcessorErrorMsg *collections.Single + // persistent collections + global *collections.Persistent + resource *collections.Persistent + ip *collections.Persistent + session *collections.Persistent + user *collections.Persistent } -func NewTransactionVariables() *TransactionVariables { +func NewTransactionVariables(persistenceEngine plugintypes.PersistenceEngine) *TransactionVariables { v := &TransactionVariables{} v.urlencodedError = collections.NewSingle(variables.UrlencodedError) v.responseContentType = collections.NewSingle(variables.ResponseContentType) @@ -1680,6 +1698,13 @@ func NewTransactionVariables() *TransactionVariables { // Only used in a concatenating collection so variable name doesn't matter. v.argsPath.Names(variables.Unknown), ) + + // Persistent collections + v.global = collections.NewPersistent(variables.Global, persistenceEngine) + v.resource = collections.NewPersistent(variables.Resource, persistenceEngine) + v.ip = collections.NewPersistent(variables.IP, persistenceEngine) + v.session = collections.NewPersistent(variables.Session, persistenceEngine) + v.user = collections.NewPersistent(variables.User, persistenceEngine) return v } diff --git a/internal/corazawaf/waf.go b/internal/corazawaf/waf.go index 7af329a10..f027ded17 100644 --- a/internal/corazawaf/waf.go +++ b/internal/corazawaf/waf.go @@ -18,6 +18,7 @@ import ( "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" "github.com/corazawaf/coraza/v3/internal/auditlog" "github.com/corazawaf/coraza/v3/internal/environment" + "github.com/corazawaf/coraza/v3/internal/persistence" stringutils "github.com/corazawaf/coraza/v3/internal/strings" "github.com/corazawaf/coraza/v3/internal/sync" "github.com/corazawaf/coraza/v3/types" @@ -131,6 +132,9 @@ type WAF struct { // Configures the maximum number of ARGS that will be accepted for processing. ArgumentLimit int + + // PersistenceEngine is used to store persistent collections + PersistenceEngine plugintypes.PersistenceEngine } // NewTransaction Creates a new initialized transaction for this WAF instance @@ -199,7 +203,7 @@ func (w *WAF) newTransactionWithID(id string) *Transaction { Limit: w.ResponseBodyLimit, }) - tx.variables = *NewTransactionVariables() + tx.variables = *NewTransactionVariables(tx.WAF.PersistenceEngine) tx.transformationCache = map[transformationKey]*transformationValue{} } @@ -228,7 +232,7 @@ func (w *WAF) newTransactionWithID(id string) *Transaction { } func resolveLogPath(path string) (io.Writer, error) { - if path == "" { + if path == "" || path == "/dev/null" { return io.Discard, nil } @@ -268,6 +272,10 @@ func NewWAF() *WAF { Err(err). Msg("error creating serial log writer") } + noopPersistenceEngine, err := persistence.Get("noop") + if err != nil { + logger.Error().Err(err).Msg("error creating noop persistence engine") + } waf := &WAF{ // Initializing pool for transactions @@ -283,6 +291,7 @@ func NewWAF() *WAF { AuditLogWriterConfig: auditlog.NewConfig(), Logger: logger, ArgumentLimit: 1000, + PersistenceEngine: noopPersistenceEngine, } if environment.HasAccessToFS { diff --git a/internal/persistence/default.go b/internal/persistence/default.go new file mode 100644 index 000000000..fe9f7f64f --- /dev/null +++ b/internal/persistence/default.go @@ -0,0 +1,189 @@ +// Copyright 2023 Juan Pablo Tosso and the OWASP Coraza contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build !tinygo +// +build !tinygo + +package persistence + +import ( + "fmt" + "strconv" + "sync" + "time" +) + +// defaultEngine +// defaultEngine is just a sample and it shouldn't be used in production. +// It's not thread safe enough and it's not persistent on disk. +type defaultEngine struct { + data sync.Map + ttl int + stopGC chan bool +} + +func (d *defaultEngine) Open(uri string, ttl int) error { + d.data = sync.Map{} + d.ttl = ttl + d.stopGC = make(chan bool) + // we start the garbage collector + go d.gc() + return nil +} + +func (d *defaultEngine) Close() error { + // Close will just stop the GC + // it won't delete the data as it would cause race conditions. + d.stopGC <- true + return nil +} + +func (d *defaultEngine) Sum(collectionName string, collectionKey string, key string, sum int) error { + col := d.getCollection(collectionName, collectionKey) + if col == nil { + d.set(collectionName, collectionKey, key, sum) + } else { + if v, ok := col[key]; ok { + if v2, ok := v.(int); ok { + d.set(collectionName, collectionKey, key, v2+sum) + } + } else { + d.set(collectionName, collectionKey, key, sum) + } + } + return nil +} + +func (d *defaultEngine) Get(collectionName string, collectionKey string, key string) (string, error) { + res := d.get(collectionName, collectionKey, key) + switch v := res.(type) { + case string: + return v, nil + case int: + return strconv.Itoa(v), nil + case nil: + return "", nil + } + + return "", nil +} + +func (d *defaultEngine) Set(collection string, collectionKey string, key string, value string) error { + d.set(collection, collectionKey, key, value) + return nil +} + +func (d *defaultEngine) Remove(collection string, collectionKey string, key string) error { + data := d.getCollection(collection, collectionKey) + if data == nil { + return nil + } + delete(data, key) + return nil +} + +func (d *defaultEngine) All(collectionName string, collectionKey string) (map[string]string, error) { + data := d.getCollection(collectionName, collectionKey) + if data == nil { + return nil, nil + } + res := map[string]string{} + for k, v := range data { + if v == nil { + res[k] = "" + } else { + switch v2 := v.(type) { + case string: + res[k] = v2 + case int: + res[k] = strconv.Itoa(v2) + } + } + } + return res, nil +} + +func (d *defaultEngine) gc() { + for { + select { + case <-d.stopGC: + return + case <-time.After(time.Second * 1): + // this is a concurrent safe sleep + default: + d.data.Range(func(key, value interface{}) bool { + col := value.(map[string]interface{}) + if col["TIMEOUT"].(int) < int(time.Now().Unix()) { + d.data.Delete(key) + } + return true + }) + } + } +} + +func (d *defaultEngine) getCollection(collectionName string, collectionKey string) map[string]interface{} { + k := d.getCollectionName(collectionName, collectionKey) + data, ok := d.data.Load(k) + if !ok { + return nil + } + return data.(map[string]interface{}) +} + +func (d *defaultEngine) get(collectionName string, collectionKey string, key string) interface{} { + data := d.getCollection(collectionName, collectionKey) + if data == nil { + return nil + } + if res, ok := data[key]; ok { + return res + } + return nil +} + +func (d *defaultEngine) set(collection string, collectionKey string, key string, value interface{}) { + data := d.getCollection(collection, collectionKey) + now := time.Now().Unix() + if data == nil { + data := map[string]interface{}{ + key: value, + "CREATE_TIME": int(now), + "IS_NEW": 1, + "KEY": collectionKey, + "LAST_UPDATE_TIME": 0, + // we timeout at now + d.ttl + "TIMEOUT": int(now) + d.ttl, + "UPDATE_COUNTER": 0, + "UPDATE_RATE": 0, + } + d.data.Store(d.getCollectionName(collection, collectionKey), data) + } else { + data[key] = value + d.updateCollection(data) + } +} + +func (*defaultEngine) getCollectionName(collectionName string, collectionKey string) string { + return fmt.Sprintf("%s_%s", collectionName, collectionKey) +} + +func (d *defaultEngine) updateCollection(col map[string]interface{}) { + update_counter := col["UPDATE_COUNTER"].(int) + time_now := int(time.Now().Unix()) + col["IS_NEW"] = 0 + col["LAST_UPDATE_TIME"] = time_now + col["UPDATE_COUNTER"] = update_counter + 1 + // we compute the update rate by using UPDATE_COUNTER and CREATE_TIME + // UPDATE_RATE = UPDATE_COUNTER / (CURRENT_TIME - CREATE_TIME) + delta := (time_now - col["CREATE_TIME"].(int)) + if delta > 0 { + col["UPDATE_RATE"] = int(update_counter / delta) + } + // we update the timeout + col["TIMEOUT"] = time_now + d.ttl +} + +func init() { + Register("default", &defaultEngine{}) +} diff --git a/internal/persistence/default_test.go b/internal/persistence/default_test.go new file mode 100644 index 000000000..1b7ffda4b --- /dev/null +++ b/internal/persistence/default_test.go @@ -0,0 +1,116 @@ +// Copyright 2023 Juan Pablo Tosso and the OWASP Coraza contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build !tinygo +// +build !tinygo + +package persistence + +import ( + "testing" + "time" +) + +func TestDefaultEngineSetAndGet(t *testing.T) { + engine := &defaultEngine{ttl: int(time.Now().Add(10 * time.Minute).Unix())} + err := engine.Set("testCol", "testColKey", "testKey", "testValue") + if err != nil { + t.Errorf("Set failed: %v", err) + } + + val, err := engine.Get("testCol", "testColKey", "testKey") + if err != nil || val != "testValue" { + t.Errorf("Get failed or returned incorrect value: %v, %v", err, val) + } + + // now we test the updates + + err = engine.Set("testCol", "testColKey", "testKey", "testValue2") + if err != nil { + t.Errorf("Set failed: %v", err) + } + + val, err = engine.Get("testCol", "testColKey", "testKey") + if err != nil || val != "testValue2" { + t.Errorf("Get failed or returned incorrect value: %v, %v", err, val) + } + + // now we validate the time update worked + time_now := int(time.Now().Unix()) + create_time := engine.get("testCol", "testColKey", "CREATE_TIME") + if err != nil { + t.Errorf("Get failed: %v", err) + } + ct, ok := create_time.(int) + if !ok { + t.Errorf("Create time is not an int: %v", create_time) + } + if ct == 0 { + t.Errorf("Create time is 0") + } + // time difference should be small + if time_now-ct > 10 { + t.Errorf("Time difference is too big: %v", time_now-int(ct)) + } + + err = engine.Sum("testCol", "testColKey", "sum", 5) + if err != nil { + t.Errorf("Set failed: %v", err) + } + if val := engine.get("testCol", "testColKey", "sum"); val != 5 { + t.Errorf("Sum failed, got %v", val) + } + + err = engine.Sum("testCol", "testColKey", "sum", 2) + if err != nil { + t.Errorf("Set failed: %v", err) + } + if val := engine.get("testCol", "testColKey", "sum"); val != 7 { + t.Errorf("Sum failed, got %v", val) + } + + err = engine.Remove("testCol", "testColKey", "sum") + if err != nil { + t.Errorf("Set failed: %v", err) + } + if val := engine.get("testCol", "testColKey", "sum"); val != nil { + t.Errorf("Sum failed, got %v", val) + } +} + +func TestDefaultGC(t *testing.T) { + engine := &defaultEngine{} + engine.Open("", 1) //nolint:errcheck + defer engine.Close() //nolint:errcheck + err := engine.Set("testCol", "testColKey", "testKey", "testValue") + if err != nil { + t.Errorf("Set failed: %v", err) + } + // we sleep 2 second + time.Sleep(2000 * time.Millisecond) + // now we should have no data + val, err := engine.Get("testCol", "testColKey", "testKey") + if err != nil { + t.Errorf("Get failed or returned incorrect value: %v, %v", err, val) + } + if val == "testValue" { + t.Errorf("Value was not deleted") + } + +} + +func TestDefaultAll(t *testing.T) { + engine := &defaultEngine{} + engine.Open("", 3600) //nolint:errcheck + defer engine.Close() //nolint:errcheck + if err := engine.Set("testCol", "testColKey", "testKey", "testValue"); err != nil { + t.Errorf("Set failed: %v", err) + } + all, err := engine.All("testCol", "testColKey") + if err != nil { + t.Errorf("All failed: %v", err) + } + if all["testKey"] != "testValue" { + t.Errorf("All failed: %v", all) + } +} diff --git a/internal/persistence/noop.go b/internal/persistence/noop.go new file mode 100644 index 000000000..34dd71530 --- /dev/null +++ b/internal/persistence/noop.go @@ -0,0 +1,38 @@ +// Copyright 2023 Juan Pablo Tosso and the OWASP Coraza contributors +// SPDX-License-Identifier: Apache-2.0 + +package persistence + +type noopEngine struct{} + +func (n noopEngine) Open(uri string, ttl int) error { + return nil +} + +func (noopEngine) Close() error { + return nil +} + +func (noopEngine) Sum(collectionName string, collectionKey string, key string, sum int) error { + return nil +} + +func (noopEngine) Get(collectionName string, collectionKey string, key string) (string, error) { + return "", nil +} + +func (noopEngine) Set(collection string, collectionKey string, key string, value string) error { + return nil +} + +func (noopEngine) Remove(collection string, collectionKey string, key string) error { + return nil +} + +func (noopEngine) All(collection string, collectionKey string) (map[string]string, error) { + return nil, nil +} + +func init() { + Register("noop", noopEngine{}) +} diff --git a/internal/persistence/noop_test.go b/internal/persistence/noop_test.go new file mode 100644 index 000000000..5acad135d --- /dev/null +++ b/internal/persistence/noop_test.go @@ -0,0 +1,18 @@ +// Copyright 2023 Juan Pablo Tosso and the OWASP Coraza contributors +// SPDX-License-Identifier: Apache-2.0 + +package persistence + +import "testing" + +func TestNoopEngine(t *testing.T) { + ne, err := Get("noop") + if err != nil { + t.Error("Failed to get noop engine") + } + ne.Open("", 100) //nolint:errcheck + ne.Close() //nolint:errcheck + ne.Get("test", "test", "test") //nolint:errcheck + ne.Set("test", "test", "test", "test") //nolint:errcheck + ne.Remove("test", "test", "test") //nolint:errcheck +} diff --git a/internal/persistence/persistence.go b/internal/persistence/persistence.go new file mode 100644 index 000000000..7661bd08b --- /dev/null +++ b/internal/persistence/persistence.go @@ -0,0 +1,24 @@ +// Copyright 2023 Juan Pablo Tosso and the OWASP Coraza contributors +// SPDX-License-Identifier: Apache-2.0 + +package persistence + +import ( + "fmt" + + "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" +) + +var persistenceEngines = map[string]plugintypes.PersistenceEngine{} + +func Register(name string, engine plugintypes.PersistenceEngine) { + persistenceEngines[name] = engine +} + +func Get(name string) (plugintypes.PersistenceEngine, error) { + if e, ok := persistenceEngines[name]; !ok { + return nil, fmt.Errorf("persistence engine %s not found", name) + } else { + return e, nil + } +} diff --git a/internal/seclang/directives.go b/internal/seclang/directives.go index 7a158c349..4b8948a6f 100644 --- a/internal/seclang/directives.go +++ b/internal/seclang/directives.go @@ -17,6 +17,7 @@ import ( "github.com/corazawaf/coraza/v3/internal/auditlog" "github.com/corazawaf/coraza/v3/internal/corazawaf" "github.com/corazawaf/coraza/v3/internal/memoize" + "github.com/corazawaf/coraza/v3/internal/persistence" utils "github.com/corazawaf/coraza/v3/internal/strings" "github.com/corazawaf/coraza/v3/types" ) @@ -466,6 +467,25 @@ func directiveSecRequestBodyInMemoryLimit(options *DirectiveOptions) error { return nil } +// Description: Set the persistence engine to be used by Coraza. +// Default: noop +// Syntax: SecPersistenceEngine [noop|default] +// --- +// The default persistence engine is a noop engine that does not store any data. +// New engines can be implemented by using experimental/plugins/persistence.go +func directiveSecPersistenceEngine(options *DirectiveOptions) error { + if len(options.Opts) == 0 { + return errEmptyOptions + } + options.WAF.Logger.Warn().Msg("Persistence is a experimental feature, be careful.") + engine, err := persistence.Get(options.Opts) + if err != nil { + return err + } + options.WAF.PersistenceEngine = engine + return nil +} + func directiveSecRemoteRulesFailAction(options *DirectiveOptions) error { if len(options.Opts) == 0 { return errEmptyOptions diff --git a/internal/seclang/directivesmap.gen.go b/internal/seclang/directivesmap.gen.go index 2114ef287..12ebda36d 100644 --- a/internal/seclang/directivesmap.gen.go +++ b/internal/seclang/directivesmap.gen.go @@ -25,6 +25,7 @@ var ( _ directive = directiveSecResponseBodyLimit _ directive = directiveSecRequestBodyLimitAction _ directive = directiveSecRequestBodyInMemoryLimit + _ directive = directiveSecPersistenceEngine _ directive = directiveSecRemoteRulesFailAction _ directive = directiveSecRemoteRules _ directive = directiveSecConnWriteStateLimit @@ -85,6 +86,7 @@ var directivesMap = map[string]directive{ "secresponsebodylimit": directiveSecResponseBodyLimit, "secrequestbodylimitaction": directiveSecRequestBodyLimitAction, "secrequestbodyinmemorylimit": directiveSecRequestBodyInMemoryLimit, + "secpersistenceengine": directiveSecPersistenceEngine, "secremoterulesfailaction": directiveSecRemoteRulesFailAction, "secremoterules": directiveSecRemoteRules, "secconnwritestatelimit": directiveSecConnWriteStateLimit, diff --git a/internal/variables/variables.go b/internal/variables/variables.go index 717b18f8b..ed56a1029 100644 --- a/internal/variables/variables.go +++ b/internal/variables/variables.go @@ -222,12 +222,20 @@ const ( MultipartUnmatchedBoundary // PathInfo is kept for compatibility PathInfo - // Sessionid is not supported + // Sessionid is the session id Sessionid - // Userid is not supported + // Userid is a persistent collection of user ids Userid - // IP is kept for compatibility + // IP is a persistent collection of IP addresses IP + // Global is a persistent collection of global variables + Global + // Resource is a persistent collection of resources + Resource + // User is a persistent collection of user variables + User + // Session is a persistent collection of session variables + Session // ResBodyError ResBodyError // ResBodyErrorMsg diff --git a/internal/variables/variablesmap.gen.go b/internal/variables/variablesmap.gen.go index 8b707aced..99a12989e 100644 --- a/internal/variables/variablesmap.gen.go +++ b/internal/variables/variablesmap.gen.go @@ -198,6 +198,14 @@ func (v RuleVariable) Name() string { return "USERID" case IP: return "IP" + case Global: + return "GLOBAL" + case Resource: + return "RESOURCE" + case User: + return "USER" + case Session: + return "SESSION" case ResBodyError: return "RES_BODY_ERROR" case ResBodyErrorMsg: @@ -305,6 +313,10 @@ var rulemapRev = map[string]RuleVariable{ "SESSIONID": Sessionid, "USERID": Userid, "IP": IP, + "GLOBAL": Global, + "RESOURCE": Resource, + "USER": User, + "SESSION": Session, "RES_BODY_ERROR": ResBodyError, "RES_BODY_ERROR_MSG": ResBodyErrorMsg, "RES_BODY_PROCESSOR_ERROR": ResBodyProcessorError, diff --git a/magefile.go b/magefile.go index 960f0e418..98557a33e 100644 --- a/magefile.go +++ b/magefile.go @@ -210,6 +210,11 @@ func Doc() error { return sh.RunV("go", "run", "golang.org/x/tools/cmd/godoc@latest", "-http=:6060") } +// Generate generates code using available generators. +func Generate() error { + return sh.RunV("go", "generate", "./...") +} + // Precommit installs a git hook to run check when committing func Precommit() error { if _, err := os.Stat(filepath.Join(".git", "hooks")); os.IsNotExist(err) { diff --git a/testing/engine/persistence.go b/testing/engine/persistence.go new file mode 100644 index 000000000..cb6ede88a --- /dev/null +++ b/testing/engine/persistence.go @@ -0,0 +1,68 @@ +// Copyright 2023 Juan Pablo Tosso and the OWASP Coraza contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build !tinygo +// +build !tinygo + +package engine + +import ( + "github.com/corazawaf/coraza/v3/testing/profile" +) + +var _ = profile.RegisterProfile(profile.Profile{ + Meta: profile.Meta{ + Author: "jptosso", + Description: "Test if persistence works", + Enabled: true, + Name: "persistence.yaml", + }, + Tests: []profile.Test{ + { + Title: "persistence", + Stages: []profile.Stage{ + { + Stage: profile.SubStage{ + Input: profile.StageInput{ + URI: "/test1", + Headers: map[string]string{ + "ghi": "pineapple", + "cookie": "session=test;", + }, + }, + Output: profile.ExpectedOutput{ + TriggeredRules: []int{1, 2}, + }, + }, + }, + }, + }, + { + Title: "persistence", + Stages: []profile.Stage{ + { + Stage: profile.SubStage{ + Input: profile.StageInput{ + URI: "/test2", + Headers: map[string]string{ + "ghi": "pineapple", + "cookie": "session=test;", + }, + }, + Output: profile.ExpectedOutput{ + TriggeredRules: []int{1, 3}, + }, + }, + }, + }, + }, + }, + Rules: ` +SecPersistenceEngine default +SecAction "id:1,phase:1,initcol:session=%{REQUEST_COOKIES.session},pass,nolog" +SecRule REQUEST_URI "test1" "id:2,phase:2,pass,nolog,setvar:session.test=1" +SecRule REQUEST_URI "test2" "id:3,phase:2,pass,nolog,chain" + SecRule SESSION:test "1" "log,chain" + SecRule SESSION:/te.*/ "1" "log" +`, +}) diff --git a/types/variables/variables.go b/types/variables/variables.go index c8363fa3b..3cb00518c 100644 --- a/types/variables/variables.go +++ b/types/variables/variables.go @@ -194,6 +194,16 @@ const ( ResBodyProcessorError = variables.ResBodyProcessorError // ResBodyProcessorErrorMsg contains the error message if the response body processor failed ResBodyProcessorErrorMsg = variables.ResBodyProcessorErrorMsg + // Global contains global persistent data + Global = variables.Global + // Resource contains the persistent resource data + Resource = variables.Resource + // IP contains the persistent IP information + IP = variables.IP + // Session contains the persistent session information + Session = variables.Session + // User contains the persistent user information + User = variables.User ) // Parse returns the byte interpretation