diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ca33df11..9f511a66 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,6 +15,8 @@ on: - master - development pull_request: + branches-ignore: + - dependabot/** jobs: checks: diff --git a/.github/workflows/review.yaml b/.github/workflows/review.yaml index 96eeb27a..d972a72d 100644 --- a/.github/workflows/review.yaml +++ b/.github/workflows/review.yaml @@ -1,13 +1,18 @@ name: Review on: + push: + branches: + - master pull_request: branches: - - development + - master + - development jobs: review: name: Code Review - runs-on: ubuntu-latest + runs-on: ubuntu-latest + if: startsWith(github.head_ref, 'dependabot/') == false steps: - name: Check out code uses: actions/checkout@v2.3.4 @@ -16,4 +21,15 @@ jobs: uses: kitabisa/sonarqube-action@development with: host: ${{ secrets.SONARQUBE_HOST }} - login: ${{ secrets.SONARQUBE_TOKEN }} \ No newline at end of file + login: ${{ secrets.SONARQUBE_TOKEN }} + + - name: Run Semgrep + uses: returntocorp/semgrep-action@v1 + with: + config: | + p/ci + p/owasp-top-ten + p/golang + p/command-injection + p/security-audit + p/secrets \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b020046..697fe624 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project should be documented in this file. +### v2.0.0-beta + +- Add supporting RAW requests from CVEs templates +- Add Zinc logs engine +- Refactoring configuration structures +- Remove `-o/--output` & `--json` flags +- Add custom threat rules + ### v1.2.2 - Add utility for get datasets diff --git a/README.md b/README.md index e96f5f7c..f53854ab 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,12 @@ [![teler](https://user-images.githubusercontent.com/25837540/97096468-f8ccaa00-1696-11eb-8830-0d3a7be45a2d.gif)](#) +> ### :warning: Important notes +> If you upgrade from prior to v2 frontwards there will be some **break changes** that affect configuration files. +> Appropriate adaptations can refer to [teler.example.yaml](https://github.com/kitabisa/teler/blob/master/teler.example.yaml) file. +> +> See the exact changes in the [CHANGELOG.md](#changes) file. + ## Table of Contents - [Features](#features) - [Why teler?](#why-teler) @@ -51,6 +57,8 @@ * **Monitoring**: We've our own metrics if you want to monitor threats easily, and we use Prometheus for that. +* **Logging**: is also provided in file form or sends detected threats to the Zinc logs search engine. + * **Latest resources**: Collections is continuously up-to-date. * **Minimal configuration**: You can just run it against your log file, write the log format and let @@ -58,6 +66,8 @@ * **Flexible log formats**: teler allows any custom log format string! It all depends on how you write the log format in configuration file. +* **Custom threat rules**: Want to reach a wider range of threats instead of engine-based _(default)_ rules? You can customize threat rules! + * **Incremental log processing**: Need data persistence rather than [buffer stream](https://linux.die.net/man/1/stdbuf)? teler has the ability to process logs incrementally through the on-disk persistence options. @@ -128,7 +138,7 @@ All external resources used in this teler are **NOT** provided by us. See all pe ## Pronunciation -/télér/ bagaimana bisa seorang pemuda itu teler hanya dengan meminum 1 sloki ciu _(?)_ +[`jv_id`](https://www.localeplanet.com/java/jv-ID/index.html) • **/télér/** — bagaimana bisa seorang pemuda itu teler hanya dengan meminum sloki ciu _(?)_ ## Changes diff --git a/common/options.go b/common/options.go index 4aa980dc..440b1686 100644 --- a/common/options.go +++ b/common/options.go @@ -13,8 +13,7 @@ type Options struct { Stdin bool // Stdin specifies whether stdin input was given to the process Version bool // Version check of teler flag Input string // Parse log from data persistence rather than buffer stream - Output string // Save detected threats to file - OutFile *os.File // Write log output into file + Output *os.File // Write log output into file Configs *parsers.Configs // Get teler configuration interface JSON bool // Display threats in the terminal as JSON format RmCache bool // To remove all cached resources on local diff --git a/internal/alert/telegram.go b/internal/alert/telegram.go index 80e2d91f..9bacdd23 100644 --- a/internal/alert/telegram.go +++ b/internal/alert/telegram.go @@ -2,8 +2,8 @@ package alert import ( "bytes" + "html/template" "strconv" - "text/template" telegramBot "github.com/go-telegram-bot-api/telegram-bot-api" "ktbs.dev/teler/pkg/errors" @@ -31,12 +31,12 @@ func toTelegram(token string, chatID string, log map[string]string) { func telegramMessage(log map[string]string) string { var buffer bytes.Buffer - template, err := template.ParseFiles("internal/alert/template/telegram.tmpl") + tpl, err := template.ParseFiles("internal/alert/template/telegram.tmpl") if err != nil { errors.Exit(err.Error()) } - err = template.Execute(&buffer, log) + err = tpl.Execute(&buffer, log) if err != nil { errors.Exit(err.Error()) } diff --git a/internal/runner/logs.go b/internal/runner/logs.go new file mode 100644 index 00000000..45f84966 --- /dev/null +++ b/internal/runner/logs.go @@ -0,0 +1,48 @@ +package runner + +import ( + "fmt" + "reflect" + + "ktbs.dev/teler/common" + "ktbs.dev/teler/pkg/errors" + "ktbs.dev/teler/pkg/logs" +) + +func log(options *common.Options, data map[string]string) { + m := options.Configs.Logs + v := reflect.ValueOf(m) + t := v.Type() + + for i := 0; i < v.NumField(); i++ { + if v.Field(i).FieldByName("Active").Bool() { + switch t.Field(i).Name { + case "File": + toFile(options, data) + case "Zinc": + toZinc(options, data) + } + } + } +} + +func toFile(options *common.Options, data map[string]string) { + err := logs.File(options, data) + if err != nil { + errors.Show(err.Error()) + } +} + +func toZinc(options *common.Options, data map[string]string) { + zinc := options.Configs.Logs.Zinc + base := "http" + if zinc.SSL { + base += "s" + } + base += fmt.Sprint("://", zinc.Host, ":", zinc.Port) + + err := logs.Zinc(base, zinc.Index, zinc.Base64Auth, data) + if err != nil { + errors.Show(err.Error()) + } +} diff --git a/internal/runner/metrics.go b/internal/runner/metrics.go new file mode 100644 index 00000000..091fb8e4 --- /dev/null +++ b/internal/runner/metrics.go @@ -0,0 +1,60 @@ +package runner + +import ( + "fmt" + "net/http" + "reflect" + "strconv" + + "github.com/projectdiscovery/gologger" + "github.com/prometheus/client_golang/prometheus/promhttp" + "ktbs.dev/teler/common" + "ktbs.dev/teler/pkg/errors" + "ktbs.dev/teler/pkg/metrics" +) + +func metric(options *common.Options) { + m := options.Configs.Metrics + v := reflect.ValueOf(m) + t := v.Type() + + for i := 0; i < v.NumField(); i++ { + if v.Field(i).FieldByName("Active").Bool() { + switch t.Field(i).Name { + case "Prometheus": + startPrometheus(options) + } + } + } +} + +func startPrometheus(options *common.Options) { + p := options.Configs.Metrics.Prometheus + + if p.Host == "" { + p.Host = "127.0.0.1" + } + + if p.Port == 0 { + p.Port = 9090 + } + + if p.Endpoint == "" { + p.Endpoint = "/metrics" + } + + s := fmt.Sprint(p.Host, ":", strconv.Itoa(p.Port)) + e := p.Endpoint + + go func() { + http.Handle(e, promhttp.Handler()) + + err := http.ListenAndServe(s, nil) // nosemgrep: go.lang.security.audit.net.use-tls.use-tls + if err != nil { + errors.Exit(err.Error()) + } + }() + + metrics.Prometheus() + gologger.Info().Msgf(fmt.Sprint("Listening metrics on http://", s, e)) +} diff --git a/internal/runner/options.go b/internal/runner/options.go index d87232fb..f9d5e211 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -25,9 +25,6 @@ func ParseOptions() *common.Options { flag.IntVar(&options.Concurrency, "x", 20, "") flag.IntVar(&options.Concurrency, "concurrent", 20, "") - flag.StringVar(&options.Output, "o", "", "") - flag.StringVar(&options.Output, "output", "", "") - flag.BoolVar(&options.Version, "v", false, "") flag.BoolVar(&options.Version, "version", false, "") @@ -47,8 +44,6 @@ func ParseOptions() *common.Options { " -c, --config teler configuration file", " -i, --input Analyze logs from data persistence rather than buffer stream", " -x, --concurrent Set the concurrency level to analyze logs (default: 20)", - " -o, --output Save detected threats to file", - " --json Display threats in the terminal as JSON format", " --rm-cache Removes all cached resources", " -v, --version Show current teler version", "", diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 24bc5739..b9321244 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -1,18 +1,13 @@ package runner import ( - "encoding/json" "fmt" "io" - "net/http" "os" "os/signal" - "regexp" - "github.com/acarl005/stripansi" "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/remeh/sizedwaitgroup" "github.com/satyrius/gonx" "ktbs.dev/teler/common" @@ -22,31 +17,14 @@ import ( "ktbs.dev/teler/pkg/teler" ) -func removeLBR(s string) string { - re := regexp.MustCompile(`\x{000D}\x{000A}|[\x{000A}\x{000B}\x{000C}\x{000D}\x{0085}\x{2028}\x{2029}]`) - return re.ReplaceAllString(s, ``) -} - // New read & pass stdin log func New(options *common.Options) { - var input *os.File - var out string - var pass int - - metric, promserve, promendpoint := prometheus(options) - if metric { - go func() { - http.Handle(promendpoint, promhttp.Handler()) + var ( + input *os.File + pass int + ) - err := http.ListenAndServe(promserve, nil) - if err != nil { - errors.Exit(err.Error()) - } - }() - - metrics.Init() - gologger.Info().Msgf("Listening metrics on http://" + promserve + promendpoint) - } + go metric(options) jobs := make(chan *gonx.Entry) gologger.Info().Msg("Analyzing...") @@ -64,48 +42,25 @@ func New(options *common.Options) { con := options.Concurrency swg := sizedwaitgroup.New(con) go func() { - for log := range jobs { + for job := range jobs { swg.Add() go func(line *gonx.Entry) { defer swg.Done() threat, obj := teler.Analyze(options, line) - if threat { - if metric { - metrics.GetThreatTotal.WithLabelValues(obj["category"]).Inc() - } - - if options.JSON { - json, err := json.Marshal(obj) - if err != nil { - errors.Exit(err.Error()) - } - out = fmt.Sprintf("%s\n", json) - } else { - out = fmt.Sprintf("[%s] [%s] [%s] %s\n", - aurora.Cyan(obj["time_local"]), - aurora.Green(obj["remote_addr"]), - aurora.Yellow(obj["category"]), - aurora.Red(obj[obj["element"]]), - ) - } - - fmt.Print(out) - - if options.Output != "" { - if !options.JSON { - out = stripansi.Strip(out) - } - - if _, write := options.OutFile.WriteString(out); write != nil { - errors.Show(write.Error()) - } - } + fmt.Printf("[%s] [%s] [%s] %s\n", + aurora.Cyan(obj["time_local"]), + aurora.Green(obj["remote_addr"]), + aurora.Yellow(obj["category"]), + aurora.Red(obj[obj["element"]]), + ) alert.New(options, common.Version, obj) + log(options, obj) + metrics.PrometheusInsert(obj) } - }(log) + }(job) } }() diff --git a/internal/runner/utils.go b/internal/runner/utils.go new file mode 100644 index 00000000..52af18f7 --- /dev/null +++ b/internal/runner/utils.go @@ -0,0 +1,8 @@ +package runner + +import "regexp" + +func removeLBR(s string) string { + re := regexp.MustCompile(`\x{000D}\x{000A}|[\x{000A}\x{000B}\x{000C}\x{000D}\x{0085}\x{2028}\x{2029}]`) + return re.ReplaceAllString(s, ``) +} diff --git a/internal/runner/validator.go b/internal/runner/validator.go index 9c2f5ca0..aba3e3b2 100644 --- a/internal/runner/validator.go +++ b/internal/runner/validator.go @@ -1,9 +1,14 @@ package runner import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" "os" "reflect" - "strconv" "strings" "gopkg.in/validator.v2" @@ -29,51 +34,144 @@ func validate(options *common.Options) { } } - if options.Output != "" { - f, errOutput := os.OpenFile(options.Output, + config, errConfig := parsers.GetConfig(options.ConfigFile) + if errConfig != nil { + errors.Exit(errors.ErrParseConfig + errConfig.Error()) + } + + if config.Logs.File.Active { + if config.Logs.File.Path == "" { + errors.Exit(errors.ErrNoFilePath) + } + + f, errOutput := os.OpenFile(config.Logs.File.Path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if errOutput != nil { errors.Exit(errOutput.Error()) } - options.OutFile = f - } - - config, errConfig := parsers.GetConfig(options.ConfigFile) - if errConfig != nil { - errors.Exit(errConfig.Error()) + options.Output = f } // Validates log format matchers.IsLogformat(config.Logformat) options.Configs = config - // Validates notification parts on configuration files + // Validates custom threat rules & notification parts configuration + customs(options) notification(options) + // Do Zinc health check, validate & set credentials + if config.Logs.Zinc.Active { + options.Configs.Logs.Zinc.Base64Auth = zinc(options) + } + if errVal := validator.Validate(options); errVal != nil { errors.Exit(errVal.Error()) } } -func prometheus(options *common.Options) (bool, string, string) { - config := options.Configs - if config.Prometheus.Active { - if config.Prometheus.Host == "" { - config.Prometheus.Host = "127.0.0.1" +func customs(options *common.Options) { + var err string + + cfg := options.Configs + cat := make(map[string]bool) + + custom := cfg.Rules.Threat.Customs + for i := 0; i < len(custom); i++ { + cond := strings.ToLower(custom[i].Condition) + matchers.IsCondition(cond) + matchers.IsBlank(custom[i].Name, "Custom threat category") + + if cat[custom[i].Name] { + err = strings.Replace(errors.ErrDupeCategory, ":category", custom[i].Name, -1) + errors.Exit(err) } + cat[custom[i].Name] = true - if config.Prometheus.Port == 0 { - config.Prometheus.Port = 9090 + rules := custom[i].Rules + if len(rules) < 1 { + err = strings.Replace(errors.ErrNoThreatRules, ":category", custom[i].Name, -1) + errors.Exit(err) } - if config.Prometheus.Endpoint == "" { - config.Prometheus.Endpoint = "/metrics" + for j := 0; j < len(rules); j++ { + matchers.IsBlank(rules[j].Element, "Custom threat rules element") + elm := fmt.Sprint("$", rules[j].Element) + + if !matchers.IsMatch(fmt.Sprint(`\`, elm), cfg.Logformat) { + err = strings.Replace(errors.ErrNoElement, ":element", elm, -1) + err = strings.Replace(err, ":category", custom[i].Name, -1) + + errors.Exit(err) + } + + matchers.IsBlank(rules[j].Pattern, "Custom threat rules pattern") } } +} + +func zinc(options *common.Options) string { + var health, auth map[string]interface{} + + zinc := options.Configs.Logs.Zinc + base := "http" + if zinc.SSL { + base += "s" + } + base += fmt.Sprint("://", zinc.Host, ":", zinc.Port) + + resp, err := http.Get(fmt.Sprint(base, "/healthz")) + if err != nil { + errors.Exit(fmt.Sprint(errors.ErrHealthZinc, ": ", err.Error())) + } + defer resp.Body.Close() - server := config.Prometheus.Host + ":" + strconv.Itoa(config.Prometheus.Port) + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + errors.Exit(fmt.Sprint(errors.ErrHealthZinc, ": ", err.Error())) + } + + if err = json.Unmarshal(body, &health); err != nil { + errors.Exit(fmt.Sprint(errors.ErrHealthZinc, ": ", err.Error())) + } + + if health["status"] != "ok" { + errors.Exit(errors.ErrHealthZinc) + } + + b64auth := base64.StdEncoding.EncodeToString([]byte( + fmt.Sprint(zinc.Username, ":", zinc.Password), + )) + data, _ := json.Marshal(map[string]string{ + "_id": zinc.Username, + "base64encoded": b64auth, + "password": zinc.Password, + }) + + resp, err = http.Post( + fmt.Sprint(base, "/api/login"), + "application/json", + bytes.NewBuffer(data), + ) + if err != nil { + errors.Exit(fmt.Sprint(errors.ErrAuthZinc, ": ", err.Error())) + } + defer resp.Body.Close() + + body, err = ioutil.ReadAll(resp.Body) + if err != nil { + errors.Exit(fmt.Sprint(errors.ErrAuthZinc, ": ", err.Error())) + } + + if err = json.Unmarshal(body, &auth); err != nil { + errors.Exit(fmt.Sprint(errors.ErrAuthZinc, ": ", err.Error())) + } + + if auth["validated"] == false { + errors.Exit(errors.ErrAuthZinc) + } - return config.Prometheus.Active, server, config.Prometheus.Endpoint + return b64auth } func notification(options *common.Options) { diff --git a/pkg/errors/constants.go b/pkg/errors/constants.go new file mode 100644 index 00000000..fb80fcf1 --- /dev/null +++ b/pkg/errors/constants.go @@ -0,0 +1,21 @@ +package errors + +const ( + ErrAlertProvider = "Provider \":platform\" not available; " + ErrCheckConfig + ErrAuthZinc = "Invalid Zinc credentials" + ErrBlankField = ":field can't be blank; " + ErrCheckConfig + ErrCheckConfig = "please check your config file" + ErrConfigValidate = "Only validates :key; " + ErrCheckConfig + ErrDupeCategory = "Duplicated name for ':category' threat category; " + ErrCheckConfig + ErrHealthZinc = "Zinc log server is not running" + ErrInsertLogZinc = "Failed to insert logs to Zinc" + ErrNoElement = "Can't find ':element' on log format for ':category' threat category; " + ErrCheckConfig + ErrNoFilePath = "No file path specified; " + ErrCheckConfig + ErrNoIndexZinc = "No index provided for Zinc log server; " + ErrCheckConfig + ErrNoInputConfig = "No config file specified" + ErrNoInputLog = "No input logs provided" + ErrNoPassZinc = "No password provided for Zinc log server; " + ErrCheckConfig + ErrNoThreatRules = "No rules for ':category' threat category; " + ErrCheckConfig + ErrNoUserZinc = "No username provided for Zinc log server; " + ErrCheckConfig + ErrParseConfig = "Can't parse config file: " +) diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 342237c7..b76fdc1f 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -1,6 +1,7 @@ package errors import ( + "bufio" "strings" "github.com/projectdiscovery/gologger" @@ -8,12 +9,22 @@ import ( // Exit will show error details and stop the program func Exit(err string) { - msg := "Error! " if err != "" { - for _, e := range strings.Split(strings.TrimSuffix(err, "\n"), "\n") { - msg += e + count := 0 + lines := bufio.NewScanner(strings.NewReader(err)) + + for lines.Scan() { + var msg string + + if count == 0 { + msg = "Error! " + } + msg += strings.TrimSpace(lines.Text()) + Show(msg) + count++ } + gologger.Info().Msgf("Use \"-h\" flag for more info about command.") Abort(9) } @@ -21,5 +32,5 @@ func Exit(err string) { // Show error message func Show(msg string) { - gologger.Error().Msgf("%s\n", msg) + gologger.Error().Msg(msg) } diff --git a/pkg/errors/messages.go b/pkg/errors/messages.go deleted file mode 100644 index f3ac7a10..00000000 --- a/pkg/errors/messages.go +++ /dev/null @@ -1,9 +0,0 @@ -package errors - -const ( - ErrCheckConfig = "please check your config file" - ErrConfigValidate = "Only validates :key; " + ErrCheckConfig - ErrAlertProvider = "Provider \":platform\" not available; " + ErrCheckConfig - ErrNoInputLog = "No input logs provided" - ErrNoInputConfig = "No config file specified" -) diff --git a/pkg/logs/file.go b/pkg/logs/file.go new file mode 100644 index 00000000..00dcb218 --- /dev/null +++ b/pkg/logs/file.go @@ -0,0 +1,37 @@ +package logs + +import ( + "encoding/json" + "fmt" + + "ktbs.dev/teler/common" +) + +// File write detected threats into it +func File(options *common.Options, data map[string]string) error { + var out string + file := options.Configs.Logs.File + + if options.Output != nil { + if file.JSON { + logJSON, err := json.Marshal(data) + if err != nil { + return err + } + out = fmt.Sprintf("%s\n", logJSON) + } else { + out = fmt.Sprintf("[%s] [%s] [%s] %s\n", + data["time_local"], + data["remote_addr"], + data["category"], + data[data["element"]], + ) + } + + if _, write := options.Output.WriteString(out); write != nil { + return write + } + } + + return nil +} diff --git a/pkg/logs/zinc.go b/pkg/logs/zinc.go new file mode 100644 index 00000000..b86f35a0 --- /dev/null +++ b/pkg/logs/zinc.go @@ -0,0 +1,51 @@ +package logs + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + + e "ktbs.dev/teler/pkg/errors" +) + +// Zinc logs insertion +func Zinc(base string, index string, auth string, log map[string]string) error { + var res map[string]string + client := &http.Client{} + + data, err := json.Marshal(log) + if err != nil { + return err + } + + req, err := http.NewRequest("PUT", fmt.Sprint(base, "/api/", index, "/document"), bytes.NewBuffer(data)) + if err != nil { + panic(err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Basic "+auth) + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + if err = json.Unmarshal(body, &res); err != nil { + return err + } + + if res["id"] != "" { + return nil + } + + return errors.New(e.ErrInsertLogZinc) +} diff --git a/pkg/matchers/config.go b/pkg/matchers/config.go index ffecab2d..b9647a11 100644 --- a/pkg/matchers/config.go +++ b/pkg/matchers/config.go @@ -46,3 +46,21 @@ func IsChatID(s string) { errValidate("chat_id") } } + +// IsCondition validates custom threat rules condition +func IsCondition(s string) { + switch s { + case "or", "and": + default: + errValidate("AND/OR for condition") + } +} + +// IsBlank validates nil field value +func IsBlank(s string, field string) { + s = strings.TrimSpace(s) + if s == "" { + err := strings.Replace(errors.ErrBlankField, ":field", field, -1) + errors.Exit(err) + } +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index f54005dc..761b7c11 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -52,7 +52,7 @@ var ( []string{"http_referer"}, ) - GetThreatTotal = prometheus.NewCounterVec( + getThreatTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "teler_threats_count_total", Help: "Total number of detected threats", @@ -61,10 +61,10 @@ var ( ) ) -// Init will register a Prometheus metrics with the specified variables -func Init() { +// Prometheus will register a metrics with the specified variables +func Prometheus() { prometheus.MustRegister( getBadCrawler, getDirBruteforce, getBadIP, - getCWA, getCVE, getBadReferrer, GetThreatTotal, + getCWA, getCVE, getBadReferrer, getThreatTotal, ) } diff --git a/pkg/metrics/prometheus.go b/pkg/metrics/prometheus.go new file mode 100644 index 00000000..a407bec3 --- /dev/null +++ b/pkg/metrics/prometheus.go @@ -0,0 +1,54 @@ +package metrics + +import ( + "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +// PrometheusInsert logs into metrics +func PrometheusInsert(data map[string]string) { + var counter prometheus.Counter + + switch { + case strings.HasPrefix(data["category"], "Common Web Attack"): + counter = getCWA.WithLabelValues( + data["category"], + data["remote_addr"], + data["request_uri"], + data["status"], + ) + case strings.HasPrefix(data["category"], "CVE-"): + counter = getCVE.WithLabelValues( + data["category"], + data["remote_addr"], + data["request_uri"], + data["status"], + ) + case data["category"] == "Bad Crawler": + counter = getBadCrawler.WithLabelValues( + data["remote_addr"], + data["http_user_agent"], + data["status"], + ) + case data["category"] == "Bad IP Address": + counter = getBadIP.WithLabelValues( + data["remote_addr"], + ) + case data["category"] == "Bad Referrer": + counter = getBadReferrer.WithLabelValues( + data["http_referer"], + ) + case data["category"] == "Directory Bruteforce": + counter = getDirBruteforce.WithLabelValues( + data["remote_addr"], + data["request_uri"], + data["status"], + ) + default: + return + } + + counter.Inc() + getThreatTotal.WithLabelValues(data["category"]).Inc() +} diff --git a/pkg/metrics/send.go b/pkg/metrics/send.go deleted file mode 100644 index ac9ba036..00000000 --- a/pkg/metrics/send.go +++ /dev/null @@ -1,51 +0,0 @@ -package metrics - -import ( - "strings" - - "github.com/prometheus/client_golang/prometheus" -) - -// Send logs to metrics -func Send(log map[string]string) { - var counter prometheus.Counter - - switch { - case strings.HasPrefix(log["category"], "Common Web Attack"): - counter = getCWA.WithLabelValues( - log["category"], - log["remote_addr"], - log["request_uri"], - log["status"], - ) - case strings.HasPrefix(log["category"], "CVE-"): - counter = getCVE.WithLabelValues( - log["category"], - log["remote_addr"], - log["request_uri"], - log["status"], - ) - case log["category"] == "Bad Crawler": - counter = getBadCrawler.WithLabelValues( - log["remote_addr"], - log["http_user_agent"], - log["status"], - ) - case log["category"] == "Bad IP Address": - counter = getBadIP.WithLabelValues( - log["remote_addr"], - ) - case log["category"] == "Bad Referrer": - counter = getBadReferrer.WithLabelValues( - log["http_referer"], - ) - case log["category"] == "Directory Bruteforce": - counter = getDirBruteforce.WithLabelValues( - log["remote_addr"], - log["request_uri"], - log["status"], - ) - } - - counter.Inc() -} diff --git a/pkg/parsers/config.go b/pkg/parsers/config.go index ef264761..116e7d5b 100644 --- a/pkg/parsers/config.go +++ b/pkg/parsers/config.go @@ -1,24 +1,6 @@ package parsers -import "io/ioutil" - -type options struct { - Excludes []string `yaml:"excludes"` - Whitelists []string `yaml:"whitelists"` -} - -type general struct { - Token string `yaml:"token"` - Color string `yaml:"color"` - Channel string `yaml:"channel"` -} - -type telegram struct { - Token string `yaml:"token"` - ChatID string `yaml:"chat_id"` -} - -// Configs default structure for config +// Configs default structure for configurations type Configs struct { Logformat string `yaml:"log_format" validate:"nonzero"` @@ -27,12 +9,14 @@ type Configs struct { Threat options `yaml:"threat" validate:"nonzero"` } `yaml:"rules" validate:"nonzero"` - Prometheus struct { - Active bool `yaml:"active"` - Host string `yaml:"host"` - Port int `yaml:"port"` - Endpoint string `yaml:"endpoint"` - } `yaml:"prometheus" validate:"nonzero"` + Metrics struct { + Prometheus prometheus `yaml:"prometheus"` + } `yaml:"metrics" validate:"nonzero"` + + Logs struct { + File file `yaml:"file"` + Zinc zinc `yaml:"zinc"` + } `yaml:"logs" validate:"nonzero"` Alert struct { Active bool `yaml:"active"` @@ -40,23 +24,8 @@ type Configs struct { } `yaml:"alert" validate:"nonzero"` Notifications struct { - Slack general `yaml:"slack"` + Slack general `yaml:"slack"` Telegram telegram `yaml:"telegram"` Discord general `yaml:"discord"` } `yaml:"notifications"` } - -// GetConfig will parse the config file -func GetConfig(f string) (*Configs, error) { - config := &Configs{} - file, err := ioutil.ReadFile(f) - if err != nil { - return nil, err - } - err = GetYaml(file, config) - if err != nil { - return nil, err - } - - return config, nil -} diff --git a/pkg/parsers/get.go b/pkg/parsers/get.go new file mode 100644 index 00000000..b63d884a --- /dev/null +++ b/pkg/parsers/get.go @@ -0,0 +1,28 @@ +package parsers + +import ( + "io/ioutil" + + "gopkg.in/yaml.v2" +) + +// GetConfig will parse the config file +func GetConfig(f string) (*Configs, error) { + config := &Configs{} + file, err := ioutil.ReadFile(f) + if err != nil { + return nil, err + } + err = GetYaml(file, config) + if err != nil { + return nil, err + } + + return config, nil +} + +// GetYaml file configuration +func GetYaml(f []byte, s interface{}) error { + y := yaml.Unmarshal(f, s) + return y +} diff --git a/pkg/parsers/logs.go b/pkg/parsers/logs.go new file mode 100644 index 00000000..f80627d1 --- /dev/null +++ b/pkg/parsers/logs.go @@ -0,0 +1,18 @@ +package parsers + +type zinc struct { + Active bool `yaml:"active"` + Host string `yaml:"host"` + Port int `yaml:"port"` + SSL bool `yaml:"ssl"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Index string `yaml:"index"` + Base64Auth string +} + +type file struct { + Active bool `yaml:"active"` + JSON bool `yaml:"json"` + Path string `yaml:"path"` +} diff --git a/pkg/parsers/metrics.go b/pkg/parsers/metrics.go new file mode 100644 index 00000000..b6a3f10a --- /dev/null +++ b/pkg/parsers/metrics.go @@ -0,0 +1,8 @@ +package parsers + +type prometheus struct { + Active bool `yaml:"active"` + Host string `yaml:"host"` + Port int `yaml:"port"` + Endpoint string `yaml:"endpoint"` +} diff --git a/pkg/parsers/notifications.go b/pkg/parsers/notifications.go new file mode 100644 index 00000000..23547493 --- /dev/null +++ b/pkg/parsers/notifications.go @@ -0,0 +1,12 @@ +package parsers + +type general struct { + Token string `yaml:"token"` + Color string `yaml:"color"` + Channel string `yaml:"channel"` +} + +type telegram struct { + Token string `yaml:"token"` + ChatID string `yaml:"chat_id"` +} diff --git a/pkg/parsers/threat.go b/pkg/parsers/threat.go new file mode 100644 index 00000000..5a9273cc --- /dev/null +++ b/pkg/parsers/threat.go @@ -0,0 +1,17 @@ +package parsers + +type customs struct { + Name string `yaml:"name"` + Condition string `yaml:"condition"` + Rules []struct { + Element string `yaml:"element"` + Pattern string `yaml:"pattern"` + Selector bool `yaml:"selector"` + } `yaml:"rules"` +} + +type options struct { + Excludes []string `yaml:"excludes"` + Whitelists []string `yaml:"whitelists"` + Customs []customs `yaml:"customs"` +} diff --git a/pkg/parsers/yaml.go b/pkg/parsers/yaml.go deleted file mode 100644 index ac5437f4..00000000 --- a/pkg/parsers/yaml.go +++ /dev/null @@ -1,10 +0,0 @@ -package parsers - -import ( - "gopkg.in/yaml.v2" -) - -func GetYaml(f []byte, s interface{}) error { - y := yaml.Unmarshal(f, s) - return y -} diff --git a/pkg/teler/teler.go b/pkg/teler/teler.go index 779d22b0..1556ae7a 100644 --- a/pkg/teler/teler.go +++ b/pkg/teler/teler.go @@ -7,19 +7,21 @@ import ( "reflect" "regexp" "strings" - "unicode/utf8" "github.com/satyrius/gonx" "github.com/valyala/fastjson" "ktbs.dev/teler/common" "ktbs.dev/teler/pkg/matchers" - "ktbs.dev/teler/pkg/metrics" ) // Analyze logs from threat resources func Analyze(options *common.Options, logs *gonx.Entry) (bool, map[string]string) { - var match bool + var ( + match bool + selector string + ) + cfg := options.Configs log := make(map[string]string) fields := reflect.ValueOf(logs).Elem().FieldByName("fields") @@ -223,27 +225,58 @@ func Analyze(options *common.Options, logs *gonx.Entry) (bool, map[string]string } if match { - metrics.Send(log) return match, log } } - return match, log -} + log["element"] = "" + customs := cfg.Rules.Threat.Customs -func trimFirst(s string) string { - _, i := utf8.DecodeRuneInString(s) - return s[i:] -} + for i := 0; i < len(customs); i++ { + log["category"] = customs[i].Name + + cond := strings.ToLower(customs[i].Condition) + if cond == "" { + cond = "or" + } + + rules := customs[i].Rules + rulesCount := len(customs[i].Rules) + matchCount := 0 + + if rulesCount < 1 { + continue + } + + for j := 0; j < rulesCount; j++ { + if matchers.IsMatch(rules[j].Pattern, log[rules[j].Element]) { + if rules[j].Selector { + log["element"] = rules[j].Element + } + selector = rules[j].Element + + matchCount++ + if cond == "or" { + break + } + } + } + + if log["element"] == "" { + log["element"] = selector + } + + switch { + case cond == "and" && matchCount == rulesCount: + match = true + case cond == "or" && matchCount > 0: + match = true + } -func isWhitelist(options *common.Options, find string) bool { - whitelist := options.Configs.Rules.Threat.Whitelists - for i := 0; i < len(whitelist); i++ { - match := matchers.IsMatch(whitelist[i], find) if match { - return true + break } } - return false + return match, log } diff --git a/pkg/teler/utils.go b/pkg/teler/utils.go index a35b61ce..9f3c675d 100644 --- a/pkg/teler/utils.go +++ b/pkg/teler/utils.go @@ -2,7 +2,10 @@ package teler import ( "reflect" + "unicode/utf8" + "ktbs.dev/teler/common" + "ktbs.dev/teler/pkg/matchers" "ktbs.dev/teler/resource" ) @@ -18,7 +21,24 @@ func getDatasets() { continue } - datasets[cat] = map[string]string{} + datasets[cat] = make(map[string]string) datasets[cat]["content"] = con } } + +func trimFirst(s string) string { + _, i := utf8.DecodeRuneInString(s) + return s[i:] +} + +func isWhitelist(options *common.Options, find string) bool { + whitelist := options.Configs.Rules.Threat.Whitelists + for i := 0; i < len(whitelist); i++ { + match := matchers.IsMatch(whitelist[i], find) + if match { + return true + } + } + + return false +} diff --git a/teler.example.yaml b/teler.example.yaml index bd998b9f..3b7f157d 100644 --- a/teler.example.yaml +++ b/teler.example.yaml @@ -1,9 +1,7 @@ -# To write log format, see https://github.com/kitabisa/teler#configuration +# To write log format, see https://www.notion.so/kitabisa/Configuration-d7c8fab40366406591875bac631bef3f log_format: | - $remote_addr - [$remote_addr] $remote_user - [$time_local] - "$request_method $request_uri $request_protocol" $status $body_bytes_sent - "$http_referer" "$http_user_agent" $request_length $request_time - [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status $req_id + $remote_addr $remote_user - [$time_local] "$request_method $request_uri $request_protocol" + $status $body_bytes_sent "$http_referer" "$http_user_agent" rules: cache: true @@ -16,18 +14,60 @@ rules: # - "Bad Crawler" # - "Directory Bruteforce" - # It can be user-agent, request path, HTTP referrer, IP address and/or request query values parsed in regExp + # It can be user-agent, request path, HTTP referrer, + # IP address and/or request query values parsed in regExp. + # This list applies only to engine defined threats, not to custom threat rules. whitelists: - # - "(curl|Go-http-client|okhttp)/*" - # - "^/wp-login\\.php" - # - "https://www\\.facebook\\.com" - # - "192\\.168\\.0\\.1" + # - (curl|Go-http-client|okhttp)/* + # - ^/wp-login\.php + # - https?:\/\/www\.facebook\.com + # - 192\.168\.0\.1 -prometheus: - active: false - host: "localhost" - port: 9099 - endpoint: "/metrics" + customs: + # - name: "Log4j Attack" + # condition: or + # rules: + # - element: "request_uri" + # pattern: \$\{.*:\/\/.*\/?\w+?\} + + # - element: "http_referer" + # pattern: \$\{.*:\/\/.*\/?\w+?\} + + # - element: "http_user_agent" + # pattern: \$\{.*:\/\/.*\/?\w+?\} + + # - name: "Large File Upload" + # condition: and + # rules: + # - element: "body_bytes_sent" + # selector: true + # pattern: \d{6,} + + # - element: "request_method" + # pattern: P(OST|UT) + + +metrics: + prometheus: + active: false + host: "localhost" + port: 9099 + endpoint: "/metrics" + +logs: + file: + active: false + json: false + path: "/path/to/output.log" + + zinc: + active: false + host: "localhost" + port: 4080 + ssl: false + username: "admin" + password: "Complexpass#123" + index: "lorem-ipsum-index" alert: active: false