-
-
Notifications
You must be signed in to change notification settings - Fork 5.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Allow code search by filename (#32210)
This is a large and complex PR, so let me explain in detail its changes. First, I had to create new index mappings for Bleve and ElasticSerach as the current ones do not support search by filename. This requires Gitea to recreate the code search indexes (I do not know if this is a breaking change, but I feel it deserves a heads-up). I've used [this approach](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/analysis-pathhierarchy-tokenizer.html) to model the filename index. It allows us to efficiently search for both the full path and the name of a file. Bleve, however, does not support this out-of-box, so I had to code a brand new [token filter](https://blevesearch.com/docs/Token-Filters/) to generate the search terms. I also did an overhaul in the `indexer_test.go` file. It now asserts the order of the expected results (this is important since matches based on the name of a file are more relevant than those based on its content). I've added new test scenarios that deal with searching by filename. They use a new repo included in the Gitea fixture. The screenshot below depicts how Gitea shows the search results. It shows results based on content in the same way as the current version does. In matches based on the filename, the first seven lines of the file contents are shown (BTW, this is how GitHub does it).  Resolves #32096 --------- Signed-off-by: Bruno Sofiato <[email protected]>
- Loading branch information
Showing
38 changed files
with
720 additions
and
49 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1517,3 +1517,40 @@ | |
| repo_admin_change_team_access: false | ||
| theme: "" | ||
| keep_activity_private: false | ||
|
|
||
| - | ||
| id: 42 | ||
| lower_name: org42 | ||
| name: org42 | ||
| full_name: Org42 | ||
| email: [email protected] | ||
| keep_email_private: false | ||
| email_notifications_preference: onmention | ||
| passwd: ZogKvWdyEx:password | ||
| passwd_hash_algo: dummy | ||
| must_change_password: false | ||
| login_source: 0 | ||
| login_name: org42 | ||
| type: 1 | ||
| salt: ZogKvWdyEx | ||
| max_repo_creation: -1 | ||
| is_active: false | ||
| is_admin: false | ||
| is_restricted: false | ||
| allow_git_hook: false | ||
| allow_import_local: false | ||
| allow_create_organization: true | ||
| prohibit_login: false | ||
| avatar: avatar42 | ||
| avatar_email: [email protected] | ||
| use_custom_avatar: false | ||
| num_followers: 0 | ||
| num_following: 0 | ||
| num_stars: 0 | ||
| num_repos: 1 | ||
| num_teams: 0 | ||
| num_members: 0 | ||
| visibility: 0 | ||
| repo_admin_change_team_access: false | ||
| theme: "" | ||
| keep_activity_private: false | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| // Copyright 2024 The Gitea Authors. All rights reserved. | ||
| // SPDX-License-Identifier: MIT | ||
|
|
||
| package path | ||
|
|
||
| import ( | ||
| "slices" | ||
| "strings" | ||
|
|
||
| "github.com/blevesearch/bleve/v2/analysis" | ||
| "github.com/blevesearch/bleve/v2/registry" | ||
| ) | ||
|
|
||
| const ( | ||
| Name = "gitea/path" | ||
| ) | ||
|
|
||
| type TokenFilter struct{} | ||
|
|
||
| func NewTokenFilter() *TokenFilter { | ||
| return &TokenFilter{} | ||
| } | ||
|
|
||
| func TokenFilterConstructor(config map[string]any, cache *registry.Cache) (analysis.TokenFilter, error) { | ||
| return NewTokenFilter(), nil | ||
| } | ||
|
|
||
| func (s *TokenFilter) Filter(input analysis.TokenStream) analysis.TokenStream { | ||
| if len(input) == 1 { | ||
| // if there is only one token, we dont need to generate the reversed chain | ||
| return generatePathTokens(input, false) | ||
| } | ||
|
|
||
| normal := generatePathTokens(input, false) | ||
| reversed := generatePathTokens(input, true) | ||
|
|
||
| return append(normal, reversed...) | ||
| } | ||
|
|
||
| // Generates path tokens from the input tokens. | ||
| // This mimics the behavior of the path hierarchy tokenizer in ES. It takes the input tokens and combine them, generating a term for each component | ||
| // in tree (e.g., foo/bar/baz.md will generate foo, foo/bar, and foo/bar/baz.md). | ||
| // | ||
| // If the reverse flag is set, the order of the tokens is reversed (the same input will generate baz.md, baz.md/bar, baz.md/bar/foo). This is useful | ||
| // to efficiently search for filenames without supplying the fullpath. | ||
| func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.TokenStream { | ||
| terms := make([]string, 0, len(input)) | ||
| longestTerm := 0 | ||
|
|
||
| if reversed { | ||
| slices.Reverse(input) | ||
| } | ||
|
|
||
| for i := 0; i < len(input); i++ { | ||
| var sb strings.Builder | ||
| sb.WriteString(string(input[0].Term)) | ||
|
|
||
| for j := 1; j < i; j++ { | ||
| sb.WriteString("/") | ||
| sb.WriteString(string(input[j].Term)) | ||
| } | ||
|
|
||
| term := sb.String() | ||
|
|
||
| if longestTerm < len(term) { | ||
| longestTerm = len(term) | ||
| } | ||
|
|
||
| terms = append(terms, term) | ||
| } | ||
|
|
||
| output := make(analysis.TokenStream, 0, len(terms)) | ||
|
|
||
| for _, term := range terms { | ||
| var start, end int | ||
|
|
||
| if reversed { | ||
| start = 0 | ||
| end = len(term) | ||
| } else { | ||
| start = longestTerm - len(term) | ||
| end = longestTerm | ||
| } | ||
|
|
||
| token := analysis.Token{ | ||
| Position: 1, | ||
| Start: start, | ||
| End: end, | ||
| Type: analysis.AlphaNumeric, | ||
| Term: []byte(term), | ||
| } | ||
|
|
||
| output = append(output, &token) | ||
| } | ||
|
|
||
| return output | ||
| } | ||
|
|
||
| func init() { | ||
| registry.RegisterTokenFilter(Name, TokenFilterConstructor) | ||
| } |
Oops, something went wrong.