Skip to content

Commit

Permalink
chore: tiny refactor on dataloader to simplify the merge process
Browse files Browse the repository at this point in the history
  • Loading branch information
@YuukanOO
YuukanOO committed May 21, 2024
1 parent 7bdae73 commit b4f6490
Show file tree
Hide file tree
Showing 5 changed files with 119 additions and 147 deletions.
208 changes: 107 additions & 101 deletions internal/deployment/infra/sqlite/gateway.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func (s *gateway) GetAllDeploymentsByApp(ctx context.Context, cmd get_app_deploy
WHERE deployments.app_id = ?`, cmd.AppID).
S(builder.MaybeValue(cmd.Environment, "AND deployments.config_environment = ?")).
F("ORDER BY deployments.deployment_number DESC").
Paginate(s.db, ctx, deploymentMapper, cmd.Page.Get(1), 5)
Paginate(s.db, ctx, deploymentMapper(nil), cmd.Page.Get(1), 5)
}

func (s *gateway) GetDeploymentByID(ctx context.Context, cmd get_deployment.Query) (get_deployment.Deployment, error) {
Expand Down Expand Up @@ -141,7 +141,7 @@ func (s *gateway) GetDeploymentByID(ctx context.Context, cmd get_deployment.Quer
INNER JOIN users ON users.id = deployments.requested_by
LEFT JOIN targets ON targets.id = deployments.config_target
WHERE deployments.app_id = ? AND deployments.deployment_number = ?`, cmd.AppID, cmd.DeploymentNumber).
One(s.db, ctx, deploymentDetailMapper)
One(s.db, ctx, deploymentDetailMapper(nil))
}

func (s *gateway) GetAllTargets(ctx context.Context, cmd get_targets.Query) ([]get_target.Target, error) {
Expand Down Expand Up @@ -202,8 +202,7 @@ var getDeploymentDataloader = builder.NewDataloader(
_, err := builder.
Query[get_app_deployments.Deployment](`
SELECT
deployments.app_id -- The first one will be used by the dataloader merge process
,deployments.app_id
deployments.app_id
,deployments.deployment_number
,deployments.config_environment
,deployments.config_target
Expand All @@ -225,7 +224,7 @@ var getDeploymentDataloader = builder.NewDataloader(
LEFT JOIN targets ON targets.id = deployments.config_target`).
S(builder.Array("WHERE deployments.app_id IN", kr.Keys())).
F("GROUP BY deployments.app_id, deployments.config_environment").
All(e, ctx, builder.Merge(kr, deploymentMapper, appMerger))
All(e, ctx, deploymentMapper(&kr))

return err
})
Expand All @@ -236,8 +235,7 @@ var getDeploymentDetailDataloader = builder.NewDataloader(
_, err := builder.
Query[get_deployment.Deployment](`
SELECT
deployments.app_id -- The first one will be used by the dataloader merge process
,deployments.app_id
deployments.app_id
,deployments.deployment_number
,deployments.config_environment
,deployments.config_target
Expand All @@ -261,31 +259,11 @@ var getDeploymentDetailDataloader = builder.NewDataloader(
LEFT JOIN targets ON targets.id = deployments.config_target`).
S(builder.Array("WHERE deployments.app_id IN", kr.Keys())).
F("GROUP BY deployments.app_id, deployments.config_environment").
All(e, ctx, builder.Merge(kr, deploymentDetailMapper, appDetailMerger))
All(e, ctx, deploymentDetailMapper(&kr))

return err
})

func appMerger(a get_apps.App, d get_app_deployments.Deployment) get_apps.App {
switch domain.Environment(d.Environment) {
case domain.Production:
a.LatestDeployments.Production.Set(d)
case domain.Staging:
a.LatestDeployments.Staging.Set(d)
}
return a
}

func appDetailMerger(a get_app_detail.App, d get_deployment.Deployment) get_app_detail.App {
switch domain.Environment(d.Environment) {
case domain.Production:
a.LatestDeployments.Production.Set(d)
case domain.Staging:
a.LatestDeployments.Staging.Set(d)
}
return a
}

// AppData scanner which include last deployments by environment.
func appDataMapper(s storage.Scanner) (a get_apps.App, err error) {
var (
Expand Down Expand Up @@ -367,90 +345,118 @@ func appDetailDataMapper(s storage.Scanner) (a get_app_detail.App, err error) {
return a, err
}

func deploymentMapper(scanner storage.Scanner) (d get_app_deployments.Deployment, err error) {
var (
maxRequestedAt string
sourceData string
targetStatus *uint8
)
func deploymentMapper(kr *builder.KeyedResult[get_apps.App]) storage.Mapper[get_app_deployments.Deployment] {
return func(scanner storage.Scanner) (d get_app_deployments.Deployment, err error) {
var (
maxRequestedAt string
sourceData string
targetStatus *uint8
)

err = scanner.Scan(
&d.AppID,
&d.DeploymentNumber,
&d.Environment,
&d.Target.ID,
&d.Target.Name,
&d.Target.Url,
&targetStatus,
&d.Source.Discriminator,
&sourceData,
&d.State.Status,
&d.State.ErrCode,
&d.State.StartedAt,
&d.State.FinishedAt,
&d.RequestedAt,
&d.RequestedBy.ID,
&d.RequestedBy.Email,
&maxRequestedAt,
)

if err != nil {
return d, err
}

err = scanner.Scan(
&d.AppID,
&d.DeploymentNumber,
&d.Environment,
&d.Target.ID,
&d.Target.Name,
&d.Target.Url,
&targetStatus,
&d.Source.Discriminator,
&sourceData,
&d.State.Status,
&d.State.ErrCode,
&d.State.StartedAt,
&d.State.FinishedAt,
&d.RequestedAt,
&d.RequestedBy.ID,
&d.RequestedBy.Email,
&maxRequestedAt,
)
if kr != nil {
kr.Update(d.AppID, func(a get_apps.App) get_apps.App {
switch domain.Environment(d.Environment) {
case domain.Production:
a.LatestDeployments.Production.Set(d)
case domain.Staging:
a.LatestDeployments.Staging.Set(d)
}
return a
})
}

if err != nil {
return d, err
}
// Can't scan directly into a monad.Maybe or it will fail with a conversion error between int64/uint8
if targetStatus != nil {
d.Target.Status.Set(*targetStatus)
}

d.Source.Data, err = get_deployment.SourceDataTypes.From(d.Source.Discriminator, sourceData)

// Can't scan directly into a monad.Maybe or it will fail with a conversion error between int64/uint8
if targetStatus != nil {
d.Target.Status.Set(*targetStatus)
return d, err
}
}

d.Source.Data, err = get_deployment.SourceDataTypes.From(d.Source.Discriminator, sourceData)
func deploymentDetailMapper(kr *builder.KeyedResult[get_app_detail.App]) storage.Mapper[get_deployment.Deployment] {

return func(scanner storage.Scanner) (d get_deployment.Deployment, err error) {
var (
maxRequestedAt string
sourceData string
targetStatus *uint8
)

err = scanner.Scan(
&d.AppID,
&d.DeploymentNumber,
&d.Environment,
&d.Target.ID,
&d.Target.Name,
&d.Target.Url,
&targetStatus,
&d.Target.Entrypoints,
&d.Source.Discriminator,
&sourceData,
&d.State.Status,
&d.State.ErrCode,
&d.State.Services,
&d.State.StartedAt,
&d.State.FinishedAt,
&d.RequestedAt,
&d.RequestedBy.ID,
&d.RequestedBy.Email,
&maxRequestedAt,
)

if err != nil {
return d, err
}

return d, err
}
if kr != nil {
kr.Update(d.AppID, func(a get_app_detail.App) get_app_detail.App {
switch domain.Environment(d.Environment) {
case domain.Production:
a.LatestDeployments.Production.Set(d)
case domain.Staging:
a.LatestDeployments.Staging.Set(d)
}
return a
})
}
// Can't scan directly into a monad.Maybe or it will fail with a conversion error between int64/uint8
if targetStatus != nil {
d.Target.Status.Set(*targetStatus)
}

func deploymentDetailMapper(scanner storage.Scanner) (d get_deployment.Deployment, err error) {
var (
maxRequestedAt string
sourceData string
targetStatus *uint8
)
d.Source.Data, err = get_deployment.SourceDataTypes.From(d.Source.Discriminator, sourceData)

err = scanner.Scan(
&d.AppID,
&d.DeploymentNumber,
&d.Environment,
&d.Target.ID,
&d.Target.Name,
&d.Target.Url,
&targetStatus,
&d.Target.Entrypoints,
&d.Source.Discriminator,
&sourceData,
&d.State.Status,
&d.State.ErrCode,
&d.State.Services,
&d.State.StartedAt,
&d.State.FinishedAt,
&d.RequestedAt,
&d.RequestedBy.ID,
&d.RequestedBy.Email,
&maxRequestedAt,
)
populateServicesUrls(&d)

if err != nil {
return d, err
}

// Can't scan directly into a monad.Maybe or it will fail with a conversion error between int64/uint8
if targetStatus != nil {
d.Target.Status.Set(*targetStatus)
}

d.Source.Data, err = get_deployment.SourceDataTypes.From(d.Source.Discriminator, sourceData)

populateServicesUrls(&d)

return d, err
}

// Since the target domain is dynamic, compute exposed service urls based on the presence
Expand Down
3 changes: 1 addition & 2 deletions pkg/storage/scanner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@ type (
Scan(...any) error
}

Mapper[T any] func(Scanner) (T, error) // Mapper function from a simple Scanner to an object of type T
Merger[TParent, TChildren any] func(TParent, TChildren) TParent // Merger function to handle relations by configuring how a child should be merged into its parent
Mapper[T any] func(Scanner) (T, error) // Mapper function from a simple Scanner to an object of type T
)

// Ease the scan of a json serialized field.
Expand Down
1 change: 0 additions & 1 deletion pkg/storage/sqlite/builder/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,6 @@ func (q *queryBuilder[T]) All(
results := make([]T, 0)

// Instantiates needed stuff for data loaders
// FIXME: maybe split this in a dedicated function to avoid the cost when no loaders are given
mappings := make([]KeysMapping, len(loaders))

for i := range mappings {
Expand Down
43 changes: 0 additions & 43 deletions pkg/storage/sqlite/builder/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,51 +2,8 @@ package builder

import "github.com/YuukanOO/seelf/pkg/storage"

// Map TChild entities and merge them in a TParent entity using the key/index mapping
// given by the KeyedResult.
//
// The mapper given to this function SHOULD NOT scan the first column of the result set
// because this function will do it and consider the first scanned value to be the key
// of the TParent entity. It means that you should always include the TParent key
// as the first column when retrieving related entities.
func Merge[TParent, TChildren any](
results KeyedResult[TParent],
mapper storage.Mapper[TChildren],
merger storage.Merger[TParent, TChildren],
) storage.Mapper[TChildren] {
return func(s storage.Scanner) (data TChildren, err error) {
var key string
data, err = mapper(keyScanner(s, &key))

idx, found := results.indexByKeys[key]

if found {
results.data[idx] = merger(results.data[idx], data)
}

return data, err
}
}

// Tiny mapper when all you have to do is retrieve a single value from a query.
func valueMapper[T any](scanner storage.Scanner) (value T, err error) {
err = scanner.Scan(&value)
return value, err
}

type keyScannerDecorator struct {
target *string
decorated storage.Scanner
}

// Wrap the given scanner to extract the key from the first column returned by the
// scanner. Used by the Merge function to retrieve the key of the parent entity and
// merge the result with the child entity.
func keyScanner(s storage.Scanner, target *string) storage.Scanner {
return &keyScannerDecorator{target, s}
}

func (s *keyScannerDecorator) Scan(dest ...any) error {
dest = append([]any{s.target}, dest...)
return s.decorated.Scan(dest...)
}
11 changes: 11 additions & 0 deletions pkg/storage/sqlite/builder/result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,14 @@ type (

// Keys returns the list of keys contained in this dataset.
func (r KeyedResult[T]) Keys() []string { return maps.Keys(r.indexByKeys) }

// Update the result with the given key by applying the given function if it exists.
func (r *KeyedResult[T]) Update(targetKey string, updateFn func(T) T) {
idx, found := r.indexByKeys[targetKey]

if !found {
return
}

r.data[idx] = updateFn(r.data[idx])
}

0 comments on commit b4f6490

Please sign in to comment.