From c92db5d15b8646b7fba32dfb606f3d96be0a73d2 Mon Sep 17 00:00:00 2001 From: SchwarzSail <119314702+SchwarzSail@users.noreply.github.com> Date: Fri, 18 Oct 2024 19:42:15 +0800 Subject: [PATCH 1/3] abstract the init of rpc client --- cmd/api/main.go | 2 +- cmd/classroom/main.go | 2 +- pkg/client/rpc.go | 35 +++++++++++++++-------------------- pkg/logger/output.go | 4 +++- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index e7da4fb..ede420d 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -42,7 +42,7 @@ func Init() { config.Init(*path, serviceName) // log - utils.InitLoggerWithHook(serviceName) + // utils.InitLoggerWithHook(serviceName) // rpc rpc.Init() diff --git a/cmd/classroom/main.go b/cmd/classroom/main.go index 3690d0e..1246c94 100644 --- a/cmd/classroom/main.go +++ b/cmd/classroom/main.go @@ -45,7 +45,7 @@ func Init() { config.Init(*path, serviceName) // log - utils.InitLoggerWithHook(serviceName) + // utils.InitLoggerWithHook(serviceName) dal.Init() InitWorkerQueue() diff --git a/pkg/client/rpc.go b/pkg/client/rpc.go index 9e8a72f..d042d1c 100644 --- a/pkg/client/rpc.go +++ b/pkg/client/rpc.go @@ -20,43 +20,38 @@ import ( "errors" "fmt" + "github.com/west2-online/fzuhelper-server/kitex_gen/classroom/classroomservice" + "github.com/west2-online/fzuhelper-server/kitex_gen/user/userservice" + "github.com/cloudwego/kitex/client" etcd "github.com/kitex-contrib/registry-etcd" "github.com/west2-online/fzuhelper-server/config" - "github.com/west2-online/fzuhelper-server/kitex_gen/classroom/classroomservice" - "github.com/west2-online/fzuhelper-server/kitex_gen/user/userservice" "github.com/west2-online/fzuhelper-server/pkg/constants" ) -// 定义一系列的RPC客户端 - -func InitUserRPC() (*userservice.Client, error) { +// 通用的RPC客户端初始化函数 +func initRPCClient[T any](serviceName string, newClientFunc func(string, ...client.Option) (T, error)) (*T, error) { if config.Etcd == nil || config.Etcd.Addr == "" { return nil, errors.New("config.Etcd.Addr is nil") } + // 初始化Etcd Resolver r, err := etcd.NewEtcdResolver([]string{config.Etcd.Addr}) if err != nil { - return nil, fmt.Errorf("InitUserRPC etcd.NewEtcdResolver failed: %w", err) + return nil, fmt.Errorf("initRPCClient etcd.NewEtcdResolver failed: %w", err) } - client, err := userservice.NewClient(constants.UserServiceName, client.WithResolver(r), client.WithMuxConnection(constants.MuxConnection)) + // 初始化具体的RPC客户端 + client, err := newClientFunc(serviceName, client.WithResolver(r), client.WithMuxConnection(constants.MuxConnection)) if err != nil { - return nil, fmt.Errorf("InitUserRPC NewClient failed: %w", err) + return nil, fmt.Errorf("initRPCClient NewClient failed: %w", err) } return &client, nil } +func InitUserRPC() (*userservice.Client, error) { + return initRPCClient(constants.UserServiceName, userservice.NewClient) +} + func InitClassroomRPC() (*classroomservice.Client, error) { - if config.Etcd == nil || config.Etcd.Addr == "" { - return nil, errors.New("config.Etcd.Addr is nil") - } - r, err := etcd.NewEtcdResolver([]string{config.Etcd.Addr}) - if err != nil { - return nil, fmt.Errorf("InitClassroomRPC etcd.NewEtcdResolver failed: %w", err) - } - client, err := classroomservice.NewClient(constants.ClassroomServiceName, client.WithResolver(r), client.WithMuxConnection(constants.MuxConnection)) - if err != nil { - return nil, fmt.Errorf("InitClassroomRPC NewClient failed: %w", err) - } - return &client, nil + return initRPCClient(constants.ClassroomServiceName, classroomservice.NewClient) } diff --git a/pkg/logger/output.go b/pkg/logger/output.go index fd55023..9fdeab1 100644 --- a/pkg/logger/output.go +++ b/pkg/logger/output.go @@ -16,7 +16,9 @@ limitations under the License. package logger -import "github.com/cloudwego/kitex/pkg/klog" +import ( + "github.com/cloudwego/kitex/pkg/klog" +) func init() { klog.SetLogger(DefaultLogger()) From b1d20121c3ee919470d2803da3faa4b5be5d7a30 Mon Sep 17 00:00:00 2001 From: SchwarzSail <119314702+SchwarzSail@users.noreply.github.com> Date: Sun, 20 Oct 2024 00:12:34 +0800 Subject: [PATCH 2/3] feat:increase the frequency of updating classroom info in one day --- cmd/classroom/dal/cache/schedule.go | 132 ++++++++++++++-------------- cmd/classroom/main.go | 6 +- cmd/classroom/workqueue.go | 45 ++++++---- pkg/constants/constants.go | 1 + 4 files changed, 102 insertions(+), 82 deletions(-) diff --git a/cmd/classroom/dal/cache/schedule.go b/cmd/classroom/dal/cache/schedule.go index 80119d1..3fbdb74 100644 --- a/cmd/classroom/dal/cache/schedule.go +++ b/cmd/classroom/dal/cache/schedule.go @@ -32,83 +32,85 @@ import ( "github.com/west2-online/jwch" ) -// ScheduledGetClassrooms 定期获取空教室信息 -func ScheduledGetClassrooms() error { - ctx := context.Background() - var dates []string - currentTime := time.Now() +// ScheduledUpdateClassroomsInfo 定期获取7天的空教室信息 +func ScheduledUpdateClassroomsInfo(date time.Time) error { + var dates []time.Time // 设定一周时间 - for i := 0; i < 7; i++ { - date := currentTime.AddDate(0, 0, i).Format("2006-01-02") - dates = append(dates, date) + for i := 1; i < 7; i++ { + d := date.AddDate(0, 0, i) + dates = append(dates, d) } - logger.Infof("ScheduledGetClassrooms: start to get empty room info in the next 7 days: %v", dates) var eg errgroup.Group - // 对每个日期启动一个 goroutine - for _, date := range dates { + for _, d := range dates { // 将 date 作为参数传递给 goroutine - eg.Go(func(date string) func() error { - return func() error { - // 创建新的 stu 对象，每个 goroutine 都有自己的 client - // 定义 jwch 的 stu 客户端 - stu := jwch.NewStudent().WithUser(config.DefaultUser.Account, config.DefaultUser.Password) - // 登录，id 和 cookies 会自动保存在 client 中 - // 如果登录失败，重试 - err := utils.RetryLogin(stu) + currentDate := d + eg.Go(func() error { + return UpdateClassroomsInfo(currentDate) + }) + } + + // 等待所有 goroutine 完成，并收集错误 + if err := eg.Wait(); err != nil { + return fmt.Errorf("ScheduledUpdateClassroomsInfo: failed to refresh empty room info: %w", err) + } + + return nil +} + +// UpdateClassroomsInfo update the classroom info in the date +func UpdateClassroomsInfo(date time.Time) error { + currentDate := date.Format("2006-01-02") + logger.Infof("UpdateClassroomsInfo: start to update empty room info in the date %v", currentDate) + ctx := context.Background() + // 定义 jwch 的 stu 客户端 + stu := jwch.NewStudent().WithUser(config.DefaultUser.Account, config.DefaultUser.Password) + // 登录，id 和 cookies 会自动保存在 client 中 + // 如果登录失败，重试 + err := utils.RetryLogin(stu) + if err != nil { + return fmt.Errorf("UpdateClassroomsInfo: failed to login: %w", err) + } + for _, campus := range constants.CampusArray { + for startTime := 1; startTime <= 11; startTime++ { + for endTime := startTime; endTime <= 11; endTime++ { + args := jwch.EmptyRoomReq{ + Campus: campus, + Time: currentDate, // 使用传递进来的 date 参数 + Start: strconv.Itoa(startTime), + End: strconv.Itoa(endTime), + } + var res []string + var err error + // 从 jwch 获取空教室信息 + switch campus { + case "旗山校区": + res, err = stu.GetQiShanEmptyRoom(args) + default: + res, err = stu.GetEmptyRoom(args) + } if err != nil { - return fmt.Errorf("ScheduledGetClassrooms: failed to login: %w", err) + return fmt.Errorf("UpdateClassroomsInfo: failed to get empty room info: %w", err) } - for _, campus := range constants.CampusArray { - for startTime := 1; startTime <= 11; startTime++ { - for endTime := startTime; endTime <= 11; endTime++ { - args := jwch.EmptyRoomReq{ - Campus: campus, - Time: date, // 使用传递进来的 date 参数 - Start: strconv.Itoa(startTime), - End: strconv.Itoa(endTime), - } - var res []string - var err error - // 从 jwch 获取空教室信息 - switch campus { - case "旗山校区": - res, err = stu.GetQiShanEmptyRoom(args) - default: - res, err = stu.GetEmptyRoom(args) - } - if err != nil { - return fmt.Errorf("ScheduledGetClassrooms: failed to get empty room info: %w", err) - } - // 收集结果并缓存 - switch campus { - case "厦门工艺美院": - err = SetXiaMenEmptyRoomCache(ctx, date, args.Start, args.End, res) - if err != nil { - return fmt.Errorf("ScheduledGetClassrooms: failed to set xiamen empty room cache: %w", err) - } - default: - key := fmt.Sprintf("%s.%s.%s.%s", args.Time, args.Campus, args.Start, args.End) - err = SetEmptyRoomCache(ctx, key, res) - if err != nil { - return fmt.Errorf("ScheduledGetClassrooms: failed to set empty room cache: %w", err) - } - } - } + // 收集结果并缓存 + switch campus { + case "厦门工艺美院": + err = SetXiaMenEmptyRoomCache(ctx, currentDate, args.Start, args.End, res) + if err != nil { + return fmt.Errorf("UpdateClassroomsInfo: failed to set xiamen empty room cache: %w", err) + } + default: + key := fmt.Sprintf("%s.%s.%s.%s", args.Time, args.Campus, args.Start, args.End) + err = SetEmptyRoomCache(ctx, key, res) + if err != nil { + return fmt.Errorf("UpdateClassroomsInfo: failed to set empty room cache: %w", err) } - - logger.Infof("ScheduledGetClassrooms: complete all tasks of campus %s in the day %s", campus, date) } - return nil } - }(date)) - } + } - // 等待所有 goroutine 完成，并收集错误 - if err := eg.Wait(); err != nil { - return fmt.Errorf("ScheduledGetClassrooms: failed to refresh empty room info: %w", err) + logger.Infof("UpdateClassroomsInfo: complete all tasks of campus %s in the day %s", campus, date) } - return nil } diff --git a/cmd/classroom/main.go b/cmd/classroom/main.go index 1246c94..659acd0 100644 --- a/cmd/classroom/main.go +++ b/cmd/classroom/main.go @@ -84,8 +84,10 @@ func main() { }), ) // 提前缓存空教室数据 - // 将signal放入队列，开启定时任务 - WorkQueue.Add("signal") + // update用于启动定期更新当天数据的任务 + WorkQueue.Add("update") + // 将scheduled放入队列，开启定时任务 + WorkQueue.Add("schedule") if err = svr.Run(); err != nil { logger.Fatalf("Classroom: server run failed: %v", err) diff --git a/cmd/classroom/workqueue.go b/cmd/classroom/workqueue.go index 6dffe06..bd92450 100644 --- a/cmd/classroom/workqueue.go +++ b/cmd/classroom/workqueue.go @@ -17,12 +17,12 @@ limitations under the License. package main import ( - "golang.org/x/time/rate" - "k8s.io/client-go/util/workqueue" - "github.com/west2-online/fzuhelper-server/cmd/classroom/dal/cache" "github.com/west2-online/fzuhelper-server/pkg/constants" "github.com/west2-online/fzuhelper-server/pkg/logger" + "golang.org/x/time/rate" + "k8s.io/client-go/util/workqueue" + "time" ) var WorkQueue workqueue.TypedRateLimitingInterface[string] @@ -50,18 +50,33 @@ func worker() { logger.Info("Classroom.worker shutdown") return } - if err := cache.ScheduledGetClassrooms(); err != nil { - logger.Errorf("Classroom.worker ScheduledGetClassrooms failed: %v", err) - // 如果失败，在使用该函数，采取避退策略 - WorkQueue.AddRateLimited(item) - } else { - // 将signal重新放入队列，实现自驱动 - WorkQueue.AddAfter(item, constants.ScheduledTime) - logger.Info("Classroom.worker ScheduledGetClassrooms success") - // 任务成功，清除其失败记录 - WorkQueue.Forget(item) + var err error + // 根据update和schedule区分爬取 + switch item { + case "update": + if err = apply(constants.UpdatedTime, item, cache.UpdateClassroomsInfo); err != nil { + logger.Infof("Classroom.worker: update failed: %v", err) + } + case "schedule": + if err = apply(constants.ScheduledTime, item, cache.ScheduledUpdateClassroomsInfo); err != nil { + logger.Infof("Classroom.worker: schedule failed: %v", err) + } } - // 任务完成, 释放资源, 防止队列阻塞 - WorkQueue.Done(item) } } + +// 抽象函数 +func apply(timeDuration time.Duration, item string, scheduledFunc func(date time.Time) error) (err error) { + if err = scheduledFunc(time.Now()); err != nil { + // 如果失败，在使用该函数，采取避退策略 + WorkQueue.AddRateLimited(item) + } else { + // 将signal重新放入队列，实现自驱动 + WorkQueue.AddAfter(item, timeDuration) + // 任务成功，清除其失败记录 + WorkQueue.Forget(item) + } + // 任务完成, 释放资源, 防止队列阻塞 + WorkQueue.Done(item) + return err +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index d1c676d..55be11c 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -62,6 +62,7 @@ const ( // 定时任务 ScheduledTime = 24 * time.Hour + UpdatedTime = 6 * time.Hour // 当天空教室更新间隔 // retry MaxRetries = 5 // 最大重试次数 From 2c71f9c932cd4be8d8c85a4d8faa72296e77e2a1 Mon Sep 17 00:00:00 2001 From: SchwarzSail <119314702+SchwarzSail@users.noreply.github.com> Date: Sun, 20 Oct 2024 00:35:54 +0800 Subject: [PATCH 3/3] fix: modify for ci_test --- cmd/classroom/workqueue.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/classroom/workqueue.go b/cmd/classroom/workqueue.go index bd92450..f0306c6 100644 --- a/cmd/classroom/workqueue.go +++ b/cmd/classroom/workqueue.go @@ -17,12 +17,14 @@ limitations under the License. package main import ( + "time" + + "golang.org/x/time/rate" + "k8s.io/client-go/util/workqueue" + "github.com/west2-online/fzuhelper-server/cmd/classroom/dal/cache" "github.com/west2-online/fzuhelper-server/pkg/constants" "github.com/west2-online/fzuhelper-server/pkg/logger" - "golang.org/x/time/rate" - "k8s.io/client-go/util/workqueue" - "time" ) var WorkQueue workqueue.TypedRateLimitingInterface[string]