From 0b67fd9ecef0e0ebc7bf966b37bb79abb2c4954f Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 14 Jun 2024 15:42:50 +0800 Subject: [PATCH] feat: added modules --- core/.editorconfig | 13 + core/.github/workflows/test.yml | 36 + core/.gitignore | 10 + core/LICENSE | 201 + core/README.md | 2 + core/apps/api.go | 126 + core/apps/api_v2.go | 122 + core/apps/docker.go | 199 + core/apps/interfaces.go | 39 + core/apps/server.go | 149 + core/apps/server_test.go | 29 + core/apps/server_v2.go | 126 + core/apps/utils.go | 92 + core/cmd/root.go | 33 + core/cmd/server.go | 25 + core/cmd/server_test.go | 17 + core/color/service.go | 82 + core/config/base.go | 21 + core/config/config.go | 88 + core/config/config_test.go | 11 + core/config/default_config.go | 40 + core/config/path.go | 23 + core/config/version.go | 12 + core/constants/action.go | 8 + core/constants/anchor.go | 8 + core/constants/auth.go | 7 + core/constants/cache.go | 8 + core/constants/channels.go | 9 + core/constants/common.go | 6 + core/constants/config_spider.go | 6 + core/constants/data_collection.go | 5 + core/constants/data_field.go | 12 + core/constants/database.go | 5 + core/constants/delegate.go | 1 + core/constants/ds.go | 31 + core/constants/encrypt.go | 5 + core/constants/errors.go | 30 + core/constants/event.go | 6 + core/constants/export.go | 6 + core/constants/file.go | 5 + core/constants/filer.go | 5 + core/constants/filter.go | 28 + core/constants/git.go | 16 + core/constants/grpc.go | 17 + core/constants/http.go | 11 + core/constants/log.go | 5 + core/constants/message.go | 9 + core/constants/node.go | 8 + core/constants/notification.go | 8 + core/constants/pagination.go | 4 + core/constants/register.go | 8 + core/constants/results.go | 10 + core/constants/rpc.go | 12 + core/constants/schedule.go | 10 + core/constants/scrapy.go | 5 + core/constants/signal.go | 5 + core/constants/sort.go | 5 + core/constants/system.go | 25 + core/constants/task.go | 39 + core/constants/user.go | 15 + core/constants/variable.go | 9 + core/container/container.go | 11 + core/controllers/base.go | 69 + core/controllers/base_v2.go | 226 + core/controllers/base_v2_test.go | 165 + core/controllers/binder.go | 14 + core/controllers/binder_json.go | 208 + core/controllers/data_collection.go | 103 + core/controllers/data_source.go | 148 + core/controllers/delegate_action.go | 17 + core/controllers/delegate_basic.go | 99 + core/controllers/delegate_list.go | 222 + core/controllers/delegate_list_action.go | 17 + core/controllers/demo.go | 73 + core/controllers/environment.go | 57 + core/controllers/export.go | 127 + core/controllers/export_v2.go | 87 + core/controllers/filer.go | 130 + core/controllers/filter.go | 100 + core/controllers/filter_v2.go | 69 + core/controllers/git.go | 3 + core/controllers/http.go | 16 + core/controllers/init.go | 42 + core/controllers/login.go | 64 + core/controllers/login_v2.go | 36 + core/controllers/node.go | 94 + core/controllers/notification.go | 158 + core/controllers/permission.go | 3 + core/controllers/project.go | 103 + core/controllers/result.go | 150 + core/controllers/result_v2.go | 119 + core/controllers/role.go | 3 + core/controllers/router_v2.go | 387 ++ core/controllers/router_v2_test.go | 91 + core/controllers/schedule.go | 221 + core/controllers/schedule_v2.go | 130 + core/controllers/setting.go | 84 + core/controllers/setting_v2.go | 56 + core/controllers/spider.go | 1333 +++++ core/controllers/spider_v2.go | 1309 +++++ core/controllers/spider_v2_test.go | 249 + core/controllers/stats.go | 87 + core/controllers/stats_v2.go | 41 + core/controllers/sync.go | 57 + core/controllers/system_info.go | 28 + core/controllers/system_info_v2.go | 15 + core/controllers/tag.go | 3 + core/controllers/task.go | 534 ++ core/controllers/task_v2.go | 465 ++ core/controllers/test/base.go | 104 + core/controllers/test/export_test.go | 59 + core/controllers/test/filter_test.go | 76 + core/controllers/test/main_test.go | 44 + core/controllers/test/project_test.go | 305 + core/controllers/test/spider_test.go | 183 + core/controllers/test/task_test.go | 102 + core/controllers/token.go | 84 + core/controllers/token_v2.go | 35 + core/controllers/user.go | 244 + core/controllers/user_v2.go | 126 + core/controllers/user_v2_test.go | 89 + core/controllers/utils_context.go | 32 + core/controllers/utils_filter.go | 123 + core/controllers/utils_http.go | 72 + core/controllers/utils_pagination.go | 30 + core/controllers/utils_sort.go | 58 + core/controllers/version.go | 23 + core/data/colors.go | 5212 +++++++++++++++++ core/docker-compose.yml | 14 + core/docs/.gitignore | 6 + core/docs/api/index.html | 10 + core/docs/api/openapi.yaml | 4225 +++++++++++++ core/docs/package.json | 16 + core/docs/scripts/publish.js | 81 + core/ds/cockroachdb.go | 70 + core/ds/default.go | 1 + core/ds/es.go | 220 + core/ds/kafka.go | 116 + core/ds/mongo.go | 118 + core/ds/mssql.go | 71 + core/ds/mssql_test.go | 8 + core/ds/mysql.go | 69 + core/ds/options.go | 14 + core/ds/postgresql.go | 71 + core/ds/service.go | 305 + core/ds/sql.go | 76 + core/ds/sql_options.go | 6 + core/ds/sqlite.go | 52 + core/entity/address.go | 56 + core/entity/color.go | 18 + core/entity/common.go | 17 + core/entity/config_spider.go | 40 + core/entity/data_field.go | 6 + core/entity/doc.go | 8 + core/entity/es.go | 54 + core/entity/event.go | 14 + core/entity/export.go | 51 + core/entity/filter.go | 68 + core/entity/filter_select_option.go | 6 + core/entity/fs_file_info.go | 60 + core/entity/git.go | 12 + core/entity/grpc_base_service_message.go | 29 + core/entity/grpc_base_service_params.go | 23 + core/entity/grpc_delegate_message.go | 34 + core/entity/grpc_event_service_message.go | 8 + core/entity/grpc_subscribe.go | 26 + core/entity/http.go | 42 + core/entity/model_delegate.go | 11 + core/entity/model_info.go | 8 + core/entity/node.go | 17 + core/entity/pagination.go | 18 + core/entity/result.go | 100 + core/entity/rpc.go | 11 + core/entity/sort.go | 6 + core/entity/spider.go | 17 + core/entity/stats.go | 12 + core/entity/system_info.go | 6 + core/entity/task.go | 30 + core/entity/translation.go | 11 + core/entity/ttl_map.go | 58 + core/entity/version.go | 23 + core/errors/base.go | 33 + core/errors/controller.go | 19 + core/errors/ds.go | 15 + core/errors/event.go | 10 + core/errors/filter.go | 8 + core/errors/fs.go | 11 + core/errors/git.go | 10 + core/errors/grpc.go | 20 + core/errors/http.go | 9 + core/errors/model.go | 19 + core/errors/node.go | 14 + core/errors/process.go | 10 + core/errors/result.go | 5 + core/errors/schedule.go | 7 + core/errors/spider.go | 10 + core/errors/stats.go | 7 + core/errors/store.go | 9 + core/errors/task.go | 19 + core/errors/user.go | 17 + core/event/func.go | 6 + core/event/options.go | 1 + core/event/service.go | 92 + core/export/csv_service.go | 344 ++ core/export/csv_service_test.go | 139 + core/export/json_service.go | 223 + core/fs/default.go | 24 + core/fs/service_v2.go | 167 + core/fs/service_v2_test.go | 238 + core/go.mod | 184 + core/go.sum | 1420 +++++ core/grpc/client/client.go | 420 ++ core/grpc/client/client_v2.go | 287 + core/grpc/client/options.go | 51 + core/grpc/client/pool.go | 88 + core/grpc/client/utils_proto.go | 5 + core/grpc/middlewares/auth_token.go | 67 + core/grpc/payload/model_service_v2_payload.go | 17 + core/grpc/server/dependencies_server_v2.go | 32 + core/grpc/server/message_server.go | 93 + core/grpc/server/model_base_service_binder.go | 59 + core/grpc/server/model_base_service_server.go | 139 + .../server/model_base_service_v2_server.go | 306 + core/grpc/server/model_delegate_binder.go | 120 + core/grpc/server/model_delegate_server.go | 67 + core/grpc/server/node_server.go | 202 + core/grpc/server/node_server_v2.go | 189 + core/grpc/server/options.go | 43 + core/grpc/server/server.go | 265 + core/grpc/server/server_v2.go | 256 + core/grpc/server/task_server.go | 238 + core/grpc/server/task_server_v2.go | 237 + core/grpc/server/utils_handle.go | 53 + core/grpc/test/auth_token_test.go | 96 + core/grpc/test/base.go | 77 + core/grpc/test/main_test.go | 7 + .../test/model_base_service_server_test.go | 312 + core/grpc/test/model_delegate_server_test.go | 147 + core/grpc/test/node_server_test.go | 140 + core/i18n/service.go | 43 + core/interfaces/address.go | 7 + core/interfaces/color.go | 7 + core/interfaces/color_service.go | 7 + core/interfaces/controller_params.go | 6 + core/interfaces/data_source_service.go | 14 + core/interfaces/entity.go | 5 + core/interfaces/event_data.go | 6 + core/interfaces/event_service.go | 9 + core/interfaces/export.go | 14 + core/interfaces/export_service.go | 7 + core/interfaces/filter.go | 9 + core/interfaces/filter_condition.go | 10 + core/interfaces/fs_file_info.go | 19 + core/interfaces/fs_service.go | 28 + core/interfaces/fs_service_options.go | 21 + core/interfaces/fs_service_v2.go | 12 + core/interfaces/grpc_base.go | 9 + core/interfaces/grpc_base_service_params.go | 5 + core/interfaces/grpc_client.go | 30 + .../grpc_client_model_base_service.go | 7 + core/interfaces/grpc_client_model_delegate.go | 7 + .../grpc_client_model_environment_service.go | 14 + .../grpc_client_model_node_service.go | 15 + core/interfaces/grpc_client_model_service.go | 6 + .../grpc_client_model_spider_service.go | 14 + .../grpc_client_model_task_service.go | 14 + .../grpc_client_model_task_stat_service.go | 14 + core/interfaces/grpc_client_pool.go | 9 + .../grpc_model_base_service_message.go | 7 + core/interfaces/grpc_model_binder.go | 5 + .../interfaces/grpc_model_delegate_message.go | 8 + core/interfaces/grpc_model_list_binder.go | 5 + core/interfaces/grpc_server.go | 16 + core/interfaces/grpc_stream.go | 12 + core/interfaces/grpc_subscribe.go | 7 + core/interfaces/i18n_service.go | 6 + core/interfaces/injectable.go | 5 + core/interfaces/list.go | 5 + core/interfaces/model.go | 98 + core/interfaces/model_artifact.go | 12 + core/interfaces/model_artifact_sys.go | 21 + core/interfaces/model_base_service.go | 28 + core/interfaces/model_binder.go | 6 + core/interfaces/model_delegate.go | 22 + core/interfaces/model_environment.go | 9 + core/interfaces/model_extra_value.go | 17 + core/interfaces/model_git.go | 18 + core/interfaces/model_list_binder.go | 6 + core/interfaces/model_node.go | 22 + core/interfaces/model_node_delegate.go | 10 + core/interfaces/model_permission.go | 14 + core/interfaces/model_result.go | 11 + core/interfaces/model_role.go | 6 + core/interfaces/model_schedule.go | 28 + core/interfaces/model_service_v2.go | 25 + core/interfaces/model_spider.go | 24 + core/interfaces/model_tag.go | 8 + core/interfaces/model_task.go | 23 + core/interfaces/model_task_stat.go | 23 + core/interfaces/model_user.go | 9 + core/interfaces/model_user_group.go | 6 + core/interfaces/module.go | 10 + core/interfaces/node_config_service.go | 13 + core/interfaces/node_master_service.go | 14 + core/interfaces/node_service.go | 8 + core/interfaces/node_service_option.go | 4 + core/interfaces/node_worker_service.go | 11 + core/interfaces/options.go | 1 + core/interfaces/process_daemon.go | 17 + core/interfaces/provide.go | 3 + core/interfaces/result_service.go | 15 + core/interfaces/result_service_mongo.go | 15 + core/interfaces/result_service_registry.go | 11 + core/interfaces/schedule_service.go | 23 + core/interfaces/spider_admin_service.go | 22 + core/interfaces/spider_service_options.go | 17 + core/interfaces/stats_service.go | 9 + core/interfaces/task_base_service.go | 11 + core/interfaces/task_handler_service.go | 60 + core/interfaces/task_hook_service.go | 6 + core/interfaces/task_runner.go | 15 + core/interfaces/task_scheduler_service.go | 16 + core/interfaces/task_stats_service.go | 9 + core/interfaces/test.go | 8 + core/interfaces/translation.go | 5 + core/interfaces/user_service.go | 19 + core/interfaces/user_service_options.go | 13 + core/interfaces/with_address.go | 6 + core/interfaces/with_config_path.go | 6 + core/interfaces/with_model_id.go | 6 + core/main.go | 4 + core/middlewares/auth.go | 50 + core/middlewares/auth_v2.go | 50 + core/middlewares/cors.go | 19 + core/middlewares/filer_auth.go | 34 + core/middlewares/middlewares.go | 16 + core/models/client/binder_basic.go | 92 + core/models/client/binder_list.go | 100 + core/models/client/model_base_service.go | 251 + core/models/client/model_delegate.go | 329 ++ .../client/model_environment_service.go | 70 + core/models/client/model_node_delegate.go | 37 + core/models/client/model_node_service.go | 74 + core/models/client/model_service.go | 48 + core/models/client/model_service_v2.go | 360 ++ core/models/client/model_service_v2_test.go | 439 ++ core/models/client/model_spider_service.go | 70 + core/models/client/model_task_service.go | 70 + core/models/client/model_task_stat_service.go | 70 + core/models/client/options.go | 33 + core/models/common/index_service.go | 145 + core/models/config_spider/common.go | 24 + core/models/config_spider/scrapy.go | 263 + core/models/delegate/base_test.go | 25 + core/models/delegate/model.go | 362 ++ core/models/delegate/model_node.go | 37 + core/models/delegate/model_node_test.go | 65 + core/models/delegate/model_role_test.go | 65 + core/models/delegate/model_test.go | 65 + core/models/delegate/model_user_role_test.go | 97 + core/models/delegate/utils_event.go | 20 + core/models/models/artifact.go | 56 + core/models/models/artifact_sys.go | 63 + core/models/models/base.go | 13 + core/models/models/base_v2.go | 64 + core/models/models/data_collection.go | 36 + core/models/models/data_collection_v2.go | 17 + core/models/models/data_source.go | 42 + core/models/models/data_source_v2.go | 20 + core/models/models/dependency_setting.go | 36 + core/models/models/dependency_setting_v2.go | 17 + core/models/models/environment.go | 46 + core/models/models/environment_v2.go | 8 + core/models/models/extra_value.go | 64 + core/models/models/git.go | 82 + core/models/models/git_v2.go | 12 + core/models/models/job.go | 29 + core/models/models/node.go | 119 + core/models/models/node_v2.go | 24 + core/models/models/notification_setting_v2.go | 32 + core/models/models/password.go | 29 + core/models/models/permission.go | 91 + core/models/models/permission_v2.go | 13 + core/models/models/project.go | 47 + core/models/models/project_v2.go | 9 + core/models/models/result.go | 60 + core/models/models/role.go | 55 + core/models/models/role_permission.go | 30 + core/models/models/role_permission_v2.go | 12 + core/models/models/role_v2.go | 9 + core/models/models/schedule.go | 113 + core/models/models/schedule_v2.go | 23 + core/models/models/setting.go | 31 + core/models/models/setting_v2.go | 12 + core/models/models/spider.go | 137 + core/models/models/spider_stat.go | 38 + core/models/models/spider_stat_v2.go | 20 + core/models/models/spider_v2.go | 30 + core/models/models/tag.go | 44 + core/models/models/task.go | 118 + core/models/models/task_queue_item.go | 30 + core/models/models/task_queue_item_v2.go | 12 + core/models/models/task_stat.go | 101 + core/models/models/task_stat_v2.go | 18 + core/models/models/task_v2.go | 30 + core/models/models/test.go | 7 + core/models/models/token.go | 30 + core/models/models/token_v2.go | 8 + core/models/models/user.go | 59 + core/models/models/user_role.go | 30 + core/models/models/user_role_v2.go | 12 + core/models/models/user_v2.go | 10 + core/models/models/utils_binder_legacy.go | 96 + core/models/models/utils_col.go | 10 + core/models/models/utils_model_map.go | 95 + core/models/models/utils_tag.go | 34 + core/models/models/variable.go | 31 + core/models/models/variable_v2.go | 9 + core/models/service/artifact_service.go | 40 + core/models/service/base_service.go | 420 ++ core/models/service/base_service_v2.go | 181 + core/models/service/base_service_v2_test.go | 259 + core/models/service/binder_basic.go | 92 + core/models/service/binder_list.go | 93 + .../models/service/data_collection_service.go | 45 + core/models/service/data_source_service.go | 40 + .../service/dependency_setting_service.go | 40 + core/models/service/environment_service.go | 40 + core/models/service/extra_value_service.go | 45 + core/models/service/git_service.go | 43 + core/models/service/interface.go | 105 + core/models/service/job_service.go | 40 + core/models/service/node_service.go | 45 + core/models/service/options.go | 25 + core/models/service/password_service.go | 40 + core/models/service/permission_service.go | 48 + core/models/service/project_service.go | 40 + .../models/service/role_permission_service.go | 51 + core/models/service/role_service.go | 53 + core/models/service/schedule_service.go | 40 + core/models/service/service.go | 62 + core/models/service/setting_service.go | 45 + core/models/service/spider_service.go | 40 + core/models/service/spider_stat_service.go | 40 + core/models/service/tag_service.go | 118 + core/models/service/tag_service_legacy.go | 118 + core/models/service/task_queue_service.go | 40 + core/models/service/task_service.go | 40 + core/models/service/task_stat_service.go | 40 + core/models/service/token_service.go | 40 + core/models/service/user_role_service.go | 51 + core/models/service/user_service.go | 58 + core/models/service/variable_service.go | 45 + core/node/config/config.go | 68 + core/node/config/config_service.go | 130 + core/node/config/options.go | 13 + core/node/service/master_service.go | 368 ++ core/node/service/master_service_v2.go | 388 ++ core/node/service/options.go | 47 + core/node/service/worker_service.go | 238 + core/node/service/worker_service_v2.go | 225 + core/node/test/base.go | 206 + core/node/test/base_test.go | 67 + core/notification/base_test.go | 73 + core/notification/constants.go | 10 + core/notification/mail.go | 178 + core/notification/mail_theme.go | 8 + core/notification/mail_theme_flat.go | 287 + core/notification/mobile.go | 62 + core/notification/models.go | 32 + core/notification/payload.go | 8 + core/notification/service.go | 395 ++ core/notification/service_test.go | 19 + core/notification/service_v2.go | 362 ++ core/process/daemon.go | 170 + core/process/daemon_test.go | 21 + core/process/manage.go | 60 + core/process/options.go | 20 + core/result/options.go | 16 + core/result/service.go | 89 + core/result/service_mongo.go | 146 + core/result/service_registry.go | 48 + core/result/test/base.go | 76 + core/result/test/service_test.go | 67 + core/routes/group.go | 20 + core/routes/router.go | 178 + core/routes/router_test.go | 26 + core/schedule/logger.go | 35 + core/schedule/options.go | 37 + core/schedule/service.go | 287 + core/schedule/service_v2.go | 283 + core/schedule/test/base.go | 91 + core/schedule/test/schedule_service_test.go | 44 + core/spider/admin/options.go | 11 + core/spider/admin/service.go | 352 ++ core/spider/admin/service_v2.go | 328 ++ core/stats/options.go | 5 + core/stats/service.go | 323 + core/sys_exec/sys_exec.go | 89 + core/sys_exec/sys_exec_darwin.go | 24 + core/sys_exec/sys_exec_linux.go | 24 + core/sys_exec/sys_exec_windows.go | 10 + core/system/service.go | 86 + core/task/base.go | 97 + core/task/handler/options.go | 40 + core/task/handler/runner.go | 691 +++ core/task/handler/runner_test.go | 103 + core/task/handler/runner_v2.go | 671 +++ core/task/handler/service.go | 506 ++ core/task/handler/service_v2.go | 426 ++ core/task/log/constants.go | 12 + core/task/log/default.go | 5 + core/task/log/driver.go | 18 + core/task/log/entity.go | 16 + core/task/log/errors.go | 8 + core/task/log/file_driver.go | 284 + core/task/log/file_driver_test.go | 132 + core/task/log/interface.go | 10 + core/task/scheduler/options.go | 20 + core/task/scheduler/service.go | 264 + core/task/scheduler/service_v2.go | 264 + core/task/stats/options.go | 11 + core/task/stats/service.go | 155 + core/task/stats/service_v2.go | 143 + core/user/options.go | 11 + core/user/service.go | 238 + core/user/service_v2.go | 212 + core/user/test/base.go | 65 + core/user/test/user_service_test.go | 61 + core/utils/args.go | 17 + core/utils/array.go | 46 + core/utils/backoff.go | 15 + core/utils/binders/binder_col_name.go | 99 + core/utils/bool.go | 12 + core/utils/bson.go | 125 + core/utils/cache.go | 57 + core/utils/chan.go | 40 + core/utils/chan_test.go | 78 + core/utils/cockroachdb.go | 60 + core/utils/cron.go | 176 + core/utils/debug.go | 18 + core/utils/demo.go | 57 + core/utils/di.go | 18 + core/utils/docker.go | 5 + core/utils/encrypt.go | 80 + core/utils/encrypt_test.go | 20 + core/utils/es.go | 159 + core/utils/file.go | 382 ++ core/utils/file_test.go | 129 + core/utils/filter.go | 49 + core/utils/git.go | 36 + core/utils/helpers.go | 36 + core/utils/http.go | 32 + core/utils/init.go | 30 + core/utils/json.go | 12 + core/utils/kafka.go | 41 + core/utils/mongo.go | 94 + core/utils/mssql.go | 60 + core/utils/mysql.go | 60 + core/utils/node.go | 13 + core/utils/os.go | 13 + core/utils/postgresql.go | 60 + core/utils/result.go | 23 + core/utils/rpc.go | 14 + core/utils/spider.go | 8 + core/utils/sql.go | 27 + core/utils/sqlite.go | 45 + core/utils/stats.go | 1 + core/utils/system.go | 7 + core/utils/task.go | 13 + core/utils/time.go | 18 + core/utils/uuid.go | 8 + grpc/.gitignore | 2 + grpc/LICENSE | 29 + grpc/README.md | 2 + grpc/bin/compile.sh | 37 + grpc/dependencies_service_v2_request.pb.go | 488 ++ grpc/dependency_service_v2.pb.go | 108 + grpc/dependency_service_v2_grpc.pb.go | 312 + grpc/go.mod | 16 + grpc/go.sum | 16 + grpc/message_service.pb.go | 79 + grpc/message_service_grpc.pb.go | 137 + grpc/model_base_service.pb.go | 134 + grpc/model_base_service_grpc.pb.go | 501 ++ grpc/model_base_service_v2.pb.go | 179 + grpc/model_base_service_v2_grpc.pb.go | 573 ++ grpc/model_delegate.pb.go | 80 + grpc/model_delegate_grpc.pb.go | 105 + grpc/model_service_v2_request.pb.go | 1316 +++++ grpc/node.pb.go | 251 + grpc/node_info.pb.go | 156 + grpc/node_service.pb.go | 104 + grpc/node_service_grpc.pb.go | 277 + grpc/plugin_request.pb.go | 166 + grpc/plugin_service.pb.go | 96 + grpc/plugin_service_grpc.pb.go | 237 + .../dependencies_service_v2_request.proto | 32 + .../entity/model_service_v2_request.proto | 95 + grpc/proto/entity/node_info.proto | 9 + grpc/proto/entity/plugin_request.proto | 10 + grpc/proto/entity/request.proto | 9 + grpc/proto/entity/response.proto | 14 + grpc/proto/entity/response_code.proto | 8 + grpc/proto/entity/stream_message.proto | 16 + grpc/proto/entity/stream_message_code.proto | 33 + .../entity/stream_message_data_task.proto | 9 + grpc/proto/models/node.proto | 19 + grpc/proto/models/task.proto | 18 + .../services/dependency_service_v2.proto | 14 + grpc/proto/services/message_service.proto | 10 + grpc/proto/services/model_base_service.proto | 22 + .../services/model_base_service_v2.proto | 24 + grpc/proto/services/model_delegate.proto | 11 + grpc/proto/services/node_service.proto | 16 + grpc/proto/services/plugin_service.proto | 14 + grpc/proto/services/task_service.proto | 14 + grpc/request.pb.go | 156 + grpc/response.pb.go | 190 + grpc/response_code.pb.go | 132 + grpc/stream_message.pb.go | 210 + grpc/stream_message_code.pb.go | 190 + grpc/stream_message_data_task.pb.go | 158 + grpc/task.pb.go | 241 + grpc/task_service.pb.go | 95 + grpc/task_service_grpc.pb.go | 212 + 626 files changed, 60104 insertions(+) create mode 100644 core/.editorconfig create mode 100644 core/.github/workflows/test.yml create mode 100644 core/.gitignore create mode 100644 core/LICENSE create mode 100644 core/README.md create mode 100644 core/apps/api.go create mode 100644 core/apps/api_v2.go create mode 100644 core/apps/docker.go create mode 100644 core/apps/interfaces.go create mode 100644 core/apps/server.go create mode 100644 core/apps/server_test.go create mode 100644 core/apps/server_v2.go create mode 100644 core/apps/utils.go create mode 100644 core/cmd/root.go create mode 100644 core/cmd/server.go create mode 100644 core/cmd/server_test.go create mode 100644 core/color/service.go create mode 100644 core/config/base.go create mode 100644 core/config/config.go create mode 100644 core/config/config_test.go create mode 100644 core/config/default_config.go create mode 100644 core/config/path.go create mode 100644 core/config/version.go create mode 100644 core/constants/action.go create mode 100644 core/constants/anchor.go create mode 100644 core/constants/auth.go create mode 100644 core/constants/cache.go create mode 100644 core/constants/channels.go create mode 100644 core/constants/common.go create mode 100644 core/constants/config_spider.go create mode 100644 core/constants/data_collection.go create mode 100644 core/constants/data_field.go create mode 100644 core/constants/database.go create mode 100644 core/constants/delegate.go create mode 100644 core/constants/ds.go create mode 100644 core/constants/encrypt.go create mode 100644 core/constants/errors.go create mode 100644 core/constants/event.go create mode 100644 core/constants/export.go create mode 100644 core/constants/file.go create mode 100644 core/constants/filer.go create mode 100644 core/constants/filter.go create mode 100644 core/constants/git.go create mode 100644 core/constants/grpc.go create mode 100644 core/constants/http.go create mode 100644 core/constants/log.go create mode 100644 core/constants/message.go create mode 100644 core/constants/node.go create mode 100644 core/constants/notification.go create mode 100644 core/constants/pagination.go create mode 100644 core/constants/register.go create mode 100644 core/constants/results.go create mode 100644 core/constants/rpc.go create mode 100644 core/constants/schedule.go create mode 100644 core/constants/scrapy.go create mode 100644 core/constants/signal.go create mode 100644 core/constants/sort.go create mode 100644 core/constants/system.go create mode 100644 core/constants/task.go create mode 100644 core/constants/user.go create mode 100644 core/constants/variable.go create mode 100644 core/container/container.go create mode 100644 core/controllers/base.go create mode 100644 core/controllers/base_v2.go create mode 100644 core/controllers/base_v2_test.go create mode 100644 core/controllers/binder.go create mode 100644 core/controllers/binder_json.go create mode 100644 core/controllers/data_collection.go create mode 100644 core/controllers/data_source.go create mode 100644 core/controllers/delegate_action.go create mode 100644 core/controllers/delegate_basic.go create mode 100644 core/controllers/delegate_list.go create mode 100644 core/controllers/delegate_list_action.go create mode 100644 core/controllers/demo.go create mode 100644 core/controllers/environment.go create mode 100644 core/controllers/export.go create mode 100644 core/controllers/export_v2.go create mode 100644 core/controllers/filer.go create mode 100644 core/controllers/filter.go create mode 100644 core/controllers/filter_v2.go create mode 100644 core/controllers/git.go create mode 100644 core/controllers/http.go create mode 100644 core/controllers/init.go create mode 100644 core/controllers/login.go create mode 100644 core/controllers/login_v2.go create mode 100644 core/controllers/node.go create mode 100644 core/controllers/notification.go create mode 100644 core/controllers/permission.go create mode 100644 core/controllers/project.go create mode 100644 core/controllers/result.go create mode 100644 core/controllers/result_v2.go create mode 100644 core/controllers/role.go create mode 100644 core/controllers/router_v2.go create mode 100644 core/controllers/router_v2_test.go create mode 100644 core/controllers/schedule.go create mode 100644 core/controllers/schedule_v2.go create mode 100644 core/controllers/setting.go create mode 100644 core/controllers/setting_v2.go create mode 100644 core/controllers/spider.go create mode 100644 core/controllers/spider_v2.go create mode 100644 core/controllers/spider_v2_test.go create mode 100644 core/controllers/stats.go create mode 100644 core/controllers/stats_v2.go create mode 100644 core/controllers/sync.go create mode 100644 core/controllers/system_info.go create mode 100644 core/controllers/system_info_v2.go create mode 100644 core/controllers/tag.go create mode 100644 core/controllers/task.go create mode 100644 core/controllers/task_v2.go create mode 100644 core/controllers/test/base.go create mode 100644 core/controllers/test/export_test.go create mode 100644 core/controllers/test/filter_test.go create mode 100644 core/controllers/test/main_test.go create mode 100644 core/controllers/test/project_test.go create mode 100644 core/controllers/test/spider_test.go create mode 100644 core/controllers/test/task_test.go create mode 100644 core/controllers/token.go create mode 100644 core/controllers/token_v2.go create mode 100644 core/controllers/user.go create mode 100644 core/controllers/user_v2.go create mode 100644 core/controllers/user_v2_test.go create mode 100644 core/controllers/utils_context.go create mode 100644 core/controllers/utils_filter.go create mode 100644 core/controllers/utils_http.go create mode 100644 core/controllers/utils_pagination.go create mode 100644 core/controllers/utils_sort.go create mode 100644 core/controllers/version.go create mode 100644 core/data/colors.go create mode 100644 core/docker-compose.yml create mode 100644 core/docs/.gitignore create mode 100644 core/docs/api/index.html create mode 100644 core/docs/api/openapi.yaml create mode 100644 core/docs/package.json create mode 100644 core/docs/scripts/publish.js create mode 100644 core/ds/cockroachdb.go create mode 100644 core/ds/default.go create mode 100644 core/ds/es.go create mode 100644 core/ds/kafka.go create mode 100644 core/ds/mongo.go create mode 100644 core/ds/mssql.go create mode 100644 core/ds/mssql_test.go create mode 100644 core/ds/mysql.go create mode 100644 core/ds/options.go create mode 100644 core/ds/postgresql.go create mode 100644 core/ds/service.go create mode 100644 core/ds/sql.go create mode 100644 core/ds/sql_options.go create mode 100644 core/ds/sqlite.go create mode 100644 core/entity/address.go create mode 100644 core/entity/color.go create mode 100644 core/entity/common.go create mode 100644 core/entity/config_spider.go create mode 100644 core/entity/data_field.go create mode 100644 core/entity/doc.go create mode 100644 core/entity/es.go create mode 100644 core/entity/event.go create mode 100644 core/entity/export.go create mode 100644 core/entity/filter.go create mode 100644 core/entity/filter_select_option.go create mode 100644 core/entity/fs_file_info.go create mode 100644 core/entity/git.go create mode 100644 core/entity/grpc_base_service_message.go create mode 100644 core/entity/grpc_base_service_params.go create mode 100644 core/entity/grpc_delegate_message.go create mode 100644 core/entity/grpc_event_service_message.go create mode 100644 core/entity/grpc_subscribe.go create mode 100644 core/entity/http.go create mode 100644 core/entity/model_delegate.go create mode 100644 core/entity/model_info.go create mode 100644 core/entity/node.go create mode 100644 core/entity/pagination.go create mode 100644 core/entity/result.go create mode 100644 core/entity/rpc.go create mode 100644 core/entity/sort.go create mode 100644 core/entity/spider.go create mode 100644 core/entity/stats.go create mode 100644 core/entity/system_info.go create mode 100644 core/entity/task.go create mode 100644 core/entity/translation.go create mode 100644 core/entity/ttl_map.go create mode 100644 core/entity/version.go create mode 100644 core/errors/base.go create mode 100644 core/errors/controller.go create mode 100644 core/errors/ds.go create mode 100644 core/errors/event.go create mode 100644 core/errors/filter.go create mode 100644 core/errors/fs.go create mode 100644 core/errors/git.go create mode 100644 core/errors/grpc.go create mode 100644 core/errors/http.go create mode 100644 core/errors/model.go create mode 100644 core/errors/node.go create mode 100644 core/errors/process.go create mode 100644 core/errors/result.go create mode 100644 core/errors/schedule.go create mode 100644 core/errors/spider.go create mode 100644 core/errors/stats.go create mode 100644 core/errors/store.go create mode 100644 core/errors/task.go create mode 100644 core/errors/user.go create mode 100644 core/event/func.go create mode 100644 core/event/options.go create mode 100644 core/event/service.go create mode 100644 core/export/csv_service.go create mode 100644 core/export/csv_service_test.go create mode 100644 core/export/json_service.go create mode 100644 core/fs/default.go create mode 100644 core/fs/service_v2.go create mode 100644 core/fs/service_v2_test.go create mode 100644 core/go.mod create mode 100644 core/go.sum create mode 100644 core/grpc/client/client.go create mode 100644 core/grpc/client/client_v2.go create mode 100644 core/grpc/client/options.go create mode 100644 core/grpc/client/pool.go create mode 100644 core/grpc/client/utils_proto.go create mode 100644 core/grpc/middlewares/auth_token.go create mode 100644 core/grpc/payload/model_service_v2_payload.go create mode 100644 core/grpc/server/dependencies_server_v2.go create mode 100644 core/grpc/server/message_server.go create mode 100644 core/grpc/server/model_base_service_binder.go create mode 100644 core/grpc/server/model_base_service_server.go create mode 100644 core/grpc/server/model_base_service_v2_server.go create mode 100644 core/grpc/server/model_delegate_binder.go create mode 100644 core/grpc/server/model_delegate_server.go create mode 100644 core/grpc/server/node_server.go create mode 100644 core/grpc/server/node_server_v2.go create mode 100644 core/grpc/server/options.go create mode 100644 core/grpc/server/server.go create mode 100644 core/grpc/server/server_v2.go create mode 100644 core/grpc/server/task_server.go create mode 100644 core/grpc/server/task_server_v2.go create mode 100644 core/grpc/server/utils_handle.go create mode 100644 core/grpc/test/auth_token_test.go create mode 100644 core/grpc/test/base.go create mode 100644 core/grpc/test/main_test.go create mode 100644 core/grpc/test/model_base_service_server_test.go create mode 100644 core/grpc/test/model_delegate_server_test.go create mode 100644 core/grpc/test/node_server_test.go create mode 100644 core/i18n/service.go create mode 100644 core/interfaces/address.go create mode 100644 core/interfaces/color.go create mode 100644 core/interfaces/color_service.go create mode 100644 core/interfaces/controller_params.go create mode 100644 core/interfaces/data_source_service.go create mode 100644 core/interfaces/entity.go create mode 100644 core/interfaces/event_data.go create mode 100644 core/interfaces/event_service.go create mode 100644 core/interfaces/export.go create mode 100644 core/interfaces/export_service.go create mode 100644 core/interfaces/filter.go create mode 100644 core/interfaces/filter_condition.go create mode 100644 core/interfaces/fs_file_info.go create mode 100644 core/interfaces/fs_service.go create mode 100644 core/interfaces/fs_service_options.go create mode 100644 core/interfaces/fs_service_v2.go create mode 100644 core/interfaces/grpc_base.go create mode 100644 core/interfaces/grpc_base_service_params.go create mode 100644 core/interfaces/grpc_client.go create mode 100644 core/interfaces/grpc_client_model_base_service.go create mode 100644 core/interfaces/grpc_client_model_delegate.go create mode 100644 core/interfaces/grpc_client_model_environment_service.go create mode 100644 core/interfaces/grpc_client_model_node_service.go create mode 100644 core/interfaces/grpc_client_model_service.go create mode 100644 core/interfaces/grpc_client_model_spider_service.go create mode 100644 core/interfaces/grpc_client_model_task_service.go create mode 100644 core/interfaces/grpc_client_model_task_stat_service.go create mode 100644 core/interfaces/grpc_client_pool.go create mode 100644 core/interfaces/grpc_model_base_service_message.go create mode 100644 core/interfaces/grpc_model_binder.go create mode 100644 core/interfaces/grpc_model_delegate_message.go create mode 100644 core/interfaces/grpc_model_list_binder.go create mode 100644 core/interfaces/grpc_server.go create mode 100644 core/interfaces/grpc_stream.go create mode 100644 core/interfaces/grpc_subscribe.go create mode 100644 core/interfaces/i18n_service.go create mode 100644 core/interfaces/injectable.go create mode 100644 core/interfaces/list.go create mode 100644 core/interfaces/model.go create mode 100644 core/interfaces/model_artifact.go create mode 100644 core/interfaces/model_artifact_sys.go create mode 100644 core/interfaces/model_base_service.go create mode 100644 core/interfaces/model_binder.go create mode 100644 core/interfaces/model_delegate.go create mode 100644 core/interfaces/model_environment.go create mode 100644 core/interfaces/model_extra_value.go create mode 100644 core/interfaces/model_git.go create mode 100644 core/interfaces/model_list_binder.go create mode 100644 core/interfaces/model_node.go create mode 100644 core/interfaces/model_node_delegate.go create mode 100644 core/interfaces/model_permission.go create mode 100644 core/interfaces/model_result.go create mode 100644 core/interfaces/model_role.go create mode 100644 core/interfaces/model_schedule.go create mode 100644 core/interfaces/model_service_v2.go create mode 100644 core/interfaces/model_spider.go create mode 100644 core/interfaces/model_tag.go create mode 100644 core/interfaces/model_task.go create mode 100644 core/interfaces/model_task_stat.go create mode 100644 core/interfaces/model_user.go create mode 100644 core/interfaces/model_user_group.go create mode 100644 core/interfaces/module.go create mode 100644 core/interfaces/node_config_service.go create mode 100644 core/interfaces/node_master_service.go create mode 100644 core/interfaces/node_service.go create mode 100644 core/interfaces/node_service_option.go create mode 100644 core/interfaces/node_worker_service.go create mode 100644 core/interfaces/options.go create mode 100644 core/interfaces/process_daemon.go create mode 100644 core/interfaces/provide.go create mode 100644 core/interfaces/result_service.go create mode 100644 core/interfaces/result_service_mongo.go create mode 100644 core/interfaces/result_service_registry.go create mode 100644 core/interfaces/schedule_service.go create mode 100644 core/interfaces/spider_admin_service.go create mode 100644 core/interfaces/spider_service_options.go create mode 100644 core/interfaces/stats_service.go create mode 100644 core/interfaces/task_base_service.go create mode 100644 core/interfaces/task_handler_service.go create mode 100644 core/interfaces/task_hook_service.go create mode 100644 core/interfaces/task_runner.go create mode 100644 core/interfaces/task_scheduler_service.go create mode 100644 core/interfaces/task_stats_service.go create mode 100644 core/interfaces/test.go create mode 100644 core/interfaces/translation.go create mode 100644 core/interfaces/user_service.go create mode 100644 core/interfaces/user_service_options.go create mode 100644 core/interfaces/with_address.go create mode 100644 core/interfaces/with_config_path.go create mode 100644 core/interfaces/with_model_id.go create mode 100644 core/main.go create mode 100644 core/middlewares/auth.go create mode 100644 core/middlewares/auth_v2.go create mode 100644 core/middlewares/cors.go create mode 100644 core/middlewares/filer_auth.go create mode 100644 core/middlewares/middlewares.go create mode 100644 core/models/client/binder_basic.go create mode 100644 core/models/client/binder_list.go create mode 100644 core/models/client/model_base_service.go create mode 100644 core/models/client/model_delegate.go create mode 100644 core/models/client/model_environment_service.go create mode 100644 core/models/client/model_node_delegate.go create mode 100644 core/models/client/model_node_service.go create mode 100644 core/models/client/model_service.go create mode 100644 core/models/client/model_service_v2.go create mode 100644 core/models/client/model_service_v2_test.go create mode 100644 core/models/client/model_spider_service.go create mode 100644 core/models/client/model_task_service.go create mode 100644 core/models/client/model_task_stat_service.go create mode 100644 core/models/client/options.go create mode 100644 core/models/common/index_service.go create mode 100644 core/models/config_spider/common.go create mode 100644 core/models/config_spider/scrapy.go create mode 100644 core/models/delegate/base_test.go create mode 100644 core/models/delegate/model.go create mode 100644 core/models/delegate/model_node.go create mode 100644 core/models/delegate/model_node_test.go create mode 100644 core/models/delegate/model_role_test.go create mode 100644 core/models/delegate/model_test.go create mode 100644 core/models/delegate/model_user_role_test.go create mode 100644 core/models/delegate/utils_event.go create mode 100644 core/models/models/artifact.go create mode 100644 core/models/models/artifact_sys.go create mode 100644 core/models/models/base.go create mode 100644 core/models/models/base_v2.go create mode 100644 core/models/models/data_collection.go create mode 100644 core/models/models/data_collection_v2.go create mode 100644 core/models/models/data_source.go create mode 100644 core/models/models/data_source_v2.go create mode 100644 core/models/models/dependency_setting.go create mode 100644 core/models/models/dependency_setting_v2.go create mode 100644 core/models/models/environment.go create mode 100644 core/models/models/environment_v2.go create mode 100644 core/models/models/extra_value.go create mode 100644 core/models/models/git.go create mode 100644 core/models/models/git_v2.go create mode 100644 core/models/models/job.go create mode 100644 core/models/models/node.go create mode 100644 core/models/models/node_v2.go create mode 100644 core/models/models/notification_setting_v2.go create mode 100644 core/models/models/password.go create mode 100644 core/models/models/permission.go create mode 100644 core/models/models/permission_v2.go create mode 100644 core/models/models/project.go create mode 100644 core/models/models/project_v2.go create mode 100644 core/models/models/result.go create mode 100644 core/models/models/role.go create mode 100644 core/models/models/role_permission.go create mode 100644 core/models/models/role_permission_v2.go create mode 100644 core/models/models/role_v2.go create mode 100644 core/models/models/schedule.go create mode 100644 core/models/models/schedule_v2.go create mode 100644 core/models/models/setting.go create mode 100644 core/models/models/setting_v2.go create mode 100644 core/models/models/spider.go create mode 100644 core/models/models/spider_stat.go create mode 100644 core/models/models/spider_stat_v2.go create mode 100644 core/models/models/spider_v2.go create mode 100644 core/models/models/tag.go create mode 100644 core/models/models/task.go create mode 100644 core/models/models/task_queue_item.go create mode 100644 core/models/models/task_queue_item_v2.go create mode 100644 core/models/models/task_stat.go create mode 100644 core/models/models/task_stat_v2.go create mode 100644 core/models/models/task_v2.go create mode 100644 core/models/models/test.go create mode 100644 core/models/models/token.go create mode 100644 core/models/models/token_v2.go create mode 100644 core/models/models/user.go create mode 100644 core/models/models/user_role.go create mode 100644 core/models/models/user_role_v2.go create mode 100644 core/models/models/user_v2.go create mode 100644 core/models/models/utils_binder_legacy.go create mode 100644 core/models/models/utils_col.go create mode 100644 core/models/models/utils_model_map.go create mode 100644 core/models/models/utils_tag.go create mode 100644 core/models/models/variable.go create mode 100644 core/models/models/variable_v2.go create mode 100644 core/models/service/artifact_service.go create mode 100644 core/models/service/base_service.go create mode 100644 core/models/service/base_service_v2.go create mode 100644 core/models/service/base_service_v2_test.go create mode 100644 core/models/service/binder_basic.go create mode 100644 core/models/service/binder_list.go create mode 100644 core/models/service/data_collection_service.go create mode 100644 core/models/service/data_source_service.go create mode 100644 core/models/service/dependency_setting_service.go create mode 100644 core/models/service/environment_service.go create mode 100644 core/models/service/extra_value_service.go create mode 100644 core/models/service/git_service.go create mode 100644 core/models/service/interface.go create mode 100644 core/models/service/job_service.go create mode 100644 core/models/service/node_service.go create mode 100644 core/models/service/options.go create mode 100644 core/models/service/password_service.go create mode 100644 core/models/service/permission_service.go create mode 100644 core/models/service/project_service.go create mode 100644 core/models/service/role_permission_service.go create mode 100644 core/models/service/role_service.go create mode 100644 core/models/service/schedule_service.go create mode 100644 core/models/service/service.go create mode 100644 core/models/service/setting_service.go create mode 100644 core/models/service/spider_service.go create mode 100644 core/models/service/spider_stat_service.go create mode 100644 core/models/service/tag_service.go create mode 100644 core/models/service/tag_service_legacy.go create mode 100644 core/models/service/task_queue_service.go create mode 100644 core/models/service/task_service.go create mode 100644 core/models/service/task_stat_service.go create mode 100644 core/models/service/token_service.go create mode 100644 core/models/service/user_role_service.go create mode 100644 core/models/service/user_service.go create mode 100644 core/models/service/variable_service.go create mode 100644 core/node/config/config.go create mode 100644 core/node/config/config_service.go create mode 100644 core/node/config/options.go create mode 100644 core/node/service/master_service.go create mode 100644 core/node/service/master_service_v2.go create mode 100644 core/node/service/options.go create mode 100644 core/node/service/worker_service.go create mode 100644 core/node/service/worker_service_v2.go create mode 100644 core/node/test/base.go create mode 100644 core/node/test/base_test.go create mode 100644 core/notification/base_test.go create mode 100644 core/notification/constants.go create mode 100644 core/notification/mail.go create mode 100644 core/notification/mail_theme.go create mode 100644 core/notification/mail_theme_flat.go create mode 100644 core/notification/mobile.go create mode 100644 core/notification/models.go create mode 100644 core/notification/payload.go create mode 100644 core/notification/service.go create mode 100644 core/notification/service_test.go create mode 100644 core/notification/service_v2.go create mode 100644 core/process/daemon.go create mode 100644 core/process/daemon_test.go create mode 100644 core/process/manage.go create mode 100644 core/process/options.go create mode 100644 core/result/options.go create mode 100644 core/result/service.go create mode 100644 core/result/service_mongo.go create mode 100644 core/result/service_registry.go create mode 100644 core/result/test/base.go create mode 100644 core/result/test/service_test.go create mode 100644 core/routes/group.go create mode 100644 core/routes/router.go create mode 100644 core/routes/router_test.go create mode 100644 core/schedule/logger.go create mode 100644 core/schedule/options.go create mode 100644 core/schedule/service.go create mode 100644 core/schedule/service_v2.go create mode 100644 core/schedule/test/base.go create mode 100644 core/schedule/test/schedule_service_test.go create mode 100644 core/spider/admin/options.go create mode 100644 core/spider/admin/service.go create mode 100644 core/spider/admin/service_v2.go create mode 100644 core/stats/options.go create mode 100644 core/stats/service.go create mode 100644 core/sys_exec/sys_exec.go create mode 100644 core/sys_exec/sys_exec_darwin.go create mode 100644 core/sys_exec/sys_exec_linux.go create mode 100644 core/sys_exec/sys_exec_windows.go create mode 100644 core/system/service.go create mode 100644 core/task/base.go create mode 100644 core/task/handler/options.go create mode 100644 core/task/handler/runner.go create mode 100644 core/task/handler/runner_test.go create mode 100644 core/task/handler/runner_v2.go create mode 100644 core/task/handler/service.go create mode 100644 core/task/handler/service_v2.go create mode 100644 core/task/log/constants.go create mode 100644 core/task/log/default.go create mode 100644 core/task/log/driver.go create mode 100644 core/task/log/entity.go create mode 100644 core/task/log/errors.go create mode 100644 core/task/log/file_driver.go create mode 100644 core/task/log/file_driver_test.go create mode 100644 core/task/log/interface.go create mode 100644 core/task/scheduler/options.go create mode 100644 core/task/scheduler/service.go create mode 100644 core/task/scheduler/service_v2.go create mode 100644 core/task/stats/options.go create mode 100644 core/task/stats/service.go create mode 100644 core/task/stats/service_v2.go create mode 100644 core/user/options.go create mode 100644 core/user/service.go create mode 100644 core/user/service_v2.go create mode 100644 core/user/test/base.go create mode 100644 core/user/test/user_service_test.go create mode 100644 core/utils/args.go create mode 100644 core/utils/array.go create mode 100644 core/utils/backoff.go create mode 100644 core/utils/binders/binder_col_name.go create mode 100644 core/utils/bool.go create mode 100644 core/utils/bson.go create mode 100644 core/utils/cache.go create mode 100644 core/utils/chan.go create mode 100644 core/utils/chan_test.go create mode 100644 core/utils/cockroachdb.go create mode 100644 core/utils/cron.go create mode 100644 core/utils/debug.go create mode 100644 core/utils/demo.go create mode 100644 core/utils/di.go create mode 100644 core/utils/docker.go create mode 100644 core/utils/encrypt.go create mode 100644 core/utils/encrypt_test.go create mode 100644 core/utils/es.go create mode 100644 core/utils/file.go create mode 100644 core/utils/file_test.go create mode 100644 core/utils/filter.go create mode 100644 core/utils/git.go create mode 100644 core/utils/helpers.go create mode 100644 core/utils/http.go create mode 100644 core/utils/init.go create mode 100644 core/utils/json.go create mode 100644 core/utils/kafka.go create mode 100644 core/utils/mongo.go create mode 100644 core/utils/mssql.go create mode 100644 core/utils/mysql.go create mode 100644 core/utils/node.go create mode 100644 core/utils/os.go create mode 100644 core/utils/postgresql.go create mode 100644 core/utils/result.go create mode 100644 core/utils/rpc.go create mode 100644 core/utils/spider.go create mode 100644 core/utils/sql.go create mode 100644 core/utils/sqlite.go create mode 100644 core/utils/stats.go create mode 100644 core/utils/system.go create mode 100644 core/utils/task.go create mode 100644 core/utils/time.go create mode 100644 core/utils/uuid.go create mode 100644 grpc/.gitignore create mode 100644 grpc/LICENSE create mode 100644 grpc/README.md create mode 100755 grpc/bin/compile.sh create mode 100644 grpc/dependencies_service_v2_request.pb.go create mode 100644 grpc/dependency_service_v2.pb.go create mode 100644 grpc/dependency_service_v2_grpc.pb.go create mode 100644 grpc/go.mod create mode 100644 grpc/go.sum create mode 100644 grpc/message_service.pb.go create mode 100644 grpc/message_service_grpc.pb.go create mode 100644 grpc/model_base_service.pb.go create mode 100644 grpc/model_base_service_grpc.pb.go create mode 100644 grpc/model_base_service_v2.pb.go create mode 100644 grpc/model_base_service_v2_grpc.pb.go create mode 100644 grpc/model_delegate.pb.go create mode 100644 grpc/model_delegate_grpc.pb.go create mode 100644 grpc/model_service_v2_request.pb.go create mode 100644 grpc/node.pb.go create mode 100644 grpc/node_info.pb.go create mode 100644 grpc/node_service.pb.go create mode 100644 grpc/node_service_grpc.pb.go create mode 100644 grpc/plugin_request.pb.go create mode 100644 grpc/plugin_service.pb.go create mode 100644 grpc/plugin_service_grpc.pb.go create mode 100644 grpc/proto/entity/dependencies_service_v2_request.proto create mode 100644 grpc/proto/entity/model_service_v2_request.proto create mode 100644 grpc/proto/entity/node_info.proto create mode 100644 grpc/proto/entity/plugin_request.proto create mode 100644 grpc/proto/entity/request.proto create mode 100644 grpc/proto/entity/response.proto create mode 100644 grpc/proto/entity/response_code.proto create mode 100644 grpc/proto/entity/stream_message.proto create mode 100644 grpc/proto/entity/stream_message_code.proto create mode 100644 grpc/proto/entity/stream_message_data_task.proto create mode 100644 grpc/proto/models/node.proto create mode 100644 grpc/proto/models/task.proto create mode 100644 grpc/proto/services/dependency_service_v2.proto create mode 100644 grpc/proto/services/message_service.proto create mode 100644 grpc/proto/services/model_base_service.proto create mode 100644 grpc/proto/services/model_base_service_v2.proto create mode 100644 grpc/proto/services/model_delegate.proto create mode 100644 grpc/proto/services/node_service.proto create mode 100644 grpc/proto/services/plugin_service.proto create mode 100644 grpc/proto/services/task_service.proto create mode 100644 grpc/request.pb.go create mode 100644 grpc/response.pb.go create mode 100644 grpc/response_code.pb.go create mode 100644 grpc/stream_message.pb.go create mode 100644 grpc/stream_message_code.pb.go create mode 100644 grpc/stream_message_data_task.pb.go create mode 100644 grpc/task.pb.go create mode 100644 grpc/task_service.pb.go create mode 100644 grpc/task_service_grpc.pb.go diff --git a/core/.editorconfig b/core/.editorconfig new file mode 100644 index 000000000..45a3e334c --- /dev/null +++ b/core/.editorconfig @@ -0,0 +1,13 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = tab +insert_final_newline = true +trim_trailing_whitespace = true + +[{*.yaml,*.yml,package.json}] +indent_size = 2 +indent_style = space diff --git a/core/.github/workflows/test.yml b/core/.github/workflows/test.yml new file mode 100644 index 000000000..945d03e7c --- /dev/null +++ b/core/.github/workflows/test.yml @@ -0,0 +1,36 @@ +name: "Test" + +on: + push: + branches: [ main, develop ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main, develop ] + +jobs: + test: + name: Test + runs-on: ubuntu-20.04 + services: + mongo: + image: mongo:5 + ports: + - 27017:27017 + env: + CRAWLAB_SERVER_PORT: 9999 + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - uses: actions/setup-go@v3 + with: + go-version: '^1.22' + - name: Run unit tests + run: | + mods=(\ + "github.com/crawlab-team/crawlab/core/controllers" \ + "github.com/crawlab-team/crawlab/core/models/client" \ + "github.com/crawlab-team/crawlab/core/models/service" \ + ) + for pkg in ${mods[@]}; do + go test ${pkg} + done diff --git a/core/.gitignore b/core/.gitignore new file mode 100644 index 000000000..53fc51810 --- /dev/null +++ b/core/.gitignore @@ -0,0 +1,10 @@ +.idea +.DS_Store +vendor/ +tmp/ +build/ +dist/ +*.log +gen/ +*.exe +*.txt diff --git a/core/LICENSE b/core/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/core/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/core/README.md b/core/README.md new file mode 100644 index 000000000..275133a59 --- /dev/null +++ b/core/README.md @@ -0,0 +1,2 @@ +# crawlab-core +Backend core modules for Crawlab diff --git a/core/apps/api.go b/core/apps/api.go new file mode 100644 index 000000000..94e887c34 --- /dev/null +++ b/core/apps/api.go @@ -0,0 +1,126 @@ +package apps + +import ( + "context" + "errors" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/controllers" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/middlewares" + "github.com/crawlab-team/crawlab/core/routes" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "net" + "net/http" + "time" +) + +func init() { + // set gin mode + if viper.GetString("gin.mode") == "" { + gin.SetMode(gin.ReleaseMode) + } else { + gin.SetMode(viper.GetString("gin.mode")) + } +} + +type Api struct { + // dependencies + interfaces.WithConfigPath + + // internals + app *gin.Engine + ln net.Listener + srv *http.Server + ready bool +} + +func (app *Api) Init() { + // initialize controllers + _ = initModule("controllers", controllers.InitControllers) + + // initialize middlewares + _ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares) + + // initialize routes + _ = app.initModuleWithApp("routes", routes.InitRoutes) +} + +func (app *Api) Start() { + // address + host := viper.GetString("server.host") + port := viper.GetString("server.port") + address := net.JoinHostPort(host, port) + + // http server + app.srv = &http.Server{ + Handler: app.app, + Addr: address, + } + + // listen + var err error + app.ln, err = net.Listen("tcp", address) + if err != nil { + panic(err) + } + app.ready = true + + // serve + if err := http.Serve(app.ln, app.app); err != nil { + if !errors.Is(err, http.ErrServerClosed) { + log.Error("run server error:" + err.Error()) + } else { + log.Info("server graceful down") + } + } +} + +func (app *Api) Wait() { + DefaultWait() +} + +func (app *Api) Stop() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := app.srv.Shutdown(ctx); err != nil { + log.Error("run server error:" + err.Error()) + } +} + +func (app *Api) GetGinEngine() *gin.Engine { + return app.app +} + +func (app *Api) GetHttpServer() *http.Server { + return app.srv +} + +func (app *Api) Ready() (ok bool) { + return app.ready +} + +func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) { + return initModule(name, func() error { + return fn(app.app) + }) +} + +func NewApi() *Api { + api := &Api{ + app: gin.New(), + } + api.Init() + return api +} + +var api *Api + +func GetApi() *Api { + if api != nil { + return api + } + api = NewApi() + return api +} diff --git a/core/apps/api_v2.go b/core/apps/api_v2.go new file mode 100644 index 000000000..966b1a087 --- /dev/null +++ b/core/apps/api_v2.go @@ -0,0 +1,122 @@ +package apps + +import ( + "context" + "errors" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/controllers" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/middlewares" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "net" + "net/http" + "time" +) + +func init() { + // set gin mode + if viper.GetString("gin.mode") == "" { + gin.SetMode(gin.ReleaseMode) + } else { + gin.SetMode(viper.GetString("gin.mode")) + } +} + +type ApiV2 struct { + // dependencies + interfaces.WithConfigPath + + // internals + app *gin.Engine + ln net.Listener + srv *http.Server + ready bool +} + +func (app *ApiV2) Init() { + // initialize middlewares + _ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares) + + // initialize routes + _ = app.initModuleWithApp("routes", controllers.InitRoutes) +} + +func (app *ApiV2) Start() { + // address + host := viper.GetString("server.host") + port := viper.GetString("server.port") + address := net.JoinHostPort(host, port) + + // http server + app.srv = &http.Server{ + Handler: app.app, + Addr: address, + } + + // listen + var err error + app.ln, err = net.Listen("tcp", address) + if err != nil { + panic(err) + } + app.ready = true + + // serve + if err := http.Serve(app.ln, app.app); err != nil { + if !errors.Is(err, http.ErrServerClosed) { + log.Error("run server error:" + err.Error()) + } else { + log.Info("server graceful down") + } + } +} + +func (app *ApiV2) Wait() { + DefaultWait() +} + +func (app *ApiV2) Stop() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := app.srv.Shutdown(ctx); err != nil { + log.Error("run server error:" + err.Error()) + } +} + +func (app *ApiV2) GetGinEngine() *gin.Engine { + return app.app +} + +func (app *ApiV2) GetHttpServer() *http.Server { + return app.srv +} + +func (app *ApiV2) Ready() (ok bool) { + return app.ready +} + +func (app *ApiV2) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) { + return initModule(name, func() error { + return fn(app.app) + }) +} + +func NewApiV2() *ApiV2 { + api := &ApiV2{ + app: gin.New(), + } + api.Init() + return api +} + +var apiV2 *ApiV2 + +func GetApiV2() *ApiV2 { + if apiV2 != nil { + return apiV2 + } + apiV2 = NewApiV2() + return apiV2 +} diff --git a/core/apps/docker.go b/core/apps/docker.go new file mode 100644 index 000000000..948a5cff9 --- /dev/null +++ b/core/apps/docker.go @@ -0,0 +1,199 @@ +package apps + +import ( + "bufio" + "fmt" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/sys_exec" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/imroc/req" + "github.com/spf13/viper" + "os" + "os/exec" + "strings" + "time" +) + +type Docker struct { + // parent + parent ServerApp + + // dependencies + interfaces.WithConfigPath + + // seaweedfs log + fsLogFilePath string + fsLogFile *os.File + fsReady bool +} + +func (app *Docker) Init() { + var err error + app.fsLogFile, err = os.OpenFile(app.fsLogFilePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.FileMode(0777)) + if err != nil { + trace.PrintError(err) + } + + // replace paths + if err := app.replacePaths(); err != nil { + panic(err) + } + + // start nginx + go app.startNginx() + + // start seaweedfs + go app.startSeaweedFs() +} + +func (app *Docker) Start() { + // import demo + //if utils.IsDemo() && utils.InitializedDemo() { + // go app.importDemo() + //} +} + +func (app *Docker) Wait() { + DefaultWait() +} + +func (app *Docker) Stop() { +} + +func (app *Docker) GetParent() (parent ServerApp) { + return app.parent +} + +func (app *Docker) SetParent(parent ServerApp) { + app.parent = parent +} + +func (app *Docker) Ready() (ok bool) { + return app.fsReady && + app.parent.GetApi().Ready() +} + +func (app *Docker) replacePaths() (err error) { + // read + indexHtmlPath := "/app/dist/index.html" + indexHtmlBytes, err := os.ReadFile(indexHtmlPath) + if err != nil { + return trace.TraceError(err) + } + indexHtml := string(indexHtmlBytes) + + // replace paths + baseUrl := viper.GetString("base.url") + if baseUrl != "" { + indexHtml = app._replacePath(indexHtml, "js", baseUrl) + indexHtml = app._replacePath(indexHtml, "css", baseUrl) + indexHtml = app._replacePath(indexHtml, " d.svc.GetMany:start") + list, err := d.svc.GetList(nil, &mongo.FindOptions{ + Sort: bson.D{{"_id", -1}}, + }) + if err != nil { + if err == mongo2.ErrNoDocuments { + HandleErrorNotFound(c, err) + } else { + HandleErrorInternalServerError(c, err) + } + return + } + log.Debugf("getAll -> d.svc.GetMany:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds()) + tic = time.Now() + + // total count + tic = time.Now() + log.Debugf("getAll -> d.svc.Count:start") + total, err := d.svc.Count(nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + log.Debugf("getAll -> d.svc.Count:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds()) + + // response + HandleSuccessWithListData(c, list, total) +} + +func (d *ListControllerDelegate) getList(c *gin.Context) (l interfaces.List, total int, err error) { + // params + pagination := MustGetPagination(c) + query := MustGetFilterQuery(c) + sort := MustGetSortOption(c) + + // get list + l, err = d.svc.GetList(query, &mongo.FindOptions{ + Sort: sort, + Skip: pagination.Size * (pagination.Page - 1), + Limit: pagination.Size, + }) + if err != nil { + if err.Error() == mongo2.ErrNoDocuments.Error() { + HandleSuccessWithListData(c, nil, 0) + } else { + HandleErrorInternalServerError(c, err) + } + return + } + + // total count + total, err = d.svc.Count(query) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + return l, total, nil +} diff --git a/core/controllers/delegate_list_action.go b/core/controllers/delegate_list_action.go new file mode 100644 index 000000000..deacf347e --- /dev/null +++ b/core/controllers/delegate_list_action.go @@ -0,0 +1,17 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" +) + +func NewListPostActionControllerDelegate(id ControllerId, svc interfaces.ModelBaseService, actions []Action) (d *ListActionControllerDelegate) { + return &ListActionControllerDelegate{ + NewListControllerDelegate(id, svc), + NewActionControllerDelegate(id, actions), + } +} + +type ListActionControllerDelegate struct { + ListController + ActionController +} diff --git a/core/controllers/demo.go b/core/controllers/demo.go new file mode 100644 index 000000000..9c25aab4a --- /dev/null +++ b/core/controllers/demo.go @@ -0,0 +1,73 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "net/http" +) + +func getDemoActions() []Action { + ctx := newDemoContext() + return []Action{ + { + Method: http.MethodGet, + Path: "/import", + HandlerFunc: ctx.import_, + }, + { + Method: http.MethodGet, + Path: "/reimport", + HandlerFunc: ctx.reimport, + }, + { + Method: http.MethodGet, + Path: "/cleanup", + HandlerFunc: ctx.cleanup, + }, + } +} + +type demoContext struct { +} + +func (ctx *demoContext) import_(c *gin.Context) { + if err := utils.ImportDemo(); err != nil { + trace.PrintError(err) + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +func (ctx *demoContext) reimport(c *gin.Context) { + if err := utils.ReimportDemo(); err != nil { + trace.PrintError(err) + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +func (ctx *demoContext) cleanup(c *gin.Context) { + if err := utils.ReimportDemo(); err != nil { + trace.PrintError(err) + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +var _demoCtx *demoContext + +func newDemoContext() *demoContext { + if _demoCtx != nil { + return _demoCtx + } + + _demoCtx = &demoContext{} + + return _demoCtx +} + +var DemoController ActionController diff --git a/core/controllers/environment.go b/core/controllers/environment.go new file mode 100644 index 000000000..560b02c9d --- /dev/null +++ b/core/controllers/environment.go @@ -0,0 +1,57 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/service" +) + +var EnvironmentController *environmentController + +var EnvironmentActions []Action + +type environmentController struct { + ListActionControllerDelegate + d ListActionControllerDelegate + ctx *environmentContext +} + +type environmentContext struct { + modelSvc service.ModelService + userSvc interfaces.UserService +} + +func newEnvironmentContext() *environmentContext { + // context + ctx := &environmentContext{} + + // dependency injection + if err := container.GetContainer().Invoke(func( + modelSvc service.ModelService, + userSvc interfaces.UserService, + ) { + ctx.modelSvc = modelSvc + ctx.userSvc = userSvc + }); err != nil { + panic(err) + } + + return ctx +} + +func newEnvironmentController() *environmentController { + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + + ctr := NewListPostActionControllerDelegate(ControllerIdEnvironment, modelSvc.GetBaseService(interfaces.ModelIdEnvironment), EnvironmentActions) + d := NewListPostActionControllerDelegate(ControllerIdEnvironment, modelSvc.GetBaseService(interfaces.ModelIdEnvironment), EnvironmentActions) + ctx := newEnvironmentContext() + + return &environmentController{ + ListActionControllerDelegate: *ctr, + d: *d, + ctx: ctx, + } +} diff --git a/core/controllers/export.go b/core/controllers/export.go new file mode 100644 index 000000000..b99ffcb9d --- /dev/null +++ b/core/controllers/export.go @@ -0,0 +1,127 @@ +package controllers + +import ( + "errors" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/export" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/gin-gonic/gin" + "net/http" +) + +var ExportController ActionController + +func getExportActions() []Action { + ctx := newExportContext() + return []Action{ + { + Method: http.MethodPost, + Path: "/:type", + HandlerFunc: ctx.postExport, + }, + { + Method: http.MethodGet, + Path: "/:type/:id", + HandlerFunc: ctx.getExport, + }, + { + Method: http.MethodGet, + Path: "/:type/:id/download", + HandlerFunc: ctx.getExportDownload, + }, + } +} + +type exportContext struct { + csvSvc interfaces.ExportService + jsonSvc interfaces.ExportService +} + +func (ctx *exportContext) postExport(c *gin.Context) { + exportType := c.Param("type") + exportTarget := c.Query("target") + exportFilter, _ := GetFilter(c) + + var exportId string + var err error + switch exportType { + case constants.ExportTypeCsv: + exportId, err = ctx.csvSvc.Export(exportType, exportTarget, exportFilter) + case constants.ExportTypeJson: + exportId, err = ctx.jsonSvc.Export(exportType, exportTarget, exportFilter) + default: + HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) + return + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, exportId) +} + +func (ctx *exportContext) getExport(c *gin.Context) { + exportType := c.Param("type") + exportId := c.Param("id") + + var exp interfaces.Export + var err error + switch exportType { + case constants.ExportTypeCsv: + exp, err = ctx.csvSvc.GetExport(exportId) + case constants.ExportTypeJson: + exp, err = ctx.jsonSvc.GetExport(exportId) + default: + HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, exp) +} + +func (ctx *exportContext) getExportDownload(c *gin.Context) { + exportType := c.Param("type") + exportId := c.Param("id") + + var exp interfaces.Export + var err error + switch exportType { + case constants.ExportTypeCsv: + exp, err = ctx.csvSvc.GetExport(exportId) + case constants.ExportTypeJson: + exp, err = ctx.jsonSvc.GetExport(exportId) + default: + HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + switch exportType { + case constants.ExportTypeCsv: + c.Header("Content-Type", "text/csv") + case constants.ExportTypeJson: + c.Header("Content-Type", "text/plain") + default: + HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", exp.GetDownloadPath())) + c.File(exp.GetDownloadPath()) +} + +func newExportContext() *exportContext { + return &exportContext{ + csvSvc: export.GetCsvService(), + jsonSvc: export.GetJsonService(), + } +} diff --git a/core/controllers/export_v2.go b/core/controllers/export_v2.go new file mode 100644 index 000000000..8fff73055 --- /dev/null +++ b/core/controllers/export_v2.go @@ -0,0 +1,87 @@ +package controllers + +import ( + "errors" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/export" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/gin-gonic/gin" +) + +func PostExport(c *gin.Context) { + exportType := c.Param("type") + exportTarget := c.Query("target") + exportFilter, _ := GetFilter(c) + + var exportId string + var err error + switch exportType { + case constants.ExportTypeCsv: + exportId, err = export.GetCsvService().Export(exportType, exportTarget, exportFilter) + case constants.ExportTypeJson: + exportId, err = export.GetJsonService().Export(exportType, exportTarget, exportFilter) + default: + HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) + return + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, exportId) +} + +func GetExport(c *gin.Context) { + exportType := c.Param("type") + exportId := c.Param("id") + + var exp interfaces.Export + var err error + switch exportType { + case constants.ExportTypeCsv: + exp, err = export.GetCsvService().GetExport(exportId) + case constants.ExportTypeJson: + exp, err = export.GetJsonService().GetExport(exportId) + default: + HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, exp) +} + +func GetExportDownload(c *gin.Context) { + exportType := c.Param("type") + exportId := c.Param("id") + + var exp interfaces.Export + var err error + switch exportType { + case constants.ExportTypeCsv: + exp, err = export.GetCsvService().GetExport(exportId) + case constants.ExportTypeJson: + exp, err = export.GetJsonService().GetExport(exportId) + default: + HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + switch exportType { + case constants.ExportTypeCsv: + c.Header("Content-Type", "text/csv") + case constants.ExportTypeJson: + c.Header("Content-Type", "text/plain") + default: + HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) + } + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", exp.GetDownloadPath())) + c.File(exp.GetDownloadPath()) +} diff --git a/core/controllers/filer.go b/core/controllers/filer.go new file mode 100644 index 000000000..4af5d1424 --- /dev/null +++ b/core/controllers/filer.go @@ -0,0 +1,130 @@ +package controllers + +import ( + "bufio" + "fmt" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/gin-gonic/gin" + "github.com/imroc/req" + "github.com/spf13/viper" + "net/http" + "strings" +) + +var FilerController ActionController + +func getFilerActions() []Action { + filerCtx := newFilerContext() + return []Action{ + { + Method: http.MethodGet, + Path: "*path", + HandlerFunc: filerCtx.do, + }, + { + Method: http.MethodPost, + Path: "*path", + HandlerFunc: filerCtx.do, + }, + { + Method: http.MethodPut, + Path: "*path", + HandlerFunc: filerCtx.do, + }, + { + Method: http.MethodDelete, + Path: "*path", + HandlerFunc: filerCtx.do, + }, + } +} + +type filerContext struct { + endpoint string +} + +func (ctx *filerContext) do(c *gin.Context) { + // request path + requestPath := strings.Replace(c.Request.URL.Path, "/filer", "", 1) + + // request url + requestUrl := fmt.Sprintf("%s%s", ctx.endpoint, requestPath) + if c.Request.URL.RawQuery != "" { + requestUrl += "?" + c.Request.URL.RawQuery + } + + // request body + bufR := bufio.NewScanner(c.Request.Body) + requestBody := req.BodyJSON(bufR.Bytes()) + + // request file uploads + var requestFileUploads []req.FileUpload + form, err := c.MultipartForm() + if err == nil { + for k, v := range form.File { + for _, fh := range v { + f, err := fh.Open() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + requestFileUploads = append(requestFileUploads, req.FileUpload{ + FileName: fh.Filename, + FieldName: k, + File: f, + }) + } + } + } + + // request header + requestHeader := req.Header{} + for k, v := range c.Request.Header { + if len(v) > 0 { + requestHeader[k] = v[0] + } + } + + // perform request + res, err := req.Do(c.Request.Method, requestUrl, requestHeader, requestBody, requestFileUploads) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // status code check + statusCode := res.Response().StatusCode + if statusCode == http.StatusNotFound { + HandleErrorNotFoundNoPrint(c, errors.ErrorControllerFilerNotFound) + return + } + + // response + for k, v := range res.Response().Header { + if len(v) > 0 { + c.Header(k, v[0]) + } + } + _, _ = c.Writer.Write(res.Bytes()) + c.AbortWithStatus(statusCode) +} + +var _filerCtx *filerContext + +func newFilerContext() *filerContext { + if _filerCtx != nil { + return _filerCtx + } + + ctx := &filerContext{ + endpoint: "http://localhost:8888", + } + + if viper.GetString("fs.filer.proxy") != "" { + ctx.endpoint = viper.GetString("fs.filer.proxy") + } + + _filerCtx = ctx + + return ctx +} diff --git a/core/controllers/filter.go b/core/controllers/filter.go new file mode 100644 index 000000000..143e4d13c --- /dev/null +++ b/core/controllers/filter.go @@ -0,0 +1,100 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "net/http" +) + +var FilterController ActionController + +func getFilterActions() []Action { + ctx := newFilterContext() + return []Action{ + { + Method: http.MethodGet, + Path: "/:col", + HandlerFunc: ctx.getColFieldOptions, + }, + { + Method: http.MethodGet, + Path: "/:col/:value", + HandlerFunc: ctx.getColFieldOptions, + }, + { + Method: http.MethodGet, + Path: "/:col/:value/:label", + HandlerFunc: ctx.getColFieldOptions, + }, + } +} + +type filterContext struct { +} + +func (ctx *filterContext) getColFieldOptions(c *gin.Context) { + colName := c.Param("col") + value := c.Param("value") + if value == "" { + value = "_id" + } + label := c.Param("label") + if label == "" { + label = "name" + } + query := MustGetFilterQuery(c) + pipelines := mongo2.Pipeline{} + if query != nil { + pipelines = append(pipelines, bson.D{{"$match", query}}) + } + pipelines = append( + pipelines, + bson.D{ + { + "$group", + bson.M{ + "_id": bson.M{ + "value": "$" + value, + "label": "$" + label, + }, + }, + }, + }, + ) + pipelines = append( + pipelines, + bson.D{ + { + "$project", + bson.M{ + "value": "$_id.value", + "label": bson.M{"$toString": "$_id.label"}, + }, + }, + }, + ) + pipelines = append( + pipelines, + bson.D{ + { + "$sort", + bson.D{ + {"value", 1}, + }, + }, + }, + ) + var options []entity.FilterSelectOption + if err := mongo.GetMongoCol(colName).Aggregate(pipelines, nil).All(&options); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccessWithData(c, options) +} + +func newFilterContext() *filterContext { + return &filterContext{} +} diff --git a/core/controllers/filter_v2.go b/core/controllers/filter_v2.go new file mode 100644 index 000000000..246750d01 --- /dev/null +++ b/core/controllers/filter_v2.go @@ -0,0 +1,69 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" +) + +func GetFilterColFieldOptions(c *gin.Context) { + colName := c.Param("col") + value := c.Param("value") + if value == "" { + value = "_id" + } + label := c.Param("label") + if label == "" { + label = "name" + } + query := MustGetFilterQuery(c) + pipelines := mongo2.Pipeline{} + if query != nil { + pipelines = append(pipelines, bson.D{{"$match", query}}) + } + pipelines = append( + pipelines, + bson.D{ + { + "$group", + bson.M{ + "_id": bson.M{ + "value": "$" + value, + "label": "$" + label, + }, + }, + }, + }, + ) + pipelines = append( + pipelines, + bson.D{ + { + "$project", + bson.M{ + "value": "$_id.value", + "label": bson.M{"$toString": "$_id.label"}, + }, + }, + }, + ) + pipelines = append( + pipelines, + bson.D{ + { + "$sort", + bson.D{ + {"value", 1}, + }, + }, + }, + ) + var options []entity.FilterSelectOption + if err := mongo.GetMongoCol(colName).Aggregate(pipelines, nil).All(&options); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccessWithData(c, options) +} diff --git a/core/controllers/git.go b/core/controllers/git.go new file mode 100644 index 000000000..128323d49 --- /dev/null +++ b/core/controllers/git.go @@ -0,0 +1,3 @@ +package controllers + +var GitController ListController diff --git a/core/controllers/http.go b/core/controllers/http.go new file mode 100644 index 000000000..8aa5169ba --- /dev/null +++ b/core/controllers/http.go @@ -0,0 +1,16 @@ +package controllers + +type Response[T any] struct { + Status string `json:"status"` + Message string `json:"message"` + Data T `json:"data"` + Error string `json:"error"` +} + +type ListResponse[T any] struct { + Status string `json:"status"` + Message string `json:"message"` + Total int `json:"total"` + Data []T `json:"data"` + Error string `json:"error"` +} diff --git a/core/controllers/init.go b/core/controllers/init.go new file mode 100644 index 000000000..e927b9c31 --- /dev/null +++ b/core/controllers/init.go @@ -0,0 +1,42 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/service" +) + +func InitControllers() (err error) { + modelSvc, err := service.GetService() + if err != nil { + return err + } + + NodeController = newNodeController() + ProjectController = newProjectController() + SpiderController = newSpiderController() + TaskController = newTaskController() + UserController = newUserController() + TagController = NewListControllerDelegate(ControllerIdTag, modelSvc.GetBaseService(interfaces.ModelIdTag)) + SettingController = newSettingController() + LoginController = NewActionControllerDelegate(ControllerIdLogin, getLoginActions()) + DataCollectionController = newDataCollectionController() + ResultController = NewActionControllerDelegate(ControllerIdResult, getResultActions()) + ScheduleController = newScheduleController() + StatsController = NewActionControllerDelegate(ControllerIdStats, getStatsActions()) + TokenController = newTokenController() + FilerController = NewActionControllerDelegate(ControllerIdFiler, getFilerActions()) + GitController = NewListControllerDelegate(ControllerIdGit, modelSvc.GetBaseService(interfaces.ModelIdGit)) + VersionController = NewActionControllerDelegate(ControllerIdVersion, getVersionActions()) + SystemInfoController = NewActionControllerDelegate(ControllerIdSystemInfo, getSystemInfoActions()) + DemoController = NewActionControllerDelegate(ControllerIdDemo, getDemoActions()) + RoleController = NewListControllerDelegate(ControllerIdRole, modelSvc.GetBaseService(interfaces.ModelIdRole)) + PermissionController = NewListControllerDelegate(ControllerIdPermission, modelSvc.GetBaseService(interfaces.ModelIdPermission)) + ExportController = NewActionControllerDelegate(ControllerIdExport, getExportActions()) + NotificationController = NewActionControllerDelegate(ControllerIdNotification, getNotificationActions()) + FilterController = NewActionControllerDelegate(ControllerIdFilter, getFilterActions()) + SyncController = NewActionControllerDelegate(ControllerIdSync, getSyncActions()) + DataSourceController = newDataSourceController() + EnvironmentController = newEnvironmentController() + + return nil +} diff --git a/core/controllers/login.go b/core/controllers/login.go new file mode 100644 index 000000000..42926fd51 --- /dev/null +++ b/core/controllers/login.go @@ -0,0 +1,64 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/gin-gonic/gin" + "net/http" +) + +var LoginController ActionController + +func getLoginActions() []Action { + loginCtx := newLoginContext() + return []Action{ + {Method: http.MethodPost, Path: "/login", HandlerFunc: loginCtx.login}, + {Method: http.MethodPost, Path: "/logout", HandlerFunc: loginCtx.logout}, + } +} + +type loginContext struct { + userSvc interfaces.UserService +} + +func (ctx *loginContext) login(c *gin.Context) { + var u models.User + if err := c.ShouldBindJSON(&u); err != nil { + HandleErrorBadRequest(c, err) + return + } + token, loggedInUser, err := ctx.userSvc.Login(&interfaces.UserLoginOptions{ + Username: u.Username, + Password: u.Password, + }) + if err != nil { + HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) + return + } + c.Set(constants.UserContextKey, loggedInUser) + HandleSuccessWithData(c, token) +} + +func (ctx *loginContext) logout(c *gin.Context) { + c.Set(constants.UserContextKey, nil) + HandleSuccess(c) +} + +func newLoginContext() *loginContext { + // context + ctx := &loginContext{} + + // dependency injection + if err := container.GetContainer().Invoke(func( + userSvc interfaces.UserService, + ) { + ctx.userSvc = userSvc + }); err != nil { + panic(err) + } + + return ctx +} diff --git a/core/controllers/login_v2.go b/core/controllers/login_v2.go new file mode 100644 index 000000000..2bb08cef1 --- /dev/null +++ b/core/controllers/login_v2.go @@ -0,0 +1,36 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/user" + "github.com/gin-gonic/gin" +) + +func PostLogin(c *gin.Context) { + var payload struct { + Username string `json:"username"` + Password string `json:"password"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + userSvc, err := user.GetUserServiceV2() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + token, loggedInUser, err := userSvc.Login(payload.Username, payload.Password) + if err != nil { + HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) + return + } + c.Set(constants.UserContextKey, loggedInUser) + HandleSuccessWithData(c, token) +} + +func PostLogout(c *gin.Context) { + c.Set(constants.UserContextKey, nil) + HandleSuccess(c) +} diff --git a/core/controllers/node.go b/core/controllers/node.go new file mode 100644 index 000000000..1ea748b26 --- /dev/null +++ b/core/controllers/node.go @@ -0,0 +1,94 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +var NodeController *nodeController + +type nodeController struct { + ListControllerDelegate +} + +func (ctr *nodeController) Post(c *gin.Context) { + var n models.Node + if err := c.ShouldBindJSON(&n); err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := ctr._post(c, &n); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func (ctr *nodeController) PostList(c *gin.Context) { + // bind + var docs []models.Node + if err := c.ShouldBindJSON(&docs); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // success ids + var ids []primitive.ObjectID + + // iterate nodes + for _, n := range docs { + if err := ctr._post(c, &n); err != nil { + trace.PrintError(err) + continue + } + ids = append(ids, n.Id) + } + + // success + HandleSuccessWithData(c, docs) +} + +func (ctr *nodeController) _post(c *gin.Context, n *models.Node) (err error) { + // set default key + if n.Key == "" { + id, err := uuid.NewUUID() + if err != nil { + return trace.TraceError(err) + } + n.Key = id.String() + } + + // set default status + if n.Status == "" { + n.Status = constants.NodeStatusUnregistered + } + + // add + if err := delegate.NewModelDelegate(n, GetUserFromContext(c)).Add(); err != nil { + return trace.TraceError(err) + } + + return nil +} + +func newNodeController() *nodeController { + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + + ctr := NewListControllerDelegate(ControllerIdNode, modelSvc.GetBaseService(interfaces.ModelIdNode)) + + return &nodeController{ + ListControllerDelegate: *ctr, + } +} diff --git a/core/controllers/notification.go b/core/controllers/notification.go new file mode 100644 index 000000000..31b96c048 --- /dev/null +++ b/core/controllers/notification.go @@ -0,0 +1,158 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/notification" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson/primitive" + "net/http" +) + +var NotificationController ActionController + +func getNotificationActions() []Action { + ctx := newNotificationContext() + return []Action{ + { + Method: http.MethodGet, + Path: "/settings", + HandlerFunc: ctx.GetSettingList, + }, + { + Method: http.MethodGet, + Path: "/settings/:id", + HandlerFunc: ctx.GetSetting, + }, + { + Method: http.MethodPost, + Path: "/settings", + HandlerFunc: ctx.PostSetting, + }, + { + Method: http.MethodPut, + Path: "/settings/:id", + HandlerFunc: ctx.PutSetting, + }, + { + Method: http.MethodDelete, + Path: "/settings/:id", + HandlerFunc: ctx.DeleteSetting, + }, + { + Method: http.MethodPost, + Path: "/settings/:id/enable", + HandlerFunc: ctx.EnableSetting, + }, + { + Method: http.MethodPost, + Path: "/settings/:id/disable", + HandlerFunc: ctx.DisableSetting, + }, + } +} + +type notificationContext struct { + svc *notification.Service +} + +func (ctx *notificationContext) GetSettingList(c *gin.Context) { + query := MustGetFilterQuery(c) + pagination := MustGetPagination(c) + sort := MustGetSortOption(c) + res, total, err := ctx.svc.GetSettingList(query, pagination, sort) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccessWithListData(c, res, total) +} + +func (ctx *notificationContext) GetSetting(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + res, err := ctx.svc.GetSetting(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccessWithData(c, res) +} + +func (ctx *notificationContext) PostSetting(c *gin.Context) { + var s notification.NotificationSetting + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorBadRequest(c, err) + return + } + if err := ctx.svc.PosSetting(&s); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +func (ctx *notificationContext) PutSetting(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + var s notification.NotificationSetting + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorBadRequest(c, err) + return + } + if err := ctx.svc.PutSetting(id, s); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +func (ctx *notificationContext) DeleteSetting(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + if err := ctx.svc.DeleteSetting(id); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +func (ctx *notificationContext) EnableSetting(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + if err := ctx.svc.EnableSetting(id); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +func (ctx *notificationContext) DisableSetting(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + if err := ctx.svc.DisableSetting(id); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +func newNotificationContext() *notificationContext { + ctx := ¬ificationContext{ + svc: notification.GetService(), + } + return ctx +} diff --git a/core/controllers/permission.go b/core/controllers/permission.go new file mode 100644 index 000000000..42be45dca --- /dev/null +++ b/core/controllers/permission.go @@ -0,0 +1,3 @@ +package controllers + +var PermissionController ListController diff --git a/core/controllers/project.go b/core/controllers/project.go new file mode 100644 index 000000000..fe13a9e80 --- /dev/null +++ b/core/controllers/project.go @@ -0,0 +1,103 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +var ProjectController *projectController + +type projectController struct { + ListControllerDelegate +} + +func (ctr *projectController) GetList(c *gin.Context) { + // get all if query field "all" is set true + all := MustGetFilterAll(c) + if all { + ctr.getAll(c) + return + } + + // get list + list, total, err := ctr.getList(c) + if err != nil { + return + } + data := list.GetModels() + + // check empty list + if len(list.GetModels()) == 0 { + HandleSuccessWithListData(c, nil, 0) + return + } + + // project ids + var ids []primitive.ObjectID + + // count cache + cache := map[primitive.ObjectID]int{} + + // iterate + for _, d := range data { + p, ok := d.(*models.Project) + if !ok { + HandleErrorInternalServerError(c, errors.ErrorControllerInvalidType) + return + } + ids = append(ids, p.Id) + cache[p.Id] = 0 + } + + // spiders + modelSvc, err := service.NewService() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + spiders, err := modelSvc.GetSpiderList(bson.M{ + "project_id": bson.M{ + "$in": ids, + }, + }, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + for _, s := range spiders { + _, ok := cache[s.ProjectId] + if !ok { + HandleErrorInternalServerError(c, errors.ErrorControllerMissingInCache) + return + } + cache[s.ProjectId]++ + } + + // assign + var projects []models.Project + for _, d := range data { + p := d.(*models.Project) + p.Spiders = cache[p.Id] + projects = append(projects, *p) + } + + HandleSuccessWithListData(c, projects, total) +} + +func newProjectController() *projectController { + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + + ctr := NewListControllerDelegate(ControllerIdProject, modelSvc.GetBaseService(interfaces.ModelIdProject)) + + return &projectController{ + ListControllerDelegate: *ctr, + } +} diff --git a/core/controllers/result.go b/core/controllers/result.go new file mode 100644 index 000000000..346145fd0 --- /dev/null +++ b/core/controllers/result.go @@ -0,0 +1,150 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/result" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "net/http" +) + +var ResultController ActionController + +func getResultActions() []Action { + var resultCtx = newResultContext() + return []Action{ + { + Method: http.MethodGet, + Path: "/:id", + HandlerFunc: resultCtx.getList, + }, + } +} + +type resultContext struct { + modelSvc service.ModelService +} + +func (ctx *resultContext) getList(c *gin.Context) { + // data collection id + dcId, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // data source id + var dsId primitive.ObjectID + dsIdStr := c.Query("data_source_id") + if dsIdStr != "" { + dsId, err = primitive.ObjectIDFromHex(dsIdStr) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + } + + // data collection + dc, err := ctx.modelSvc.GetDataCollectionById(dcId) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // data source + ds, err := ctx.modelSvc.GetDataSourceById(dsId) + if err != nil { + if err.Error() == mongo2.ErrNoDocuments.Error() { + ds = &models.DataSource{} + } else { + HandleErrorInternalServerError(c, err) + return + } + } + + // spider + sq := bson.M{ + "col_id": dc.Id, + "data_source_id": ds.Id, + } + s, err := ctx.modelSvc.GetSpider(sq, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // service + svc, err := result.GetResultService(s.Id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // params + pagination := MustGetPagination(c) + query := ctx.getListQuery(c) + + // get results + data, err := svc.List(query, &generic.ListOptions{ + Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, + Skip: pagination.Size * (pagination.Page - 1), + Limit: pagination.Size, + }) + if err != nil { + if err.Error() == mongo2.ErrNoDocuments.Error() { + HandleSuccessWithListData(c, nil, 0) + return + } + HandleErrorInternalServerError(c, err) + return + } + + // validate results + if len(data) == 0 { + HandleSuccessWithListData(c, nil, 0) + return + } + + // total count + total, err := svc.Count(query) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // response + HandleSuccessWithListData(c, data, total) +} + +func (ctx *resultContext) getListQuery(c *gin.Context) (q generic.ListQuery) { + f, err := GetFilter(c) + if err != nil { + return q + } + for _, cond := range f.Conditions { + q = append(q, generic.ListQueryCondition{ + Key: cond.Key, + Op: cond.Op, + Value: utils.NormalizeObjectId(cond.Value), + }) + } + return q +} + +func newResultContext() *resultContext { + // context + ctx := &resultContext{} + + var err error + ctx.modelSvc, err = service.NewService() + if err != nil { + panic(err) + } + + return ctx +} diff --git a/core/controllers/result_v2.go b/core/controllers/result_v2.go new file mode 100644 index 000000000..edbeedcad --- /dev/null +++ b/core/controllers/result_v2.go @@ -0,0 +1,119 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/result" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" +) + +func GetResultList(c *gin.Context) { + // data collection id + dcId, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // data source id + var dsId primitive.ObjectID + dsIdStr := c.Query("data_source_id") + if dsIdStr != "" { + dsId, err = primitive.ObjectIDFromHex(dsIdStr) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + } + + // data collection + dc, err := service.NewModelServiceV2[models.DataCollectionV2]().GetById(dcId) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // data source + ds, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(dsId) + if err != nil { + if err.Error() == mongo2.ErrNoDocuments.Error() { + ds = &models.DataSourceV2{} + } else { + HandleErrorInternalServerError(c, err) + return + } + } + + // spider + sq := bson.M{ + "col_id": dc.Id, + "data_source_id": ds.Id, + } + s, err := service.NewModelServiceV2[models.SpiderV2]().GetOne(sq, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // service + svc, err := result.GetResultService(s.Id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // params + pagination := MustGetPagination(c) + query := getResultListQuery(c) + + // get results + data, err := svc.List(query, &generic.ListOptions{ + Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, + Skip: pagination.Size * (pagination.Page - 1), + Limit: pagination.Size, + }) + if err != nil { + if err.Error() == mongo2.ErrNoDocuments.Error() { + HandleSuccessWithListData(c, nil, 0) + return + } + HandleErrorInternalServerError(c, err) + return + } + + // validate results + if len(data) == 0 { + HandleSuccessWithListData(c, nil, 0) + return + } + + // total count + total, err := svc.Count(query) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // response + HandleSuccessWithListData(c, data, total) +} + +func getResultListQuery(c *gin.Context) (q generic.ListQuery) { + f, err := GetFilter(c) + if err != nil { + return q + } + for _, cond := range f.Conditions { + q = append(q, generic.ListQueryCondition{ + Key: cond.Key, + Op: cond.Op, + Value: utils.NormalizeObjectId(cond.Value), + }) + } + return q +} diff --git a/core/controllers/role.go b/core/controllers/role.go new file mode 100644 index 000000000..d4a286e54 --- /dev/null +++ b/core/controllers/role.go @@ -0,0 +1,3 @@ +package controllers + +var RoleController ListController diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go new file mode 100644 index 000000000..91e42ae8b --- /dev/null +++ b/core/controllers/router_v2.go @@ -0,0 +1,387 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/middlewares" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/gin-gonic/gin" + "net/http" +) + +type RouterGroups struct { + AuthGroup *gin.RouterGroup + AnonymousGroup *gin.RouterGroup +} + +func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) { + return &RouterGroups{ + AuthGroup: app.Group("/", middlewares.AuthorizationMiddlewareV2()), + AnonymousGroup: app.Group("/"), + } +} + +func RegisterController[T any](group *gin.RouterGroup, basePath string, ctr *BaseControllerV2[T]) { + actionPaths := make(map[string]bool) + for _, action := range ctr.actions { + group.Handle(action.Method, basePath+action.Path, action.HandlerFunc) + path := basePath + action.Path + key := action.Method + " - " + path + actionPaths[key] = true + } + registerBuiltinHandler(group, http.MethodGet, basePath+"", ctr.GetList, actionPaths) + registerBuiltinHandler(group, http.MethodGet, basePath+"/:id", ctr.GetById, actionPaths) + registerBuiltinHandler(group, http.MethodPost, basePath+"", ctr.Post, actionPaths) + registerBuiltinHandler(group, http.MethodPut, basePath+"/:id", ctr.PutById, actionPaths) + registerBuiltinHandler(group, http.MethodPatch, basePath+"", ctr.PatchList, actionPaths) + registerBuiltinHandler(group, http.MethodDelete, basePath+"/:id", ctr.DeleteById, actionPaths) + registerBuiltinHandler(group, http.MethodDelete, basePath+"", ctr.DeleteList, actionPaths) +} + +func RegisterActions(group *gin.RouterGroup, basePath string, actions []Action) { + for _, action := range actions { + group.Handle(action.Method, basePath+action.Path, action.HandlerFunc) + } +} + +func registerBuiltinHandler(group *gin.RouterGroup, method, path string, handlerFunc gin.HandlerFunc, existingActionPaths map[string]bool) { + key := method + " - " + path + _, ok := existingActionPaths[key] + if ok { + return + } + group.Handle(method, path, handlerFunc) +} + +func InitRoutes(app *gin.Engine) (err error) { + // routes groups + groups := NewRouterGroups(app) + + RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models.DataCollectionV2]()) + RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models.DataSourceV2]()) + RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]()) + RegisterController(groups.AuthGroup, "/gits", NewControllerV2[models.GitV2]()) + RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]()) + RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.SettingV2]()) + RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]()) + RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models.ProjectV2]()) + RegisterController(groups.AuthGroup, "/roles", NewControllerV2[models.RoleV2]()) + RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models.ScheduleV2]( + Action{ + Method: http.MethodPost, + Path: "", + HandlerFunc: PostSchedule, + }, + Action{ + Method: http.MethodPut, + Path: "/:id", + HandlerFunc: PutScheduleById, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/enable", + HandlerFunc: PostScheduleEnable, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/disable", + HandlerFunc: PostScheduleDisable, + }, + )) + RegisterController(groups.AuthGroup, "/spiders", NewControllerV2[models.SpiderV2]( + Action{ + Method: http.MethodGet, + Path: "/:id", + HandlerFunc: GetSpiderById, + }, + Action{ + Method: http.MethodGet, + Path: "", + HandlerFunc: GetSpiderList, + }, + Action{ + Method: http.MethodPost, + Path: "", + HandlerFunc: PostSpider, + }, + Action{ + Method: http.MethodPut, + Path: "/:id", + HandlerFunc: PutSpiderById, + }, + Action{ + Method: http.MethodDelete, + Path: "/:id", + HandlerFunc: DeleteSpiderById, + }, + Action{ + Method: http.MethodDelete, + Path: "", + HandlerFunc: DeleteSpiderList, + }, + Action{ + Method: http.MethodGet, + Path: "/:id/files/list", + HandlerFunc: GetSpiderListDir, + }, + Action{ + Method: http.MethodGet, + Path: "/:id/files/get", + HandlerFunc: GetSpiderFile, + }, + Action{ + Method: http.MethodGet, + Path: "/:id/files/info", + HandlerFunc: GetSpiderFileInfo, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/files/save", + HandlerFunc: PostSpiderSaveFile, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/files/save/batch", + HandlerFunc: PostSpiderSaveFiles, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/files/save/dir", + HandlerFunc: PostSpiderSaveDir, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/files/rename", + HandlerFunc: PostSpiderRenameFile, + }, + Action{ + Method: http.MethodDelete, + Path: "/:id/files", + HandlerFunc: DeleteSpiderFile, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/files/copy", + HandlerFunc: PostSpiderCopyFile, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/files/export", + HandlerFunc: PostSpiderExport, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/run", + HandlerFunc: PostSpiderRun, + }, + Action{ + Method: http.MethodGet, + Path: "/:id/git", + HandlerFunc: GetSpiderGit, + }, + Action{ + Method: http.MethodGet, + Path: "/:id/git/remote-refs", + HandlerFunc: GetSpiderGitRemoteRefs, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/git/checkout", + HandlerFunc: PostSpiderGitCheckout, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/git/pull", + HandlerFunc: PostSpiderGitPull, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/git/commit", + HandlerFunc: PostSpiderGitCommit, + }, + Action{ + Method: http.MethodGet, + Path: "/:id/data-source", + HandlerFunc: GetSpiderDataSource, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/data-source/:ds_id", + HandlerFunc: PostSpiderDataSource, + }, + )) + RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models.TaskV2]( + Action{ + Method: http.MethodGet, + Path: "/:id", + HandlerFunc: GetTaskById, + }, + Action{ + Method: http.MethodGet, + Path: "", + HandlerFunc: GetTaskList, + }, + Action{ + Method: http.MethodDelete, + Path: "/:id", + HandlerFunc: DeleteTaskById, + }, + Action{ + Method: http.MethodDelete, + Path: "", + HandlerFunc: DeleteList, + }, + Action{ + Method: http.MethodPost, + Path: "/run", + HandlerFunc: PostTaskRun, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/restart", + HandlerFunc: PostTaskRestart, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/cancel", + HandlerFunc: PostTaskCancel, + }, + Action{ + Method: http.MethodGet, + Path: "/:id/logs", + HandlerFunc: GetTaskLogs, + }, + Action{ + Method: http.MethodGet, + Path: "/:id/data", + HandlerFunc: GetTaskData, + }, + )) + RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models.TokenV2]( + Action{ + Method: http.MethodPost, + Path: "", + HandlerFunc: PostToken, + }, + )) + RegisterController(groups.AuthGroup, "/users", NewControllerV2[models.UserV2]( + Action{ + Method: http.MethodPost, + Path: "", + HandlerFunc: PostUser, + }, + Action{ + Method: http.MethodPost, + Path: "/:id/change-password", + HandlerFunc: PostUserChangePassword, + }, + Action{ + Method: http.MethodGet, + Path: "/me", + HandlerFunc: GetUserMe, + }, + Action{ + Method: http.MethodPut, + Path: "/me", + HandlerFunc: PutUserById, + }, + )) + + RegisterActions(groups.AuthGroup, "/results", []Action{ + { + Method: http.MethodGet, + Path: "/:id", + HandlerFunc: GetResultList, + }, + }) + RegisterActions(groups.AuthGroup, "/export", []Action{ + { + Method: http.MethodPost, + Path: "/:type", + HandlerFunc: PostExport, + }, + { + Method: http.MethodGet, + Path: "/:type/:id", + HandlerFunc: GetExport, + }, + { + Method: http.MethodGet, + Path: "/:type/:id/download", + HandlerFunc: GetExportDownload, + }, + }) + RegisterActions(groups.AuthGroup, "/filters", []Action{ + { + Method: http.MethodGet, + Path: "/:col", + HandlerFunc: GetFilterColFieldOptions, + }, + { + Method: http.MethodGet, + Path: "/:col/:value", + HandlerFunc: GetFilterColFieldOptions, + }, + { + Method: http.MethodGet, + Path: "/:col/:value/:label", + HandlerFunc: GetFilterColFieldOptions, + }, + }) + RegisterActions(groups.AuthGroup, "/settings", []Action{ + { + Method: http.MethodGet, + Path: "/:id", + HandlerFunc: GetSetting, + }, + { + Method: http.MethodPut, + Path: "/:id", + HandlerFunc: PutSetting, + }, + }) + RegisterActions(groups.AuthGroup, "/stats", []Action{ + { + Method: http.MethodGet, + Path: "/overview", + HandlerFunc: GetStatsOverview, + }, + { + Method: http.MethodGet, + Path: "/daily", + HandlerFunc: GetStatsDaily, + }, + { + Method: http.MethodGet, + Path: "/tasks", + HandlerFunc: GetStatsTasks, + }, + }) + + RegisterActions(groups.AnonymousGroup, "/system-info", []Action{ + { + Path: "", + Method: http.MethodGet, + HandlerFunc: GetSystemInfo, + }, + }) + RegisterActions(groups.AnonymousGroup, "/version", []Action{ + { + Method: http.MethodGet, + Path: "", + HandlerFunc: GetVersion, + }, + }) + RegisterActions(groups.AnonymousGroup, "/", []Action{ + { + Method: http.MethodPost, + Path: "/login", + HandlerFunc: PostLogin, + }, + { + Method: http.MethodPost, + Path: "/logout", + HandlerFunc: PostLogout, + }, + }) + + return nil +} diff --git a/core/controllers/router_v2_test.go b/core/controllers/router_v2_test.go new file mode 100644 index 000000000..47642324d --- /dev/null +++ b/core/controllers/router_v2_test.go @@ -0,0 +1,91 @@ +package controllers_test + +import ( + "github.com/crawlab-team/crawlab/core/controllers" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestRouterGroups(t *testing.T) { + router := gin.Default() + groups := controllers.NewRouterGroups(router) + + assertions := []struct { + group *gin.RouterGroup + name string + }{ + {groups.AuthGroup, "AuthGroup"}, + {groups.AnonymousGroup, "AnonymousGroup"}, + } + + for _, a := range assertions { + assert.NotNil(t, a.group, a.name+" should not be nil") + } +} + +func TestRegisterController_Routes(t *testing.T) { + router := gin.Default() + groups := controllers.NewRouterGroups(router) + ctr := controllers.NewControllerV2[models.TestModel]() + basePath := "/testmodels" + + controllers.RegisterController(groups.AuthGroup, basePath, ctr) + + // Check if all routes are registered + routes := router.Routes() + + var methodPaths []string + for _, route := range routes { + methodPaths = append(methodPaths, route.Method+" - "+route.Path) + } + + expectedRoutes := []gin.RouteInfo{ + {Method: "GET", Path: basePath}, + {Method: "GET", Path: basePath + "/:id"}, + {Method: "POST", Path: basePath}, + {Method: "PUT", Path: basePath + "/:id"}, + {Method: "PATCH", Path: basePath}, + {Method: "DELETE", Path: basePath + "/:id"}, + {Method: "DELETE", Path: basePath}, + } + + assert.Equal(t, len(expectedRoutes), len(routes)) + for _, route := range expectedRoutes { + assert.Contains(t, methodPaths, route.Method+" - "+route.Path) + } +} + +func TestInitRoutes_ProjectsRoute(t *testing.T) { + router := gin.Default() + + controllers.InitRoutes(router) + + // Check if the projects route is registered + routes := router.Routes() + + var methodPaths []string + for _, route := range routes { + methodPaths = append(methodPaths, route.Method+" - "+route.Path) + } + + expectedRoutes := []gin.RouteInfo{ + {Method: "GET", Path: "/projects"}, + {Method: "GET", Path: "/projects/:id"}, + {Method: "POST", Path: "/projects"}, + {Method: "PUT", Path: "/projects/:id"}, + {Method: "PATCH", Path: "/projects"}, + {Method: "DELETE", Path: "/projects/:id"}, + {Method: "DELETE", Path: "/projects"}, + } + + for _, route := range expectedRoutes { + assert.Contains(t, methodPaths, route.Method+" - "+route.Path) + } +} + +func TestMain(m *testing.M) { + gin.SetMode(gin.TestMode) + m.Run() +} diff --git a/core/controllers/schedule.go b/core/controllers/schedule.go new file mode 100644 index 000000000..71ba2b670 --- /dev/null +++ b/core/controllers/schedule.go @@ -0,0 +1,221 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "net/http" +) + +var ScheduleController *scheduleController + +func getScheduleActions() []Action { + scheduleCtx := newScheduleContext() + return []Action{ + { + Method: http.MethodPost, + Path: "/:id/enable", + HandlerFunc: scheduleCtx.enable, + }, + { + Method: http.MethodPost, + Path: "/:id/disable", + HandlerFunc: scheduleCtx.disable, + }, + } +} + +type scheduleController struct { + ListActionControllerDelegate + d ListActionControllerDelegate + ctx *scheduleContext +} + +func (ctr *scheduleController) Post(c *gin.Context) { + var s models.Schedule + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorBadRequest(c, err) + return + } + if err := delegate.NewModelDelegate(&s, GetUserFromContext(c)).Add(); err != nil { + HandleErrorInternalServerError(c, err) + return + } + if s.Enabled { + if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + HandleSuccessWithData(c, s) +} + +func (ctr *scheduleController) Put(c *gin.Context) { + id := c.Param("id") + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + var s models.Schedule + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorBadRequest(c, err) + return + } + if s.GetId() != oid { + HandleErrorBadRequest(c, errors.ErrorHttpBadRequest) + return + } + if err := delegate.NewModelDelegate(&s).Save(); err != nil { + HandleErrorInternalServerError(c, err) + return + } + if s.Enabled { + if err := ctr.ctx.scheduleSvc.Disable(&s, GetUserFromContext(c)); err != nil { + HandleErrorInternalServerError(c, err) + return + } + if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + HandleSuccessWithData(c, s) +} + +func (ctr *scheduleController) Delete(c *gin.Context) { + id := c.Param("id") + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + s, err := ctr.ctx.modelSvc.GetScheduleById(oid) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + if err := ctr.ctx.scheduleSvc.Disable(s); err != nil { + HandleErrorInternalServerError(c, err) + return + } + if err := delegate.NewModelDelegate(s, GetUserFromContext(c)).Delete(); err != nil { + HandleErrorInternalServerError(c, err) + return + } +} + +func (ctr *scheduleController) DeleteList(c *gin.Context) { + payload, err := NewJsonBinder(interfaces.ModelIdSchedule).BindBatchRequestPayload(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + for _, id := range payload.Ids { + s, err := ctr.ctx.modelSvc.GetScheduleById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + if err := ctr.ctx.scheduleSvc.Disable(s); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdSchedule).DeleteList(bson.M{ + "_id": bson.M{ + "$in": payload.Ids, + }, + }); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +func (ctx *scheduleContext) enable(c *gin.Context) { + s, err := ctx._getSchedule(c) + if err != nil { + return + } + if err := ctx.scheduleSvc.Enable(s, GetUserFromContext(c)); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +func (ctx *scheduleContext) disable(c *gin.Context) { + s, err := ctx._getSchedule(c) + if err != nil { + return + } + if err := ctx.scheduleSvc.Disable(s, GetUserFromContext(c)); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +func (ctx *scheduleContext) _getSchedule(c *gin.Context) (s *models.Schedule, err error) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + s, err = ctx.modelSvc.GetScheduleById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + return s, nil +} + +type scheduleContext struct { + modelSvc service.ModelService + scheduleSvc interfaces.ScheduleService +} + +func newScheduleContext() *scheduleContext { + // context + ctx := &scheduleContext{} + + // dependency injection + if err := container.GetContainer().Invoke(func( + modelSvc service.ModelService, + scheduleSvc interfaces.ScheduleService, + ) { + ctx.modelSvc = modelSvc + ctx.scheduleSvc = scheduleSvc + }); err != nil { + panic(err) + } + + return ctx +} + +func newScheduleController() *scheduleController { + actions := getScheduleActions() + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + + ctr := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions) + d := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions) + ctx := newScheduleContext() + + return &scheduleController{ + ListActionControllerDelegate: *ctr, + d: *d, + ctx: ctx, + } +} diff --git a/core/controllers/schedule_v2.go b/core/controllers/schedule_v2.go new file mode 100644 index 000000000..3e19da769 --- /dev/null +++ b/core/controllers/schedule_v2.go @@ -0,0 +1,130 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/schedule" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func PostSchedule(c *gin.Context) { + var s models.ScheduleV2 + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorBadRequest(c, err) + return + } + + u := GetUserFromContextV2(c) + + modelSvc := service.NewModelServiceV2[models.ScheduleV2]() + + s.SetCreated(u.Id) + s.SetUpdated(u.Id) + id, err := modelSvc.InsertOne(s) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + s.Id = id + + if s.Enabled { + scheduleSvc, err := schedule.GetScheduleServiceV2() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + if err := scheduleSvc.Enable(s, u.Id); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + + HandleSuccessWithData(c, s) +} + +func PutScheduleById(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + var s models.ScheduleV2 + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorBadRequest(c, err) + return + } + if s.Id != id { + HandleErrorBadRequest(c, errors.ErrorHttpBadRequest) + return + } + + modelSvc := service.NewModelServiceV2[models.ScheduleV2]() + err = modelSvc.ReplaceById(id, s) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + scheduleSvc, err := schedule.GetScheduleServiceV2() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + u := GetUserFromContextV2(c) + + if s.Enabled { + if err := scheduleSvc.Enable(s, u.Id); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } else { + if err := scheduleSvc.Disable(s, u.Id); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + + HandleSuccessWithData(c, s) +} + +func PostScheduleEnable(c *gin.Context) { + postScheduleEnableDisableFunc(true)(c) +} + +func PostScheduleDisable(c *gin.Context) { + postScheduleEnableDisableFunc(false)(c) +} + +func postScheduleEnableDisableFunc(isEnable bool) func(c *gin.Context) { + return func(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + svc, err := schedule.GetScheduleServiceV2() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + s, err := service.NewModelServiceV2[models.ScheduleV2]().GetById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + u := GetUserFromContextV2(c) + if isEnable { + err = svc.Enable(*s, u.Id) + } else { + err = svc.Disable(*s, u.Id) + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) + } +} diff --git a/core/controllers/setting.go b/core/controllers/setting.go new file mode 100644 index 000000000..068e23796 --- /dev/null +++ b/core/controllers/setting.go @@ -0,0 +1,84 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/gin-gonic/gin" +) + +var SettingController *settingController + +type settingController struct { + ListControllerDelegate +} + +func (ctr *settingController) Get(c *gin.Context) { + // key + key := c.Param("id") + + // model service + modelSvc, err := service.NewService() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // setting + s, err := modelSvc.GetSettingByKey(key, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, s) +} + +func (ctr *settingController) Put(c *gin.Context) { + // key + key := c.Param("id") + + // settings + var s models.Setting + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // model service + modelSvc, err := service.NewService() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // setting + _s, err := modelSvc.GetSettingByKey(key, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // save + _s.Value = s.Value + if err := delegate.NewModelDelegate(_s).Save(); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func newSettingController() *settingController { + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + + ctr := NewListControllerDelegate(ControllerIdSetting, modelSvc.GetBaseService(interfaces.ModelIdSetting)) + + return &settingController{ + ListControllerDelegate: *ctr, + } +} diff --git a/core/controllers/setting_v2.go b/core/controllers/setting_v2.go new file mode 100644 index 000000000..326873b99 --- /dev/null +++ b/core/controllers/setting_v2.go @@ -0,0 +1,56 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" +) + +func GetSetting(c *gin.Context) { + // key + key := c.Param("id") + + // setting + s, err := service.NewModelServiceV2[models.SettingV2]().GetOne(bson.M{"key": key}, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, s) +} + +func PutSetting(c *gin.Context) { + // key + key := c.Param("id") + + // settings + var s models.Setting + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + modelSvc := service.NewModelServiceV2[models.SettingV2]() + + // setting + _s, err := modelSvc.GetOne(bson.M{"key": key}, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + u := GetUserFromContextV2(c) + + // save + _s.Value = s.Value + _s.SetUpdated(u.Id) + err = modelSvc.ReplaceOne(bson.M{"key": key}, *_s) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} diff --git a/core/controllers/spider.go b/core/controllers/spider.go new file mode 100644 index 000000000..e76ba4d29 --- /dev/null +++ b/core/controllers/spider.go @@ -0,0 +1,1333 @@ +package controllers + +import ( + "bytes" + "fmt" + "github.com/crawlab-team/crawlab-db/mongo" + vcs "github.com/crawlab-team/crawlab-vcs" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + fs2 "github.com/crawlab-team/crawlab/core/fs" + "github.com/crawlab-team/crawlab/core/interfaces" + delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "io" + "math" + "net/http" + "os" + "path/filepath" + "strings" +) + +var SpiderController *spiderController + +func getSpiderActions() []Action { + ctx := newSpiderContext() + return []Action{ + { + Method: http.MethodGet, + Path: "/:id/files/list", + HandlerFunc: ctx.listDir, + }, + { + Method: http.MethodGet, + Path: "/:id/files/get", + HandlerFunc: ctx.getFile, + }, + { + Method: http.MethodGet, + Path: "/:id/files/info", + HandlerFunc: ctx.getFileInfo, + }, + { + Method: http.MethodPost, + Path: "/:id/files/save", + HandlerFunc: ctx.saveFile, + }, + { + Method: http.MethodPost, + Path: "/:id/files/save/dir", + HandlerFunc: ctx.saveDir, + }, + { + Method: http.MethodPost, + Path: "/:id/files/rename", + HandlerFunc: ctx.renameFile, + }, + { + Method: http.MethodPost, + Path: "/:id/files/delete", + HandlerFunc: ctx.deleteFile, + }, + { + Method: http.MethodPost, + Path: "/:id/files/copy", + HandlerFunc: ctx.copyFile, + }, + { + Path: "/:id/files/export", + Method: http.MethodPost, + HandlerFunc: ctx.postExport, + }, + { + Method: http.MethodPost, + Path: "/:id/run", + HandlerFunc: ctx.run, + }, + { + Method: http.MethodGet, + Path: "/:id/git", + HandlerFunc: ctx.getGit, + }, + { + Method: http.MethodGet, + Path: "/:id/git/remote-refs", + HandlerFunc: ctx.getGitRemoteRefs, + }, + { + Method: http.MethodPost, + Path: "/:id/git/checkout", + HandlerFunc: ctx.gitCheckout, + }, + { + Method: http.MethodPost, + Path: "/:id/git/pull", + HandlerFunc: ctx.gitPull, + }, + { + Method: http.MethodPost, + Path: "/:id/git/commit", + HandlerFunc: ctx.gitCommit, + }, + //{ + // Method: http.MethodPost, + // Path: "/:id/clone", + // HandlerFunc: ctx.clone, + //}, + { + Path: "/:id/data-source", + Method: http.MethodGet, + HandlerFunc: ctx.getDataSource, + }, + { + Path: "/:id/data-source/:ds_id", + Method: http.MethodPost, + HandlerFunc: ctx.postDataSource, + }, + } +} + +type spiderController struct { + ListActionControllerDelegate + d ListActionControllerDelegate + ctx *spiderContext +} + +func (ctr *spiderController) Get(c *gin.Context) { + ctr.ctx._get(c) +} + +func (ctr *spiderController) Post(c *gin.Context) { + s, err := ctr.ctx._post(c) + if err != nil { + return + } + HandleSuccessWithData(c, s) +} + +func (ctr *spiderController) Put(c *gin.Context) { + s, err := ctr.ctx._put(c) + if err != nil { + return + } + HandleSuccessWithData(c, s) +} + +func (ctr *spiderController) Delete(c *gin.Context) { + if err := ctr.ctx._delete(c); err != nil { + return + } + HandleSuccess(c) +} + +func (ctr *spiderController) GetList(c *gin.Context) { + withStats := c.Query("stats") + if withStats == "" { + ctr.d.GetList(c) + return + } + ctr.ctx._getListWithStats(c) +} + +func (ctr *spiderController) DeleteList(c *gin.Context) { + if err := ctr.ctx._deleteList(c); err != nil { + return + } + HandleSuccess(c) +} + +type spiderContext struct { + modelSvc service.ModelService + modelSpiderSvc interfaces.ModelBaseService + modelSpiderStatSvc interfaces.ModelBaseService + modelTaskSvc interfaces.ModelBaseService + modelTaskStatSvc interfaces.ModelBaseService + adminSvc interfaces.SpiderAdminService +} + +func (ctx *spiderContext) listDir(c *gin.Context) { + _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodGet) + if err != nil { + return + } + + files, err := fsSvc.List(payload.Path) + if err != nil { + if err.Error() != "response status code: 404" { + HandleErrorInternalServerError(c, err) + return + } + } + + HandleSuccessWithData(c, files) +} + +func (ctx *spiderContext) getFile(c *gin.Context) { + _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodGet) + if err != nil { + return + } + + data, err := fsSvc.GetFile(payload.Path) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + data = utils.TrimFileData(data) + + HandleSuccessWithData(c, string(data)) +} + +func (ctx *spiderContext) getFileInfo(c *gin.Context) { + _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodGet) + if err != nil { + return + } + + info, err := fsSvc.GetFileInfo(payload.Path) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, info) +} + +func (ctx *spiderContext) saveFile(c *gin.Context) { + _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) + if err != nil { + return + } + + if err := fsSvc.Save(payload.Path, []byte(payload.Data)); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func (ctx *spiderContext) saveDir(c *gin.Context) { + _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) + if err != nil { + return + } + + if err := fsSvc.CreateDir(payload.Path); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func (ctx *spiderContext) renameFile(c *gin.Context) { + _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) + if err != nil { + return + } + + if err := fsSvc.Rename(payload.Path, payload.NewPath); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func (ctx *spiderContext) deleteFile(c *gin.Context) { + _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) + if err != nil { + return + } + + if err := fsSvc.Delete(payload.Path); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func (ctx *spiderContext) copyFile(c *gin.Context) { + _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) + if err != nil { + return + } + + if err := fsSvc.Copy(payload.Path, payload.NewPath); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func (ctx *spiderContext) run(c *gin.Context) { + // spider id + id, err := ctx._processActionRequest(c) + if err != nil { + return + } + + // options + var opts interfaces.SpiderRunOptions + if err := c.ShouldBindJSON(&opts); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // user + if u := GetUserFromContext(c); u != nil { + opts.UserId = u.GetId() + } + + // schedule + taskIds, err := ctx.adminSvc.Schedule(id, &opts) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, taskIds) +} + +func (ctx *spiderContext) getGit(c *gin.Context) { + // spider id + id, err := ctx._processActionRequest(c) + if err != nil { + return + } + + // git client + gitClient, err := ctx._getGitClient(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // return null if git client is empty + if gitClient == nil { + HandleSuccess(c) + return + } + + // current branch + currentBranch, err := gitClient.GetCurrentBranch() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // branches + branches, err := gitClient.GetBranches() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + if branches == nil || len(branches) == 0 && currentBranch != "" { + branches = []vcs.GitRef{{Name: currentBranch}} + } + + // changes + changes, err := gitClient.GetStatus() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // logs + logs, err := gitClient.GetLogsWithRefs() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // ignore + ignore, err := ctx._getGitIgnore(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // git + _git, err := ctx.modelSvc.GetGitById(id) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + HandleErrorInternalServerError(c, err) + return + } + } + + // response + res := bson.M{ + "current_branch": currentBranch, + "branches": branches, + "changes": changes, + "logs": logs, + "ignore": ignore, + "git": _git, + } + + HandleSuccessWithData(c, res) +} + +func (ctx *spiderContext) getGitRemoteRefs(c *gin.Context) { + // spider id + id, err := ctx._processActionRequest(c) + if err != nil { + return + } + + // remote name + remoteName := c.Query("remote") + if remoteName == "" { + remoteName = vcs.GitRemoteNameOrigin + } + + // git client + gitClient, err := ctx._getGitClient(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // return null if git client is empty + if gitClient == nil { + HandleSuccess(c) + return + } + + // refs + refs, err := gitClient.GetRemoteRefs(remoteName) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, refs) +} + +func (ctx *spiderContext) gitCheckout(c *gin.Context) { + // payload + var payload entity.GitPayload + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // spider id + id, err := ctx._processActionRequest(c) + if err != nil { + return + } + + // git client + gitClient, err := ctx._getGitClient(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // return null if git client is empty + if gitClient == nil { + HandleSuccess(c) + return + } + + // branch to pull + var branch string + if payload.Branch == "" { + // by default current branch + branch, err = gitClient.GetCurrentBranch() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + } else { + // payload branch + branch = payload.Branch + } + + // checkout + if err := ctx._gitCheckout(gitClient, constants.GitRemoteNameOrigin, branch); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func (ctx *spiderContext) gitPull(c *gin.Context) { + // payload + var payload entity.GitPayload + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // spider id + id, err := ctx._processActionRequest(c) + if err != nil { + return + } + + // git + g, err := ctx.modelSvc.GetGitById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // attempt to sync git + _ = ctx.adminSvc.SyncGitOne(g) + + HandleSuccess(c) +} + +func (ctx *spiderContext) gitCommit(c *gin.Context) { + // payload + var payload entity.GitPayload + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // spider id + id, err := ctx._processActionRequest(c) + if err != nil { + return + } + + // git client + gitClient, err := ctx._getGitClient(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // return null if git client is empty + if gitClient == nil { + HandleSuccess(c) + return + } + + // add + for _, p := range payload.Paths { + if err := gitClient.Add(p); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + + // commit + if err := gitClient.Commit(payload.CommitMessage); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // push + if err := gitClient.Push( + vcs.WithRemoteNamePush(vcs.GitRemoteNameOrigin), + ); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func (ctx *spiderContext) getDataSource(c *gin.Context) { + // spider id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // spider + s, err := ctx.modelSvc.GetSpiderById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // data source + ds, err := ctx.modelSvc.GetDataSourceById(s.DataSourceId) + if err != nil { + if err.Error() == mongo2.ErrNoDocuments.Error() { + HandleSuccess(c) + return + } + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, ds) +} + +func (ctx *spiderContext) postDataSource(c *gin.Context) { + // spider id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // data source id + dsId, err := primitive.ObjectIDFromHex(c.Param("ds_id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // spider + s, err := ctx.modelSvc.GetSpiderById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // data source + if !dsId.IsZero() { + _, err = ctx.modelSvc.GetDataSourceById(dsId) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + + // save data source id + s.DataSourceId = dsId + if err := delegate2.NewModelDelegate(s).Save(); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func (ctx *spiderContext) postExport(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // zip file path + zipFilePath, err := ctx.adminSvc.Export(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // download + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath)) + c.File(zipFilePath) +} + +func (ctx *spiderContext) _get(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + s, err := ctx.modelSvc.GetSpiderById(id) + if err == mongo2.ErrNoDocuments { + HandleErrorNotFound(c, err) + return + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // stat + s.Stat, err = ctx.modelSvc.GetSpiderStatById(s.GetId()) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // data collection + if !s.ColId.IsZero() { + col, err := ctx.modelSvc.GetDataCollectionById(s.ColId) + if err != nil { + if err != mongo2.ErrNoDocuments { + HandleErrorInternalServerError(c, err) + return + } + } else { + s.ColName = col.Name + } + } + + HandleSuccessWithData(c, s) +} + +func (ctx *spiderContext) _post(c *gin.Context) (s *models.Spider, err error) { + // bind + s = &models.Spider{} + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorBadRequest(c, err) + return nil, err + } + + // upsert data collection + if err := ctx._upsertDataCollection(c, s); err != nil { + HandleErrorInternalServerError(c, err) + return nil, err + } + + // add + if err := delegate2.NewModelDelegate(s, GetUserFromContext(c)).Add(); err != nil { + HandleErrorInternalServerError(c, err) + return nil, err + } + + // add stat + st := &models.SpiderStat{ + Id: s.GetId(), + } + if err := delegate2.NewModelDelegate(st, GetUserFromContext(c)).Add(); err != nil { + HandleErrorInternalServerError(c, err) + return nil, err + } + + return s, nil +} + +func (ctx *spiderContext) _put(c *gin.Context) (s *models.Spider, err error) { + // bind + s = &models.Spider{} + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorBadRequest(c, err) + return nil, err + } + + // upsert data collection + if err := ctx._upsertDataCollection(c, s); err != nil { + HandleErrorInternalServerError(c, err) + return nil, err + } + + // save + if err := delegate2.NewModelDelegate(s, GetUserFromContext(c)).Save(); err != nil { + HandleErrorInternalServerError(c, err) + return nil, err + } + + return s, nil +} + +func (ctx *spiderContext) _delete(c *gin.Context) (err error) { + id := c.Param("id") + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { + // delete spider + s, err := ctx.modelSvc.GetSpiderById(oid) + if err != nil { + return err + } + if err := delegate2.NewModelDelegate(s, GetUserFromContext(c)).Delete(); err != nil { + return err + } + + // delete spider stat + ss, err := ctx.modelSvc.GetSpiderStatById(oid) + if err != nil { + return err + } + if err := delegate2.NewModelDelegate(ss, GetUserFromContext(c)).Delete(); err != nil { + return err + } + + // related tasks + tasks, err := ctx.modelSvc.GetTaskList(bson.M{"spider_id": oid}, nil) + if err != nil { + return err + } + + // task ids + var taskIds []primitive.ObjectID + for _, t := range tasks { + taskIds = append(taskIds, t.Id) + } + + // delete related tasks + if err := ctx.modelTaskSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { + return err + } + + // delete related task stats + if err := ctx.modelTaskStatSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { + return err + } + + return nil + }); err != nil { + HandleErrorInternalServerError(c, err) + return err + } + + return nil +} + +func (ctx *spiderContext) _getListWithStats(c *gin.Context) { + // params + pagination := MustGetPagination(c) + query := MustGetFilterQuery(c) + sort := MustGetSortOption(c) + + // get list + l, err := ctx.modelSpiderSvc.GetList(query, &mongo.FindOptions{ + Sort: sort, + Skip: pagination.Size * (pagination.Page - 1), + Limit: pagination.Size, + }) + if err != nil { + if err.Error() == mongo2.ErrNoDocuments.Error() { + HandleErrorNotFound(c, err) + } else { + HandleErrorInternalServerError(c, err) + } + return + } + + // check empty list + if len(l.GetModels()) == 0 { + HandleSuccessWithListData(c, nil, 0) + return + } + + // ids + var ids []primitive.ObjectID + for _, d := range l.GetModels() { + s := d.(*models.Spider) + ids = append(ids, s.GetId()) + } + + // total count + total, err := ctx.modelSpiderSvc.Count(query) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // stat list + query = bson.M{ + "_id": bson.M{ + "$in": ids, + }, + } + stats, err := ctx.modelSvc.GetSpiderStatList(query, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // cache stat list to dict + dict := map[primitive.ObjectID]models.SpiderStat{} + var tids []primitive.ObjectID + for _, st := range stats { + if st.Tasks > 0 { + taskCount := int64(st.Tasks) + st.AverageWaitDuration = int64(math.Round(float64(st.WaitDuration) / float64(taskCount))) + st.AverageRuntimeDuration = int64(math.Round(float64(st.RuntimeDuration) / float64(taskCount))) + st.AverageTotalDuration = int64(math.Round(float64(st.TotalDuration) / float64(taskCount))) + } + dict[st.GetId()] = st + + if !st.LastTaskId.IsZero() { + tids = append(tids, st.LastTaskId) + } + } + + // task list and stats + var tasks []models.Task + dictTask := map[primitive.ObjectID]models.Task{} + dictTaskStat := map[primitive.ObjectID]models.TaskStat{} + if len(tids) > 0 { + // task list + queryTask := bson.M{ + "_id": bson.M{ + "$in": tids, + }, + } + tasks, err = ctx.modelSvc.GetTaskList(queryTask, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // task stats list + taskStats, err := ctx.modelSvc.GetTaskStatList(queryTask, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // cache task stats to dict + for _, st := range taskStats { + dictTaskStat[st.GetId()] = st + } + + // cache task list to dict + for _, t := range tasks { + st, ok := dictTaskStat[t.GetId()] + if ok { + t.Stat = &st + } + dictTask[t.GetSpiderId()] = t + } + } + + // iterate list again + var data []interface{} + for _, d := range l.GetModels() { + s := d.(*models.Spider) + + // spider stat + st, ok := dict[s.GetId()] + if ok { + s.Stat = &st + + // last task + t, ok := dictTask[s.GetId()] + if ok { + s.Stat.LastTask = &t + } + } + + // add to list + data = append(data, *s) + } + + // response + HandleSuccessWithListData(c, data, total) +} + +func (ctx *spiderContext) _deleteList(c *gin.Context) (err error) { + payload, err := NewJsonBinder(ControllerIdSpider).BindBatchRequestPayload(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { + // delete spiders + if err := ctx.modelSpiderSvc.DeleteList(bson.M{ + "_id": bson.M{ + "$in": payload.Ids, + }, + }); err != nil { + return err + } + + // delete spider stats + if err := ctx.modelSpiderStatSvc.DeleteList(bson.M{ + "_id": bson.M{ + "$in": payload.Ids, + }, + }); err != nil { + return err + } + + // related tasks + tasks, err := ctx.modelSvc.GetTaskList(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil) + if err != nil { + return err + } + + // task ids + var taskIds []primitive.ObjectID + for _, t := range tasks { + taskIds = append(taskIds, t.Id) + } + + // delete related tasks + if err := ctx.modelTaskSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { + return err + } + + // delete related task stats + if err := ctx.modelTaskStatSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { + return err + } + + return nil + }); err != nil { + HandleErrorInternalServerError(c, err) + return err + } + + return nil +} + +func (ctx *spiderContext) _processFileRequest(c *gin.Context, method string) (id primitive.ObjectID, payload entity.FileRequestPayload, fsSvc interfaces.FsServiceV2, err error) { + // id + id, err = primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // payload + contentType := c.GetHeader("Content-Type") + if strings.HasPrefix(contentType, "multipart/form-data") { + // multipart/form-data + payload, err = ctx._getFileRequestMultipartPayload(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + } else { + // query or application/json + switch method { + case http.MethodGet: + err = c.ShouldBindQuery(&payload) + default: + err = c.ShouldBindJSON(&payload) + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + + // fs service + workspacePath := viper.GetString("workspace") + fsSvc = fs2.NewFsServiceV2(filepath.Join(workspacePath, id.Hex())) + + return +} + +func (ctx *spiderContext) _getFileRequestMultipartPayload(c *gin.Context) (payload entity.FileRequestPayload, err error) { + fh, err := c.FormFile("file") + if err != nil { + return + } + f, err := fh.Open() + if err != nil { + return + } + buf := bytes.NewBuffer(nil) + if _, err = io.Copy(buf, f); err != nil { + return + } + payload.Path = c.PostForm("path") + payload.Data = buf.String() + return +} + +func (ctx *spiderContext) _processActionRequest(c *gin.Context) (id primitive.ObjectID, err error) { + // id + id, err = primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + return +} + +func (ctx *spiderContext) _upsertDataCollection(c *gin.Context, s *models.Spider) (err error) { + if s.ColId.IsZero() { + // validate + if s.ColName == "" { + return trace.TraceError(errors.ErrorControllerMissingRequestFields) + } + // no id + dc, err := ctx.modelSvc.GetDataCollectionByName(s.ColName, nil) + if err != nil { + if err == mongo2.ErrNoDocuments { + // not exists, add new + dc = &models.DataCollection{Name: s.ColName} + if err := delegate2.NewModelDelegate(dc, GetUserFromContext(c)).Add(); err != nil { + return err + } + } else { + // error + return err + } + } + s.ColId = dc.Id + + // create index + _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.TaskKey: 1}}) + _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.HashKey: 1}}) + } else { + // with id + dc, err := ctx.modelSvc.GetDataCollectionById(s.ColId) + if err != nil { + return err + } + s.ColId = dc.Id + } + return nil +} + +func (ctx *spiderContext) _getGitIgnore(id primitive.ObjectID) (ignore []string, err error) { + workspacePath := viper.GetString("workspace") + filePath := filepath.Join(workspacePath, id.Hex(), ".gitignore") + if !utils.Exists(filePath) { + return nil, nil + } + data, err := os.ReadFile(filePath) + if err != nil { + return nil, trace.TraceError(err) + } + ignore = strings.Split(string(data), "\n") + return ignore, nil +} + +func (ctx *spiderContext) _gitCheckout(gitClient *vcs.GitClient, remote, branch string) (err error) { + if err := gitClient.CheckoutBranch(branch, vcs.WithBranch(branch)); err != nil { + return trace.TraceError(err) + } + + // pull + return ctx._gitPull(gitClient, remote, branch) +} + +func (ctx *spiderContext) _gitPull(gitClient *vcs.GitClient, remote, branch string) (err error) { + // pull + if err := gitClient.Pull( + vcs.WithRemoteNamePull(remote), + vcs.WithBranchNamePull(branch), + ); err != nil { + return trace.TraceError(err) + } + + // reset + if err := gitClient.Reset(); err != nil { + return trace.TraceError(err) + } + + return nil +} + +func (ctx *spiderContext) _getGitClient(id primitive.ObjectID) (gitClient *vcs.GitClient, err error) { + // git + g, err := ctx.modelSvc.GetGitById(id) + if err != nil { + if err != mongo2.ErrNoDocuments { + return nil, trace.TraceError(err) + } + return nil, nil + } + + // git client + workspacePath := viper.GetString("workspace") + gitClient, err = vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, id.Hex()))) + if err != nil { + return nil, err + } + + // set auth + utils.InitGitClientAuth(g, gitClient) + + // remote name + remoteName := vcs.GitRemoteNameOrigin + + // update remote + r, err := gitClient.GetRemote(remoteName) + if err == git.ErrRemoteNotFound { + // remote not exists, create + if _, err := gitClient.CreateRemote(&config.RemoteConfig{ + Name: remoteName, + URLs: []string{g.Url}, + }); err != nil { + return nil, trace.TraceError(err) + } + } else if err == nil { + // remote exists, update if different + if g.Url != r.Config().URLs[0] { + if err := gitClient.DeleteRemote(remoteName); err != nil { + return nil, trace.TraceError(err) + } + if _, err := gitClient.CreateRemote(&config.RemoteConfig{ + Name: remoteName, + URLs: []string{g.Url}, + }); err != nil { + return nil, trace.TraceError(err) + } + } + gitClient.SetRemoteUrl(g.Url) + } else { + // error + return nil, trace.TraceError(err) + } + + // check if head reference exists + _, err = gitClient.GetRepository().Head() + if err == nil { + return gitClient, nil + } + + // align master/main branch + ctx._alignBranch(gitClient) + + return gitClient, nil +} + +func (ctx *spiderContext) _alignBranch(gitClient *vcs.GitClient) { + // current branch + currentBranch, err := gitClient.GetCurrentBranch() + if err != nil { + trace.PrintError(err) + return + } + + // skip if current branch is not master + if currentBranch != vcs.GitBranchNameMaster { + return + } + + // remote refs + refs, err := gitClient.GetRemoteRefs(vcs.GitRemoteNameOrigin) + if err != nil { + trace.PrintError(err) + return + } + + // main branch + defaultRemoteBranch, err := ctx._getDefaultRemoteBranch(refs) + if err != nil || defaultRemoteBranch == "" { + return + } + + // move branch + if err := gitClient.MoveBranch(vcs.GitBranchNameMaster, defaultRemoteBranch); err != nil { + trace.PrintError(err) + } +} + +func (ctx *spiderContext) _getDefaultRemoteBranch(refs []vcs.GitRef) (defaultRemoteBranchName string, err error) { + // remote branch name + for _, r := range refs { + if r.Type != vcs.GitRefTypeBranch { + continue + } + + if r.Name == vcs.GitBranchNameMain { + defaultRemoteBranchName = r.Name + break + } + + if r.Name == vcs.GitBranchNameMaster { + defaultRemoteBranchName = r.Name + continue + } + + if defaultRemoteBranchName == "" { + defaultRemoteBranchName = r.Name + continue + } + } + + return defaultRemoteBranchName, nil +} + +var _spiderCtx *spiderContext + +func newSpiderContext() *spiderContext { + if _spiderCtx != nil { + return _spiderCtx + } + + // context + ctx := &spiderContext{} + + // dependency injection + if err := container.GetContainer().Invoke(func( + modelSvc service.ModelService, + adminSvc interfaces.SpiderAdminService, + ) { + ctx.modelSvc = modelSvc + ctx.adminSvc = adminSvc + }); err != nil { + panic(err) + } + + // model spider service + ctx.modelSpiderSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdSpider) + + // model spider stat service + ctx.modelSpiderStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdSpiderStat) + + // model task service + ctx.modelTaskSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTask) + + // model task stat service + ctx.modelTaskStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTaskStat) + + _spiderCtx = ctx + + return ctx +} + +func newSpiderController() *spiderController { + actions := getSpiderActions() + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + + ctr := NewListPostActionControllerDelegate(ControllerIdSpider, modelSvc.GetBaseService(interfaces.ModelIdSpider), actions) + d := NewListPostActionControllerDelegate(ControllerIdSpider, modelSvc.GetBaseService(interfaces.ModelIdSpider), actions) + ctx := newSpiderContext() + + return &spiderController{ + ListActionControllerDelegate: *ctr, + d: *d, + ctx: ctx, + } +} diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go new file mode 100644 index 000000000..95d28491c --- /dev/null +++ b/core/controllers/spider_v2.go @@ -0,0 +1,1309 @@ +package controllers + +import ( + "errors" + "fmt" + log2 "github.com/apex/log" + "github.com/crawlab-team/crawlab-db/mongo" + vcs "github.com/crawlab-team/crawlab-vcs" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/fs" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/spider/admin" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "io" + "math" + "os" + "path/filepath" + "strings" + "sync" +) + +func GetSpiderById(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + if errors.Is(err, mongo2.ErrNoDocuments) { + HandleErrorNotFound(c, err) + return + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // stat + s.Stat, err = service.NewModelServiceV2[models.SpiderStatV2]().GetById(s.Id) + if err != nil { + if !errors.Is(err, mongo2.ErrNoDocuments) { + HandleErrorInternalServerError(c, err) + return + } + } + + // data collection + if !s.ColId.IsZero() { + col, err := service.NewModelServiceV2[models.DataCollectionV2]().GetById(s.ColId) + if err != nil { + if !errors.Is(err, mongo2.ErrNoDocuments) { + HandleErrorInternalServerError(c, err) + return + } + } else { + s.ColName = col.Name + } + } + + HandleSuccessWithData(c, s) +} + +func GetSpiderList(c *gin.Context) { + withStats := c.Query("stats") + if withStats == "" { + NewControllerV2[models.SpiderV2]().GetList(c) + return + } + + // params + pagination := MustGetPagination(c) + query := MustGetFilterQuery(c) + sort := MustGetSortOption(c) + + // get list + spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(query, &mongo.FindOptions{ + Sort: sort, + Skip: pagination.Size * (pagination.Page - 1), + Limit: pagination.Size, + }) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + HandleErrorInternalServerError(c, err) + } + return + } + if len(spiders) == 0 { + HandleSuccessWithListData(c, []models.SpiderV2{}, 0) + return + } + + // ids + var ids []primitive.ObjectID + for _, s := range spiders { + ids = append(ids, s.Id) + } + + // total count + total, err := service.NewModelServiceV2[models.SpiderV2]().Count(query) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // stat list + spiderStats, err := service.NewModelServiceV2[models.SpiderStatV2]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // cache stat list to dict + dict := map[primitive.ObjectID]models.SpiderStatV2{} + var taskIds []primitive.ObjectID + for _, st := range spiderStats { + if st.Tasks > 0 { + taskCount := int64(st.Tasks) + st.AverageWaitDuration = int64(math.Round(float64(st.WaitDuration) / float64(taskCount))) + st.AverageRuntimeDuration = int64(math.Round(float64(st.RuntimeDuration) / float64(taskCount))) + st.AverageTotalDuration = int64(math.Round(float64(st.TotalDuration) / float64(taskCount))) + } + dict[st.Id] = st + + if !st.LastTaskId.IsZero() { + taskIds = append(taskIds, st.LastTaskId) + } + } + + // task list and stats + var tasks []models.TaskV2 + dictTask := map[primitive.ObjectID]models.TaskV2{} + dictTaskStat := map[primitive.ObjectID]models.TaskStatV2{} + if len(taskIds) > 0 { + // task list + queryTask := bson.M{ + "_id": bson.M{ + "$in": taskIds, + }, + } + tasks, err = service.NewModelServiceV2[models.TaskV2]().GetMany(queryTask, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // task stats list + taskStats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(queryTask, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // cache task stats to dict + for _, st := range taskStats { + dictTaskStat[st.Id] = st + } + + // cache task list to dict + for _, t := range tasks { + st, ok := dictTaskStat[t.Id] + if ok { + t.Stat = &st + } + dictTask[t.SpiderId] = t + } + } + + // iterate list again + var data []models.SpiderV2 + for _, s := range spiders { + // spider stat + st, ok := dict[s.Id] + if ok { + s.Stat = &st + + // last task + t, ok := dictTask[s.Id] + if ok { + s.Stat.LastTask = &t + } + } + + // add to list + data = append(data, s) + } + + // response + HandleSuccessWithListData(c, data, total) +} + +func PostSpider(c *gin.Context) { + // bind + var s models.SpiderV2 + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // upsert data collection + if err := upsertSpiderDataCollection(&s); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + u := GetUserFromContextV2(c) + + // add + s.SetCreated(u.Id) + s.SetUpdated(u.Id) + id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(s) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + s.SetId(id) + + // add stat + st := models.SpiderStatV2{} + st.SetId(id) + st.SetCreated(u.Id) + st.SetUpdated(u.Id) + _, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(st) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // create folder + err = getSpiderFsSvcById(id).CreateDir(".") + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, s) +} + +func PutSpiderById(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // bind + var s models.SpiderV2 + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // upsert data collection + if err := upsertSpiderDataCollection(&s); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + u := GetUserFromContextV2(c) + + modelSvc := service.NewModelServiceV2[models.SpiderV2]() + + // save + s.SetUpdated(u.Id) + err = modelSvc.ReplaceById(id, s) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + _s, err := modelSvc.GetById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + s = *_s + + HandleSuccessWithData(c, s) +} + +func DeleteSpiderById(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { + // delete spider + err = service.NewModelServiceV2[models.SpiderV2]().DeleteById(id) + if err != nil { + return err + } + + // delete spider stat + err = service.NewModelServiceV2[models.SpiderStatV2]().DeleteById(id) + if err != nil { + return err + } + + // related tasks + tasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{"spider_id": id}, nil) + if err != nil { + return err + } + + if len(tasks) == 0 { + return nil + } + + // task ids + var taskIds []primitive.ObjectID + for _, t := range tasks { + taskIds = append(taskIds, t.Id) + } + + // delete related tasks + err = service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) + if err != nil { + return err + } + + // delete related task stats + err = service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) + if err != nil { + return err + } + + // delete tasks logs + wg := sync.WaitGroup{} + wg.Add(len(taskIds)) + for _, id := range taskIds { + go func(id string) { + // delete task logs + logPath := filepath.Join(viper.GetString("log.path"), id) + if err := os.RemoveAll(logPath); err != nil { + log2.Warnf("failed to remove task log directory: %s", logPath) + } + wg.Done() + }(id.Hex()) + } + wg.Wait() + + return nil + }); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func DeleteSpiderList(c *gin.Context) { + var payload struct { + Ids []primitive.ObjectID `json:"ids"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { + // delete spiders + if err := service.NewModelServiceV2[models.SpiderV2]().DeleteMany(bson.M{ + "_id": bson.M{ + "$in": payload.Ids, + }, + }); err != nil { + return err + } + + // delete spider stats + if err := service.NewModelServiceV2[models.SpiderStatV2]().DeleteMany(bson.M{ + "_id": bson.M{ + "$in": payload.Ids, + }, + }); err != nil { + return err + } + + // related tasks + tasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil) + if err != nil { + return err + } + + if len(tasks) == 0 { + return nil + } + + // task ids + var taskIds []primitive.ObjectID + for _, t := range tasks { + taskIds = append(taskIds, t.Id) + } + + // delete related tasks + if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { + return err + } + + // delete related task stats + if err := service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { + return err + } + + // delete tasks logs + wg := sync.WaitGroup{} + wg.Add(len(taskIds)) + for _, id := range taskIds { + go func(id string) { + // delete task logs + logPath := filepath.Join(viper.GetString("log.path"), id) + if err := os.RemoveAll(logPath); err != nil { + log2.Warnf("failed to remove task log directory: %s", logPath) + } + wg.Done() + }(id.Hex()) + } + wg.Wait() + + return nil + }); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func GetSpiderListDir(c *gin.Context) { + path := c.Query("path") + + fsSvc, err := getSpiderFsSvc(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + files, err := fsSvc.List(path) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + HandleErrorInternalServerError(c, err) + return + } + } + + HandleSuccessWithData(c, files) +} + +func GetSpiderFile(c *gin.Context) { + path := c.Query("path") + + fsSvc, err := getSpiderFsSvc(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + data, err := fsSvc.GetFile(path) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, string(data)) +} + +func GetSpiderFileInfo(c *gin.Context) { + path := c.Query("path") + + fsSvc, err := getSpiderFsSvc(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + info, err := fsSvc.GetFileInfo(path) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, info) +} + +func PostSpiderSaveFile(c *gin.Context) { + fsSvc, err := getSpiderFsSvc(c) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + if c.GetHeader("Content-Type") == "application/json" { + var payload struct { + Path string `json:"path"` + Data string `json:"data"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + if err := fsSvc.Save(payload.Path, []byte(payload.Data)); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } else { + path, ok := c.GetPostForm("path") + if !ok { + HandleErrorBadRequest(c, errors.New("missing required field 'path'")) + return + } + file, err := c.FormFile("file") + if err != nil { + HandleErrorBadRequest(c, err) + return + } + f, err := file.Open() + if err != nil { + HandleErrorBadRequest(c, err) + return + } + fileData, err := io.ReadAll(f) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + if err := fsSvc.Save(path, fileData); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + + HandleSuccess(c) +} + +func PostSpiderSaveFiles(c *gin.Context) { + fsSvc, err := getSpiderFsSvc(c) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + form, err := c.MultipartForm() + if err != nil { + HandleErrorBadRequest(c, err) + return + } + wg := sync.WaitGroup{} + wg.Add(len(form.File)) + for path := range form.File { + go func(path string) { + file, err := c.FormFile(path) + if err != nil { + log2.Warnf("invalid file header: %s", path) + log2.Error(err.Error()) + wg.Done() + return + } + f, err := file.Open() + if err != nil { + log2.Warnf("unable to open file: %s", path) + log2.Error(err.Error()) + wg.Done() + return + } + fileData, err := io.ReadAll(f) + if err != nil { + log2.Warnf("unable to read file: %s", path) + log2.Error(err.Error()) + wg.Done() + return + } + if err := fsSvc.Save(path, fileData); err != nil { + log2.Warnf("unable to save file: %s", path) + log2.Error(err.Error()) + wg.Done() + return + } + wg.Done() + }(path) + } + wg.Wait() + + HandleSuccess(c) +} + +func PostSpiderSaveDir(c *gin.Context) { + var payload struct { + Path string `json:"path"` + NewPath string `json:"new_path"` + Data string `json:"data"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + fsSvc, err := getSpiderFsSvc(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := fsSvc.CreateDir(payload.Path); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func PostSpiderRenameFile(c *gin.Context) { + var payload struct { + Path string `json:"path"` + NewPath string `json:"new_path"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + fsSvc, err := getSpiderFsSvc(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := fsSvc.Rename(payload.Path, payload.NewPath); err != nil { + HandleErrorInternalServerError(c, err) + return + } +} + +func DeleteSpiderFile(c *gin.Context) { + var payload struct { + Path string `json:"path"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + if payload.Path == "~" { + payload.Path = "." + } + + fsSvc, err := getSpiderFsSvc(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := fsSvc.Delete(payload.Path); err != nil { + HandleErrorInternalServerError(c, err) + return + } + _, err = fsSvc.GetFileInfo(".") + if err != nil { + _ = fsSvc.CreateDir("/") + } + + HandleSuccess(c) +} + +func PostSpiderCopyFile(c *gin.Context) { + var payload struct { + Path string `json:"path"` + NewPath string `json:"new_path"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + fsSvc, err := getSpiderFsSvc(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := fsSvc.Copy(payload.Path, payload.NewPath); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func PostSpiderExport(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + adminSvc, err := admin.GetSpiderAdminServiceV2() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // zip file path + zipFilePath, err := adminSvc.Export(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // download + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath)) + c.File(zipFilePath) +} + +func PostSpiderRun(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // options + var opts interfaces.SpiderRunOptions + if err := c.ShouldBindJSON(&opts); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // user + if u := GetUserFromContext(c); u != nil { + opts.UserId = u.GetId() + } + + adminSvc, err := admin.GetSpiderAdminServiceV2() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // schedule + taskIds, err := adminSvc.Schedule(id, &opts) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, taskIds) +} + +func GetSpiderGit(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // git client + gitClient, err := getSpiderGitClient(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // return null if git client is empty + if gitClient == nil { + HandleSuccess(c) + return + } + + // current branch + currentBranch, err := gitClient.GetCurrentBranch() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // branches + branches, err := gitClient.GetBranches() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + if branches == nil || len(branches) == 0 && currentBranch != "" { + branches = []vcs.GitRef{{Name: currentBranch}} + } + + // changes + changes, err := gitClient.GetStatus() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // logs + logs, err := gitClient.GetLogsWithRefs() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // ignore + ignore, err := getSpiderGitIgnore(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // git + _git, err := service.NewModelServiceV2[models.GitV2]().GetById(id) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + HandleErrorInternalServerError(c, err) + return + } + } + + // response + res := bson.M{ + "current_branch": currentBranch, + "branches": branches, + "changes": changes, + "logs": logs, + "ignore": ignore, + "git": _git, + } + + HandleSuccessWithData(c, res) +} + +func GetSpiderGitRemoteRefs(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // remote name + remoteName := c.Query("remote") + if remoteName == "" { + remoteName = vcs.GitRemoteNameOrigin + } + + // git client + gitClient, err := getSpiderGitClient(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // return null if git client is empty + if gitClient == nil { + HandleSuccess(c) + return + } + + // refs + refs, err := gitClient.GetRemoteRefs(remoteName) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, refs) +} + +func PostSpiderGitCheckout(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // payload + var payload struct { + Paths []string `json:"paths"` + CommitMessage string `json:"commit_message"` + Branch string `json:"branch"` + Tag string `json:"tag"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // git client + gitClient, err := getSpiderGitClient(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // return null if git client is empty + if gitClient == nil { + HandleSuccess(c) + return + } + + // branch to pull + var branch string + if payload.Branch == "" { + // by default current branch + branch, err = gitClient.GetCurrentBranch() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + } else { + // payload branch + branch = payload.Branch + } + + // checkout + if err := gitSpiderCheckout(gitClient, constants.GitRemoteNameOrigin, branch); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func PostSpiderGitPull(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // payload + var payload struct { + Paths []string `json:"paths"` + CommitMessage string `json:"commit_message"` + Branch string `json:"branch"` + Tag string `json:"tag"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // git + g, err := service.NewModelServiceV2[models.GitV2]().GetById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // attempt to sync git + adminSvc, err := admin.GetSpiderAdminServiceV2() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + _ = adminSvc.SyncGitOne(g) + + HandleSuccess(c) +} + +func PostSpiderGitCommit(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // payload + var payload entity.GitPayload + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // git client + gitClient, err := getSpiderGitClient(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // return null if git client is empty + if gitClient == nil { + HandleSuccess(c) + return + } + + // add + for _, p := range payload.Paths { + if err := gitClient.Add(p); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + + // commit + if err := gitClient.Commit(payload.CommitMessage); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // push + if err := gitClient.Push( + vcs.WithRemoteNamePush(vcs.GitRemoteNameOrigin), + ); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func GetSpiderDataSource(c *gin.Context) { + // spider id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // spider + s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // data source + ds, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(s.DataSourceId) + if err != nil { + if err.Error() == mongo2.ErrNoDocuments.Error() { + HandleSuccess(c) + return + } + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, ds) +} + +func PostSpiderDataSource(c *gin.Context) { + // spider id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // data source id + dsId, err := primitive.ObjectIDFromHex(c.Param("ds_id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // spider + s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // data source + if !dsId.IsZero() { + _, err = service.NewModelServiceV2[models.DataSourceV2]().GetById(dsId) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + + // save data source id + u := GetUserFromContextV2(c) + s.DataSourceId = dsId + s.SetUpdatedBy(u.Id) + _, err = service.NewModelServiceV2[models.SpiderV2]().InsertOne(*s) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func getSpiderFsSvc(c *gin.Context) (svc interfaces.FsServiceV2, err error) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + return nil, err + } + + workspacePath := viper.GetString("workspace") + fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, id.Hex())) + + return fsSvc, nil +} + +func getSpiderFsSvcById(id primitive.ObjectID) interfaces.FsServiceV2 { + workspacePath := viper.GetString("workspace") + fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, id.Hex())) + return fsSvc +} + +func getSpiderGitClient(id primitive.ObjectID) (client *vcs.GitClient, err error) { + // git + g, err := service.NewModelServiceV2[models.GitV2]().GetById(id) + if err != nil { + if !errors.Is(err, mongo2.ErrNoDocuments) { + return nil, trace.TraceError(err) + } + return nil, nil + } + + // git client + workspacePath := viper.GetString("workspace") + client, err = vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, id.Hex()))) + if err != nil { + return nil, err + } + + // set auth + utils.InitGitClientAuthV2(g, client) + + // remote name + remoteName := vcs.GitRemoteNameOrigin + + // update remote + r, err := client.GetRemote(remoteName) + if errors.Is(err, git.ErrRemoteNotFound) { + // remote not exists, create + if _, err := client.CreateRemote(&config.RemoteConfig{ + Name: remoteName, + URLs: []string{g.Url}, + }); err != nil { + return nil, trace.TraceError(err) + } + } else if err == nil { + // remote exists, update if different + if g.Url != r.Config().URLs[0] { + if err := client.DeleteRemote(remoteName); err != nil { + return nil, trace.TraceError(err) + } + if _, err := client.CreateRemote(&config.RemoteConfig{ + Name: remoteName, + URLs: []string{g.Url}, + }); err != nil { + return nil, trace.TraceError(err) + } + } + client.SetRemoteUrl(g.Url) + } else { + // error + return nil, trace.TraceError(err) + } + + // check if head reference exists + _, err = client.GetRepository().Head() + if err == nil { + return client, nil + } + + // align master/main branch + alignSpiderGitBranch(client) + + return client, nil +} + +func alignSpiderGitBranch(gitClient *vcs.GitClient) { + // current branch + currentBranch, err := gitClient.GetCurrentBranch() + if err != nil { + trace.PrintError(err) + return + } + + // skip if current branch is not master + if currentBranch != vcs.GitBranchNameMaster { + return + } + + // remote refs + refs, err := gitClient.GetRemoteRefs(vcs.GitRemoteNameOrigin) + if err != nil { + trace.PrintError(err) + return + } + + // main branch + defaultRemoteBranch, err := getSpiderDefaultRemoteBranch(refs) + if err != nil || defaultRemoteBranch == "" { + return + } + + // move branch + if err := gitClient.MoveBranch(vcs.GitBranchNameMaster, defaultRemoteBranch); err != nil { + trace.PrintError(err) + } +} + +func getSpiderDefaultRemoteBranch(refs []vcs.GitRef) (defaultRemoteBranchName string, err error) { + // remote branch name + for _, r := range refs { + if r.Type != vcs.GitRefTypeBranch { + continue + } + + if r.Name == vcs.GitBranchNameMain { + defaultRemoteBranchName = r.Name + break + } + + if r.Name == vcs.GitBranchNameMaster { + defaultRemoteBranchName = r.Name + continue + } + + if defaultRemoteBranchName == "" { + defaultRemoteBranchName = r.Name + continue + } + } + + return defaultRemoteBranchName, nil +} + +func getSpiderGitIgnore(id primitive.ObjectID) (ignore []string, err error) { + workspacePath := viper.GetString("workspace") + filePath := filepath.Join(workspacePath, id.Hex(), ".gitignore") + if !utils.Exists(filePath) { + return nil, nil + } + data, err := os.ReadFile(filePath) + if err != nil { + return nil, trace.TraceError(err) + } + ignore = strings.Split(string(data), "\n") + return ignore, nil +} + +func gitSpiderCheckout(gitClient *vcs.GitClient, remote, branch string) (err error) { + if err := gitClient.CheckoutBranch(branch, vcs.WithBranch(branch)); err != nil { + return trace.TraceError(err) + } + + // pull + return spiderGitPull(gitClient, remote, branch) +} + +func spiderGitPull(gitClient *vcs.GitClient, remote, branch string) (err error) { + // pull + if err := gitClient.Pull( + vcs.WithRemoteNamePull(remote), + vcs.WithBranchNamePull(branch), + ); err != nil { + return trace.TraceError(err) + } + + // reset + if err := gitClient.Reset(); err != nil { + return trace.TraceError(err) + } + + return nil +} + +func upsertSpiderDataCollection(s *models.SpiderV2) (err error) { + modelSvc := service.NewModelServiceV2[models.DataCollectionV2]() + if s.ColId.IsZero() { + // validate + if s.ColName == "" { + return errors.New("data collection name is required") + } + // no id + dc, err := modelSvc.GetOne(bson.M{"name": s.ColName}, nil) + if err != nil { + if errors.Is(err, mongo2.ErrNoDocuments) { + // not exists, add new + dc = &models.DataCollectionV2{Name: s.ColName} + dcId, err := modelSvc.InsertOne(*dc) + if err != nil { + return err + } + dc.SetId(dcId) + } else { + // error + return err + } + } + s.ColId = dc.Id + + // create index + _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.TaskKey: 1}}) + _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.HashKey: 1}}) + } else { + // with id + dc, err := modelSvc.GetById(s.ColId) + if err != nil { + return err + } + s.ColId = dc.Id + } + return nil +} diff --git a/core/controllers/spider_v2_test.go b/core/controllers/spider_v2_test.go new file mode 100644 index 000000000..69c50f280 --- /dev/null +++ b/core/controllers/spider_v2_test.go @@ -0,0 +1,249 @@ +package controllers_test + +import ( + "bytes" + "encoding/json" + "github.com/crawlab-team/crawlab/core/controllers" + "github.com/crawlab-team/crawlab/core/middlewares" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "net/http" + "net/http/httptest" + "testing" +) + +func TestCreateSpider(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + router := gin.Default() + router.Use(middlewares.AuthorizationMiddlewareV2()) + router.POST("/spiders", controllers.PostSpider) + + payload := models.SpiderV2{ + Name: "Test Spider", + ColName: "test_spiders", + } + jsonValue, _ := json.Marshal(payload) + req, _ := http.NewRequest("POST", "/spiders", bytes.NewBuffer(jsonValue)) + req.Header.Set("Authorization", TestToken) + resp := httptest.NewRecorder() + + router.ServeHTTP(resp, req) + + assert.Equal(t, http.StatusOK, resp.Code) + + var response controllers.Response[models.SpiderV2] + err := json.Unmarshal(resp.Body.Bytes(), &response) + require.Nil(t, err) + assert.False(t, response.Data.Id.IsZero()) + assert.Equal(t, payload.Name, response.Data.Name) + assert.False(t, response.Data.ColId.IsZero()) +} + +func TestGetSpiderById(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + router := gin.Default() + router.Use(middlewares.AuthorizationMiddlewareV2()) + router.GET("/spiders/:id", controllers.GetSpiderById) + + model := models.SpiderV2{ + Name: "Test Spider", + ColName: "test_spiders", + } + id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model) + require.Nil(t, err) + ts := models.SpiderStatV2{} + ts.SetId(id) + _, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts) + require.Nil(t, err) + + req, _ := http.NewRequest("GET", "/spiders/"+id.Hex(), nil) + req.Header.Set("Authorization", TestToken) + resp := httptest.NewRecorder() + + router.ServeHTTP(resp, req) + + assert.Equal(t, http.StatusOK, resp.Code) + + var response controllers.Response[models.SpiderV2] + err = json.Unmarshal(resp.Body.Bytes(), &response) + require.Nil(t, err) + assert.Equal(t, model.Name, response.Data.Name) +} + +func TestUpdateSpiderById(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + router := gin.Default() + router.Use(middlewares.AuthorizationMiddlewareV2()) + router.PUT("/spiders/:id", controllers.PutSpiderById) + + model := models.SpiderV2{ + Name: "Test Spider", + ColName: "test_spiders", + } + id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model) + require.Nil(t, err) + ts := models.SpiderStatV2{} + ts.SetId(id) + _, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts) + require.Nil(t, err) + + spiderId := id.Hex() + payload := models.SpiderV2{ + Name: "Updated Spider", + ColName: "test_spider", + } + payload.SetId(id) + jsonValue, _ := json.Marshal(payload) + req, _ := http.NewRequest("PUT", "/spiders/"+spiderId, bytes.NewBuffer(jsonValue)) + req.Header.Set("Authorization", TestToken) + resp := httptest.NewRecorder() + + router.ServeHTTP(resp, req) + + assert.Equal(t, http.StatusOK, resp.Code) + + var response controllers.Response[models.SpiderV2] + err = json.Unmarshal(resp.Body.Bytes(), &response) + require.Nil(t, err) + assert.Equal(t, payload.Name, response.Data.Name) + + svc := service.NewModelServiceV2[models.SpiderV2]() + resModel, err := svc.GetById(id) + require.Nil(t, err) + assert.Equal(t, payload.Name, resModel.Name) +} + +func TestDeleteSpiderById(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + router := gin.Default() + router.Use(middlewares.AuthorizationMiddlewareV2()) + router.DELETE("/spiders/:id", controllers.DeleteSpiderById) + + model := models.SpiderV2{ + Name: "Test Spider", + ColName: "test_spiders", + } + id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model) + require.Nil(t, err) + ts := models.SpiderStatV2{} + ts.SetId(id) + _, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts) + require.Nil(t, err) + task := models.TaskV2{} + task.SpiderId = id + taskId, err := service.NewModelServiceV2[models.TaskV2]().InsertOne(task) + require.Nil(t, err) + taskStat := models.TaskStatV2{} + taskStat.SetId(taskId) + _, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(taskStat) + require.Nil(t, err) + + req, _ := http.NewRequest("DELETE", "/spiders/"+id.Hex(), nil) + req.Header.Set("Authorization", TestToken) + resp := httptest.NewRecorder() + + router.ServeHTTP(resp, req) + + assert.Equal(t, http.StatusOK, resp.Code) + + _, err = service.NewModelServiceV2[models.SpiderV2]().GetById(id) + assert.NotNil(t, err) + _, err = service.NewModelServiceV2[models.SpiderStatV2]().GetById(id) + assert.NotNil(t, err) + taskCount, err := service.NewModelServiceV2[models.TaskV2]().Count(bson.M{"spider_id": id}) + require.Nil(t, err) + assert.Equal(t, 0, taskCount) + taskStatCount, err := service.NewModelServiceV2[models.TaskStatV2]().Count(bson.M{"_id": taskId}) + require.Nil(t, err) + assert.Equal(t, 0, taskStatCount) + +} + +func TestDeleteSpiderList(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + gin.SetMode(gin.TestMode) + + router := gin.Default() + router.Use(middlewares.AuthorizationMiddlewareV2()) + router.DELETE("/spiders", controllers.DeleteSpiderList) + + modelList := []models.SpiderV2{ + { + Name: "Test Name 1", + ColName: "test_spiders", + }, { + Name: "Test Name 2", + ColName: "test_spiders", + }, + } + var ids []primitive.ObjectID + var taskIds []primitive.ObjectID + for _, model := range modelList { + id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(model) + require.Nil(t, err) + ts := models.SpiderStatV2{} + ts.SetId(id) + _, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(ts) + require.Nil(t, err) + task := models.TaskV2{} + task.SpiderId = id + taskId, err := service.NewModelServiceV2[models.TaskV2]().InsertOne(task) + require.Nil(t, err) + taskStat := models.TaskStatV2{} + taskStat.SetId(taskId) + _, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(taskStat) + require.Nil(t, err) + ids = append(ids, id) + taskIds = append(taskIds, taskId) + } + + payload := struct { + Ids []primitive.ObjectID `json:"ids"` + }{ + Ids: ids, + } + jsonValue, _ := json.Marshal(payload) + req, _ := http.NewRequest("DELETE", "/spiders", bytes.NewBuffer(jsonValue)) + req.Header.Set("Authorization", TestToken) + resp := httptest.NewRecorder() + + router.ServeHTTP(resp, req) + + assert.Equal(t, http.StatusOK, resp.Code) + + spiderCount, err := service.NewModelServiceV2[models.SpiderV2]().Count(bson.M{"_id": bson.M{"$in": ids}}) + require.Nil(t, err) + assert.Equal(t, 0, spiderCount) + spiderStatCount, err := service.NewModelServiceV2[models.SpiderStatV2]().Count(bson.M{"_id": bson.M{"$in": ids}}) + require.Nil(t, err) + assert.Equal(t, 0, spiderStatCount) + taskCount, err := service.NewModelServiceV2[models.TaskV2]().Count(bson.M{"_id": bson.M{"$in": taskIds}}) + require.Nil(t, err) + assert.Equal(t, 0, taskCount) + taskStatCount, err := service.NewModelServiceV2[models.TaskStatV2]().Count(bson.M{"_id": bson.M{"$in": taskIds}}) + require.Nil(t, err) + assert.Equal(t, 0, taskStatCount) +} diff --git a/core/controllers/stats.go b/core/controllers/stats.go new file mode 100644 index 000000000..3d474956c --- /dev/null +++ b/core/controllers/stats.go @@ -0,0 +1,87 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + "net/http" + "time" +) + +var StatsController ActionController + +func getStatsActions() []Action { + statsCtx := newStatsContext() + return []Action{ + { + Method: http.MethodGet, + Path: "/overview", + HandlerFunc: statsCtx.getOverview, + }, + { + Method: http.MethodGet, + Path: "/daily", + HandlerFunc: statsCtx.getDaily, + }, + { + Method: http.MethodGet, + Path: "/tasks", + HandlerFunc: statsCtx.getTasks, + }, + } +} + +type statsContext struct { + statsSvc interfaces.StatsService + defaultQuery bson.M +} + +func (svc *statsContext) getOverview(c *gin.Context) { + data, err := svc.statsSvc.GetOverviewStats(svc.defaultQuery) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccessWithData(c, data) +} + +func (svc *statsContext) getDaily(c *gin.Context) { + data, err := svc.statsSvc.GetDailyStats(svc.defaultQuery) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccessWithData(c, data) +} + +func (svc *statsContext) getTasks(c *gin.Context) { + data, err := svc.statsSvc.GetTaskStats(svc.defaultQuery) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccessWithData(c, data) +} + +func newStatsContext() *statsContext { + // context + ctx := &statsContext{ + defaultQuery: bson.M{ + "create_ts": bson.M{ + "$gte": time.Now().Add(-30 * 24 * time.Hour), + }, + }, + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + statsSvc interfaces.StatsService, + ) { + ctx.statsSvc = statsSvc + }); err != nil { + panic(err) + } + + return ctx +} diff --git a/core/controllers/stats_v2.go b/core/controllers/stats_v2.go new file mode 100644 index 000000000..cf7bda039 --- /dev/null +++ b/core/controllers/stats_v2.go @@ -0,0 +1,41 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/stats" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + "time" +) + +var statsDefaultQuery = bson.M{ + "create_ts": bson.M{ + "$gte": time.Now().Add(-30 * 24 * time.Hour), + }, +} + +func GetStatsOverview(c *gin.Context) { + data, err := stats.GetStatsService().GetOverviewStats(statsDefaultQuery) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccessWithData(c, data) +} + +func GetStatsDaily(c *gin.Context) { + data, err := stats.GetStatsService().GetDailyStats(statsDefaultQuery) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccessWithData(c, data) +} + +func GetStatsTasks(c *gin.Context) { + data, err := stats.GetStatsService().GetTaskStats(statsDefaultQuery) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccessWithData(c, data) +} diff --git a/core/controllers/sync.go b/core/controllers/sync.go new file mode 100644 index 000000000..674bff76f --- /dev/null +++ b/core/controllers/sync.go @@ -0,0 +1,57 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/utils" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "net/http" + "path/filepath" +) + +var SyncController ActionController + +func getSyncActions() []Action { + var ctx = newSyncContext() + return []Action{ + { + Method: http.MethodGet, + Path: "/:id/scan", + HandlerFunc: ctx.scan, + }, + { + Method: http.MethodGet, + Path: "/:id/download", + HandlerFunc: ctx.download, + }, + } +} + +type syncContext struct { +} + +func (ctx *syncContext) scan(c *gin.Context) { + id := c.Param("id") + dir := ctx._getDir(id) + files, err := utils.ScanDirectory(dir) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + c.AbortWithStatusJSON(http.StatusOK, files) +} + +func (ctx *syncContext) download(c *gin.Context) { + id := c.Param("id") + filePath := c.Query("path") + dir := ctx._getDir(id) + c.File(filepath.Join(dir, filePath)) +} + +func (ctx *syncContext) _getDir(id string) string { + workspacePath := viper.GetString("workspace") + return filepath.Join(workspacePath, id) +} + +func newSyncContext() syncContext { + return syncContext{} +} diff --git a/core/controllers/system_info.go b/core/controllers/system_info.go new file mode 100644 index 000000000..a89edc15d --- /dev/null +++ b/core/controllers/system_info.go @@ -0,0 +1,28 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/entity" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "net/http" +) + +func getSystemInfo(c *gin.Context) { + info := &entity.SystemInfo{ + Edition: viper.GetString("info.edition"), + Version: viper.GetString("info.version"), + } + HandleSuccessWithData(c, info) +} + +func getSystemInfoActions() []Action { + return []Action{ + { + Path: "", + Method: http.MethodGet, + HandlerFunc: getSystemInfo, + }, + } +} + +var SystemInfoController ActionController diff --git a/core/controllers/system_info_v2.go b/core/controllers/system_info_v2.go new file mode 100644 index 000000000..fc01ea6f0 --- /dev/null +++ b/core/controllers/system_info_v2.go @@ -0,0 +1,15 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/entity" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" +) + +func GetSystemInfo(c *gin.Context) { + info := &entity.SystemInfo{ + Edition: viper.GetString("info.edition"), + Version: viper.GetString("info.version"), + } + HandleSuccessWithData(c, info) +} diff --git a/core/controllers/tag.go b/core/controllers/tag.go new file mode 100644 index 000000000..3f74abf24 --- /dev/null +++ b/core/controllers/tag.go @@ -0,0 +1,3 @@ +package controllers + +var TagController ListController diff --git a/core/controllers/task.go b/core/controllers/task.go new file mode 100644 index 000000000..7d9015f2a --- /dev/null +++ b/core/controllers/task.go @@ -0,0 +1,534 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/result" + "github.com/crawlab-team/crawlab/core/task/log" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "net/http" + "strings" +) + +var TaskController *taskController + +func getTaskActions() []Action { + taskCtx := newTaskContext() + return []Action{ + { + Method: http.MethodPost, + Path: "/run", + HandlerFunc: taskCtx.run, + }, + { + Method: http.MethodPost, + Path: "/:id/restart", + HandlerFunc: taskCtx.restart, + }, + { + Method: http.MethodPost, + Path: "/:id/cancel", + HandlerFunc: taskCtx.cancel, + }, + { + Method: http.MethodGet, + Path: "/:id/logs", + HandlerFunc: taskCtx.getLogs, + }, + { + Method: http.MethodGet, + Path: "/:id/data", + HandlerFunc: taskCtx.getData, + }, + } +} + +type taskController struct { + ListActionControllerDelegate + d ListActionControllerDelegate + ctx *taskContext +} + +func (ctr *taskController) Get(c *gin.Context) { + ctr.ctx.getWithStatsSpider(c) +} + +func (ctr *taskController) Delete(c *gin.Context) { + if err := ctr.ctx._delete(c); err != nil { + return + } + HandleSuccess(c) +} + +func (ctr *taskController) GetList(c *gin.Context) { + withStats := c.Query("stats") + if withStats == "" { + ctr.d.GetList(c) + return + } + ctr.ctx.getListWithStats(c) +} + +func (ctr *taskController) DeleteList(c *gin.Context) { + if err := ctr.ctx._deleteList(c); err != nil { + return + } + HandleSuccess(c) +} + +type taskContext struct { + modelSvc service.ModelService + modelTaskSvc interfaces.ModelBaseService + modelTaskStatSvc interfaces.ModelBaseService + adminSvc interfaces.SpiderAdminService + schedulerSvc interfaces.TaskSchedulerService + l log.Driver +} + +func (ctx *taskContext) run(c *gin.Context) { + // task + var t models.Task + if err := c.ShouldBindJSON(&t); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // validate spider id + if t.GetSpiderId().IsZero() { + HandleErrorBadRequest(c, errors.ErrorTaskEmptySpiderId) + return + } + + // spider + s, err := ctx.modelSvc.GetSpiderById(t.GetSpiderId()) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // options + opts := &interfaces.SpiderRunOptions{ + Mode: t.Mode, + NodeIds: t.NodeIds, + Cmd: t.Cmd, + Param: t.Param, + Priority: t.Priority, + } + + // user + if u := GetUserFromContext(c); u != nil { + opts.UserId = u.GetId() + } + + // run + taskIds, err := ctx.adminSvc.Schedule(s.GetId(), opts) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, taskIds) +} + +func (ctx *taskContext) restart(c *gin.Context) { + // id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // task + t, err := ctx.modelSvc.GetTaskById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // options + opts := &interfaces.SpiderRunOptions{ + Mode: t.Mode, + NodeIds: t.NodeIds, + Cmd: t.Cmd, + Param: t.Param, + Priority: t.Priority, + } + + // user + if u := GetUserFromContext(c); u != nil { + opts.UserId = u.GetId() + } + + // run + taskIds, err := ctx.adminSvc.Schedule(t.SpiderId, opts) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, taskIds) +} + +func (ctx *taskContext) cancel(c *gin.Context) { + // id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // task + t, err := ctx.modelSvc.GetTaskById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // validate + if !utils.IsCancellable(t.Status) { + HandleErrorInternalServerError(c, errors.ErrorControllerNotCancellable) + return + } + + // cancel + if err := ctx.schedulerSvc.Cancel(id, GetUserFromContext(c)); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func (ctx *taskContext) getLogs(c *gin.Context) { + // id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // pagination + p, err := GetPagination(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // logs + logs, err := ctx.l.Find(id.Hex(), "", (p.Page-1)*p.Size, p.Size) + if err != nil { + if strings.HasSuffix(err.Error(), "Status:404 Not Found") { + HandleSuccess(c) + return + } + HandleErrorInternalServerError(c, err) + return + } + total, err := ctx.l.Count(id.Hex(), "") + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithListData(c, logs, total) +} + +func (ctx *taskContext) getListWithStats(c *gin.Context) { + // params + pagination := MustGetPagination(c) + query := MustGetFilterQuery(c) + sort := MustGetSortOption(c) + + // get list + list, err := ctx.modelTaskSvc.GetList(query, &mongo.FindOptions{ + Sort: sort, + Skip: pagination.Size * (pagination.Page - 1), + Limit: pagination.Size, + }) + if err != nil { + if err == mongo2.ErrNoDocuments { + HandleErrorNotFound(c, err) + } else { + HandleErrorInternalServerError(c, err) + } + return + } + + // check empty list + if len(list.GetModels()) == 0 { + HandleSuccessWithListData(c, nil, 0) + return + } + + // ids + var ids []primitive.ObjectID + for _, d := range list.GetModels() { + t := d.(interfaces.Model) + ids = append(ids, t.GetId()) + } + + // total count + total, err := ctx.modelTaskSvc.Count(query) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // stat list + query = bson.M{ + "_id": bson.M{ + "$in": ids, + }, + } + stats, err := ctx.modelSvc.GetTaskStatList(query, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // cache stat list to dict + dict := map[primitive.ObjectID]models.TaskStat{} + for _, s := range stats { + dict[s.GetId()] = s + } + + // iterate list again + var data []interface{} + for _, d := range list.GetModels() { + t := d.(*models.Task) + s, ok := dict[t.GetId()] + if ok { + t.Stat = &s + } + data = append(data, *t) + } + + // response + HandleSuccessWithListData(c, data, total) +} + +func (ctx *taskContext) getWithStatsSpider(c *gin.Context) { + // id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // task + t, err := ctx.modelSvc.GetTaskById(id) + if err == mongo2.ErrNoDocuments { + HandleErrorNotFound(c, err) + return + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // spider + t.Spider, _ = ctx.modelSvc.GetSpiderById(t.SpiderId) + + // skip if task status is pending + if t.Status == constants.TaskStatusPending { + HandleSuccessWithData(c, t) + return + } + + // task stat + t.Stat, _ = ctx.modelSvc.GetTaskStatById(id) + + HandleSuccessWithData(c, t) +} + +func (ctx *taskContext) getData(c *gin.Context) { + // id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // pagination + p, err := GetPagination(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // task + t, err := ctx.modelSvc.GetTaskById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // result service + resultSvc, err := result.GetResultService(t.SpiderId) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // query + query := generic.ListQuery{ + generic.ListQueryCondition{ + Key: constants.TaskKey, + Op: generic.OpEqual, + Value: t.Id, + }, + } + + // list + data, err := resultSvc.List(query, &generic.ListOptions{ + Skip: (p.Page - 1) * p.Size, + Limit: p.Size, + Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, + }) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // total + total, err := resultSvc.Count(query) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithListData(c, data, total) +} + +func (ctx *taskContext) _delete(c *gin.Context) (err error) { + id := c.Param("id") + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { + // delete task + task, err := ctx.modelSvc.GetTaskById(oid) + if err != nil { + return err + } + if err := delegate2.NewModelDelegate(task, GetUserFromContext(c)).Delete(); err != nil { + return err + } + + // delete task stat + taskStat, err := ctx.modelSvc.GetTaskStatById(oid) + if err != nil { + return err + } + if err := delegate2.NewModelDelegate(taskStat, GetUserFromContext(c)).Delete(); err != nil { + return err + } + + return nil + }); err != nil { + HandleErrorInternalServerError(c, err) + return err + } + + return nil +} + +func (ctx *taskContext) _deleteList(c *gin.Context) (err error) { + payload, err := NewJsonBinder(ControllerIdTask).BindBatchRequestPayload(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := mongo.RunTransaction(func(context mongo2.SessionContext) error { + // delete tasks + if err := ctx.modelTaskSvc.DeleteList(bson.M{ + "_id": bson.M{ + "$in": payload.Ids, + }, + }); err != nil { + return err + } + + // delete task stats + if err := ctx.modelTaskStatSvc.DeleteList(bson.M{ + "_id": bson.M{ + "$in": payload.Ids, + }, + }); err != nil { + return err + } + + return nil + }); err != nil { + HandleErrorInternalServerError(c, err) + return err + } + + return nil +} + +func newTaskContext() *taskContext { + // context + ctx := &taskContext{} + + // dependency injection + if err := container.GetContainer().Invoke(func( + modelSvc service.ModelService, + adminSvc interfaces.SpiderAdminService, + schedulerSvc interfaces.TaskSchedulerService, + ) { + ctx.modelSvc = modelSvc + ctx.adminSvc = adminSvc + ctx.schedulerSvc = schedulerSvc + }); err != nil { + panic(err) + } + + // model task service + ctx.modelTaskSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTask) + + // model task stat service + ctx.modelTaskStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTaskStat) + + // log driver + l, err := log.GetLogDriver(log.DriverTypeFile) + if err != nil { + panic(err) + } + ctx.l = l + + return ctx +} + +func newTaskController() *taskController { + actions := getTaskActions() + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + + ctr := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions) + d := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions) + ctx := newTaskContext() + + return &taskController{ + ListActionControllerDelegate: *ctr, + d: *d, + ctx: ctx, + } +} diff --git a/core/controllers/task_v2.go b/core/controllers/task_v2.go new file mode 100644 index 000000000..ab8619cf7 --- /dev/null +++ b/core/controllers/task_v2.go @@ -0,0 +1,465 @@ +package controllers + +import ( + "errors" + log2 "github.com/apex/log" + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/result" + "github.com/crawlab-team/crawlab/core/spider/admin" + "github.com/crawlab-team/crawlab/core/task/log" + "github.com/crawlab-team/crawlab/core/task/scheduler" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "os" + "path/filepath" + "strings" + "sync" +) + +func GetTaskById(c *gin.Context) { + // id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // task + t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + if errors.Is(err, mongo2.ErrNoDocuments) { + HandleErrorNotFound(c, err) + return + } + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // spider + t.Spider, _ = service.NewModelServiceV2[models.SpiderV2]().GetById(t.SpiderId) + + // skip if task status is pending + if t.Status == constants.TaskStatusPending { + HandleSuccessWithData(c, t) + return + } + + // task stat + t.Stat, _ = service.NewModelServiceV2[models.TaskStatV2]().GetById(id) + + HandleSuccessWithData(c, t) +} + +func GetTaskList(c *gin.Context) { + withStats := c.Query("stats") + if withStats == "" { + NewControllerV2[models.TaskV2]().GetList(c) + return + } + + // params + pagination := MustGetPagination(c) + query := MustGetFilterQuery(c) + sort := MustGetSortOption(c) + + // get list + list, err := service.NewModelServiceV2[models.TaskV2]().GetMany(query, &mongo.FindOptions{ + Sort: sort, + Skip: pagination.Size * (pagination.Page - 1), + Limit: pagination.Size, + }) + if err != nil { + if errors.Is(err, mongo2.ErrNoDocuments) { + HandleErrorNotFound(c, err) + } else { + HandleErrorInternalServerError(c, err) + } + return + } + + // check empty list + if len(list) == 0 { + HandleSuccessWithListData(c, nil, 0) + return + } + + // ids + var ids []primitive.ObjectID + for _, t := range list { + ids = append(ids, t.Id) + } + + // total count + total, err := service.NewModelServiceV2[models.TaskV2]().Count(query) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // stat list + query = bson.M{ + "_id": bson.M{ + "$in": ids, + }, + } + stats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(query, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // cache stat list to dict + dict := map[primitive.ObjectID]models.TaskStatV2{} + for _, s := range stats { + dict[s.Id] = s + } + + // iterate list again + for i, t := range list { + ts, ok := dict[t.Id] + if ok { + list[i].Stat = &ts + } + } + + // response + HandleSuccessWithListData(c, list, total) +} + +func DeleteTaskById(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // delete in db + if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { + // delete task + _, err = service.NewModelServiceV2[models.TaskV2]().GetById(id) + if err != nil { + return err + } + err = service.NewModelServiceV2[models.TaskV2]().DeleteById(id) + if err != nil { + return err + } + + // delete task stat + _, err = service.NewModelServiceV2[models.TaskStatV2]().GetById(id) + if err != nil { + log2.Warnf("delete task stat error: %s", err.Error()) + return nil + } + err = service.NewModelServiceV2[models.TaskStatV2]().DeleteById(id) + if err != nil { + log2.Warnf("delete task stat error: %s", err.Error()) + return nil + } + + return nil + }); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // delete task logs + logPath := filepath.Join(viper.GetString("log.path"), id.Hex()) + if err := os.RemoveAll(logPath); err != nil { + log2.Warnf("failed to remove task log directory: %s", logPath) + } + + HandleSuccess(c) +} + +func DeleteList(c *gin.Context) { + var payload struct { + Ids []primitive.ObjectID `json:"ids"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := mongo.RunTransaction(func(context mongo2.SessionContext) error { + // delete tasks + if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{ + "_id": bson.M{ + "$in": payload.Ids, + }, + }); err != nil { + return err + } + + // delete task stats + if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{ + "_id": bson.M{ + "$in": payload.Ids, + }, + }); err != nil { + log2.Warnf("delete task stat error: %s", err.Error()) + return nil + } + + return nil + }); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // delete tasks logs + wg := sync.WaitGroup{} + wg.Add(len(payload.Ids)) + for _, id := range payload.Ids { + go func(id string) { + // delete task logs + logPath := filepath.Join(viper.GetString("log.path"), id) + if err := os.RemoveAll(logPath); err != nil { + log2.Warnf("failed to remove task log directory: %s", logPath) + } + wg.Done() + }(id.Hex()) + } + wg.Wait() + + HandleSuccess(c) +} + +func PostTaskRun(c *gin.Context) { + // task + var t models.TaskV2 + if err := c.ShouldBindJSON(&t); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // validate spider id + if t.SpiderId.IsZero() { + HandleErrorBadRequest(c, errors.New("spider id is required")) + return + } + + // spider + s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(t.SpiderId) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // options + opts := &interfaces.SpiderRunOptions{ + Mode: t.Mode, + NodeIds: t.NodeIds, + Cmd: t.Cmd, + Param: t.Param, + Priority: t.Priority, + } + + // user + if u := GetUserFromContextV2(c); u != nil { + opts.UserId = u.Id + } + + // run + adminSvc, err := admin.GetSpiderAdminServiceV2() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + taskIds, err := adminSvc.Schedule(s.Id, opts) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, taskIds) + +} + +func PostTaskRestart(c *gin.Context) { + // id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // task + t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // options + opts := &interfaces.SpiderRunOptions{ + Mode: t.Mode, + NodeIds: t.NodeIds, + Cmd: t.Cmd, + Param: t.Param, + Priority: t.Priority, + } + + // user + if u := GetUserFromContextV2(c); u != nil { + opts.UserId = u.Id + } + + // run + adminSvc, err := admin.GetSpiderAdminServiceV2() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + taskIds, err := adminSvc.Schedule(t.SpiderId, opts) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, taskIds) +} + +func PostTaskCancel(c *gin.Context) { + // id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // task + t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // validate + if !utils.IsCancellable(t.Status) { + HandleErrorInternalServerError(c, errors.New("task is not cancellable")) + return + } + + u := GetUserFromContextV2(c) + + // cancel + schedulerSvc, err := scheduler.GetTaskSchedulerServiceV2() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + if err := schedulerSvc.Cancel(id, u.Id); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func GetTaskLogs(c *gin.Context) { + // id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // pagination + p, err := GetPagination(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // logs + logDriver, err := log.GetFileLogDriver() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + logs, err := logDriver.Find(id.Hex(), "", (p.Page-1)*p.Size, p.Size) + if err != nil { + if strings.HasSuffix(err.Error(), "Status:404 Not Found") { + HandleSuccess(c) + return + } + HandleErrorInternalServerError(c, err) + return + } + total, err := logDriver.Count(id.Hex(), "") + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithListData(c, logs, total) +} + +func GetTaskData(c *gin.Context) { + // id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // pagination + p, err := GetPagination(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // task + t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // result service + resultSvc, err := result.GetResultService(t.SpiderId) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // query + query := generic.ListQuery{ + generic.ListQueryCondition{ + Key: constants.TaskKey, + Op: generic.OpEqual, + Value: t.Id, + }, + } + + // list + data, err := resultSvc.List(query, &generic.ListOptions{ + Skip: (p.Page - 1) * p.Size, + Limit: p.Size, + Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, + }) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // total + total, err := resultSvc.Count(query) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithListData(c, data, total) +} diff --git a/core/controllers/test/base.go b/core/controllers/test/base.go new file mode 100644 index 000000000..fe8b18356 --- /dev/null +++ b/core/controllers/test/base.go @@ -0,0 +1,104 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/controllers" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/routes" + "github.com/crawlab-team/go-trace" + "github.com/gavv/httpexpect/v2" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "go.uber.org/dig" + "net/http/httptest" + "testing" + "time" +) + +func init() { + var err error + T, err = NewTest() + if err != nil { + panic(err) + } +} + +type Test struct { + // dependencies + modelSvc service.ModelService + + // internals + app *gin.Engine + svr *httptest.Server + + // test data + TestUsername string + TestPassword string + TestToken string +} + +func (t *Test) Setup(t2 *testing.T) { + //if err := controllers.InitControllers(); err != nil { + // panic(err) + //} + //t2.Cleanup(t.Cleanup) +} + +func (t *Test) Cleanup() { + _ = t.modelSvc.DropAll() + time.Sleep(200 * time.Millisecond) +} + +func (t *Test) NewExpect(t2 *testing.T) (e *httpexpect.Expect) { + e = httpexpect.New(t2, t.svr.URL) + res := e.POST("/login").WithJSON(map[string]string{ + "username": t.TestUsername, + "password": t.TestPassword, + }).Expect().JSON().Object() + t.TestToken = res.Path("$.data").String().Raw() + require.NotEmpty(t2, t.TestToken) + return e +} + +func (t *Test) WithAuth(req *httpexpect.Request) *httpexpect.Request { + return req.WithHeader("Authorization", t.TestToken) +} + +var T *Test + +func NewTest() (res *Test, err error) { + // test + t := &Test{} + + // gin app + t.app = gin.New() + + // http test server + t.svr = httptest.NewServer(t.app) + + // init controllers + if err := controllers.InitControllers(); err != nil { + return nil, err + } + + // init routes + if err := routes.InitRoutes(t.app); err != nil { + return nil, err + } + + // dependency injection + c := dig.New() + if err := c.Provide(service.NewService); err != nil { + return nil, trace.TraceError(err) + } + if err := c.Invoke(func(modelSvc service.ModelService) { + t.modelSvc = modelSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + // test data + t.TestUsername = "admin" + t.TestPassword = "admin" + + return t, nil +} diff --git a/core/controllers/test/export_test.go b/core/controllers/test/export_test.go new file mode 100644 index 000000000..6c11e9133 --- /dev/null +++ b/core/controllers/test/export_test.go @@ -0,0 +1,59 @@ +package test + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "net/http" + "testing" + "time" +) + +func init() { + viper.Set("mongo.db", "crawlab_test") +} + +func TestExportController_Csv(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + // mongo collection + colName := "test_collection_for_export" + col := mongo.GetMongoCol(colName) + + // insert test data to mongo collection + for i := 0; i < 10; i++ { + _, err := col.Insert(bson.M{ + "field1": i + 1, + "field2": i + 2, + "field3": i + 3, + "field4": i + 4, + }) + require.Nil(t, err) + } + + // export from mongo collection + res := T.WithAuth(e.POST("/export/csv")). + WithQuery("target", colName). + Expect().Status(http.StatusOK).JSON().Object() + res.Path("$.data").NotNull() + + // export id + exportId := res.Path("$.data").String().Raw() + + // poll export with export id + for i := 0; i < 10; i++ { + res = T.WithAuth(e.GET("/export/csv/" + exportId)).Expect().Status(http.StatusOK).JSON().Object() + status := res.Path("$.data.status").String().Raw() + if status == constants.TaskStatusFinished { + break + } + time.Sleep(1 * time.Second) + } + + // download exported csv file + csvFileBody := T.WithAuth(e.GET("/export/csv/" + exportId + "/download")).Expect().Status(http.StatusOK).Body() + csvFileBody.NotEmpty() +} diff --git a/core/controllers/test/filter_test.go b/core/controllers/test/filter_test.go new file mode 100644 index 000000000..466d93b16 --- /dev/null +++ b/core/controllers/test/filter_test.go @@ -0,0 +1,76 @@ +package test + +import ( + "encoding/json" + "fmt" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "net/http" + "testing" +) + +func init() { + viper.Set("mongo.db", "crawlab_test") +} + +func TestFilterController_GetColFieldOptions(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + // mongo collection + colName := "test_collection_for_filter" + field1 := "field1" + field2 := "field2" + value1 := "value1" + col := mongo.GetMongoCol(colName) + n := 10 + var ids []primitive.ObjectID + var names []string + for i := 0; i < n; i++ { + _id := primitive.NewObjectID() + ids = append(ids, _id) + name := fmt.Sprintf("name_%d", i) + names = append(names, name) + _, err := col.Insert(bson.M{field1: value1, field2: i % 2, "name": name, "_id": _id}) + require.Nil(t, err) + } + + // validate filter options field 1 + res := T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field1, field1))). + Expect().Status(http.StatusOK).JSON().Object() + res.Path("$.data").NotNull() + res.Path("$.data").Array().Length().Equal(1) + res.Path("$.data").Array().Element(0).Path("$.value").Equal(value1) + + // validate filter options field 2 + res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))). + Expect().Status(http.StatusOK).JSON().Object() + res.Path("$.data").NotNull() + res.Path("$.data").Array().Length().Equal(2) + + // validate filter options with query + conditions := []entity.Condition{{field2, constants.FilterOpEqual, 0}} + conditionsJson, err := json.Marshal(conditions) + conditionsJsonStr := string(conditionsJson) + require.Nil(t, err) + res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))). + WithQuery(constants.FilterQueryFieldConditions, conditionsJsonStr). + Expect().Status(http.StatusOK).JSON().Object() + res.Path("$.data").NotNull() + res.Path("$.data").Array().Length().Equal(1) + + // validate filter options (basic path) + res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s", colName))). + Expect().Status(http.StatusOK).JSON().Object() + res.Path("$.data").NotNull() + res.Path("$.data").Array().Length().Equal(n) + for i := 0; i < n; i++ { + res.Path("$.data").Array().Element(i).Object().Value("value").Equal(ids[i]) + res.Path("$.data").Array().Element(i).Object().Value("label").Equal(names[i]) + } +} diff --git a/core/controllers/test/main_test.go b/core/controllers/test/main_test.go new file mode 100644 index 000000000..2b9bb76bf --- /dev/null +++ b/core/controllers/test/main_test.go @@ -0,0 +1,44 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/controllers" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/user" + "go.mongodb.org/mongo-driver/mongo" + "testing" +) + +func TestMain(m *testing.M) { + // init user + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + _, err = modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil) + if err != nil { + if err.Error() != mongo.ErrNoDocuments.Error() { + panic(err) + } + userSvc, err := user.GetUserService() + if err != nil { + panic(err) + } + if err := userSvc.Create(&interfaces.UserCreateOptions{ + Username: constants.DefaultAdminUsername, + Password: constants.DefaultAdminPassword, + Role: constants.RoleAdmin, + }); err != nil { + panic(err) + } + } + + if err := controllers.InitControllers(); err != nil { + panic(err) + } + + m.Run() + + T.Cleanup() +} diff --git a/core/controllers/test/project_test.go b/core/controllers/test/project_test.go new file mode 100644 index 000000000..b0a1b2c68 --- /dev/null +++ b/core/controllers/test/project_test.go @@ -0,0 +1,305 @@ +package test + +import ( + "encoding/json" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "net/http" + "testing" + "time" +) + +func TestProjectController_Get(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + p := models.Project{ + Name: "test project", + } + res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() + res.Path("$.data._id").NotNull() + id := res.Path("$.data._id").String().Raw() + oid, err := primitive.ObjectIDFromHex(id) + require.Nil(t, err) + require.False(t, oid.IsZero()) + + res = T.WithAuth(e.GET("/projects/" + id)).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() + res.Path("$.data._id").NotNull() + res.Path("$.data.name").Equal("test project") +} + +func TestProjectController_Put(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + p := models.Project{ + Name: "old name", + Description: "old description", + } + + // add + res := T.WithAuth(e.POST("/projects")). + WithJSON(p). + Expect().Status(http.StatusOK). + JSON().Object() + res.Path("$.data._id").NotNull() + id := res.Path("$.data._id").String().Raw() + oid, err := primitive.ObjectIDFromHex(id) + require.Nil(t, err) + require.False(t, oid.IsZero()) + + // change object + p.Id = oid + p.Name = "new name" + p.Description = "new description" + + // update + T.WithAuth(e.PUT("/projects/" + id)). + WithJSON(p). + Expect().Status(http.StatusOK) + + // check + res = T.WithAuth(e.GET("/projects/" + id)).Expect().Status(http.StatusOK).JSON().Object() + res.Path("$.data._id").Equal(id) + res.Path("$.data.name").Equal("new name") + res.Path("$.data.description").Equal("new description") +} + +func TestProjectController_Post(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + p := models.Project{ + Name: "test project", + Description: "this is a test project", + } + + res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() + res.Path("$.data._id").NotNull() + res.Path("$.data.name").Equal("test project") + res.Path("$.data.description").Equal("this is a test project") +} + +func TestProjectController_Delete(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + p := models.Project{ + Name: "test project", + Description: "this is a test project", + } + + // add + res := T.WithAuth(e.POST("/projects")). + WithJSON(p). + Expect().Status(http.StatusOK). + JSON().Object() + res.Path("$.data._id").NotNull() + id := res.Path("$.data._id").String().Raw() + oid, err := primitive.ObjectIDFromHex(id) + require.Nil(t, err) + require.False(t, oid.IsZero()) + + // get + res = T.WithAuth(e.GET("/projects/" + id)). + Expect().Status(http.StatusOK). + JSON().Object() + res.Path("$.data._id").NotNull() + id = res.Path("$.data._id").String().Raw() + oid, err = primitive.ObjectIDFromHex(id) + require.Nil(t, err) + require.False(t, oid.IsZero()) + + // delete + T.WithAuth(e.DELETE("/projects/" + id)). + Expect().Status(http.StatusOK). + JSON().Object() + + // get + T.WithAuth(e.GET("/projects/" + id)). + Expect().Status(http.StatusNotFound) +} + +func TestProjectController_GetList(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + n := 100 // total + bn := 10 // batch + + for i := 0; i < n; i++ { + p := models.Project{ + Name: fmt.Sprintf("test name %d", i+1), + } + obj := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() + obj.Path("$.data._id").NotNull() + } + + f := entity.Filter{ + //IsOr: false, + Conditions: []*entity.Condition{ + {Key: "name", Op: constants.FilterOpContains, Value: "test name"}, + }, + } + condBytes, err := json.Marshal(&f.Conditions) + require.Nil(t, err) + + pagination := entity.Pagination{ + Page: 1, + Size: bn, + } + + // get list with pagination + res := T.WithAuth(e.GET("/projects")). + WithQuery("conditions", string(condBytes)). + WithQueryObject(pagination). + Expect().Status(http.StatusOK).JSON().Object() + res.Path("$.data").Array().Length().Equal(bn) + res.Path("$.total").Number().Equal(n) + + data := res.Path("$.data").Array() + for i := 0; i < bn; i++ { + obj := data.Element(i) + obj.Path("$.name").Equal(fmt.Sprintf("test name %d", i+1)) + } + +} + +func TestProjectController_PostList(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + n := 10 + var docs []models.Project + for i := 0; i < n; i++ { + docs = append(docs, models.Project{ + Name: fmt.Sprintf("project %d", i+1), + Description: "this is a project", + }) + } + + T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK) + + res := T.WithAuth(e.GET("/projects")). + WithQueryObject(entity.Pagination{Page: 1, Size: 10}). + Expect().Status(http.StatusOK). + JSON().Object() + res.Path("$.data").Array().Length().Equal(n) +} + +func TestProjectController_DeleteList(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + n := 10 + var docs []models.Project + for i := 0; i < n; i++ { + docs = append(docs, models.Project{ + Name: fmt.Sprintf("project %d", i+1), + Description: "this is a project", + }) + } + + // add + res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK). + JSON().Object() + var ids []primitive.ObjectID + data := res.Path("$.data").Array() + for i := 0; i < n; i++ { + obj := data.Element(i) + id := obj.Path("$._id").String().Raw() + oid, err := primitive.ObjectIDFromHex(id) + require.Nil(t, err) + require.False(t, oid.IsZero()) + ids = append(ids, oid) + } + + // delete + payload := entity.BatchRequestPayload{ + Ids: ids, + } + T.WithAuth(e.DELETE("/projects")). + WithJSON(payload). + Expect().Status(http.StatusOK) + + // check + for _, id := range ids { + T.WithAuth(e.GET("/projects/" + id.Hex())). + Expect().Status(http.StatusNotFound) + } + +} + +func TestProjectController_PutList(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + // now + now := time.Now() + + n := 10 + var docs []models.Project + for i := 0; i < n; i++ { + docs = append(docs, models.Project{ + Name: "old name", + Description: "old description", + }) + } + + // add + res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK). + JSON().Object() + var ids []primitive.ObjectID + data := res.Path("$.data").Array() + for i := 0; i < n; i++ { + obj := data.Element(i) + id := obj.Path("$._id").String().Raw() + oid, err := primitive.ObjectIDFromHex(id) + require.Nil(t, err) + require.False(t, oid.IsZero()) + ids = append(ids, oid) + } + + // wait for 100 millisecond + time.Sleep(100 * time.Millisecond) + + // update + p := models.Project{ + Name: "new name", + Description: "new description", + } + dataBytes, err := json.Marshal(&p) + require.Nil(t, err) + payload := entity.BatchRequestPayloadWithStringData{ + Ids: ids, + Data: string(dataBytes), + Fields: []string{ + "name", + "description", + }, + } + T.WithAuth(e.PUT("/projects")).WithJSON(payload).Expect().Status(http.StatusOK) + + // check response data + for i := 0; i < n; i++ { + res = T.WithAuth(e.GET("/projects/" + ids[i].Hex())).Expect().Status(http.StatusOK).JSON().Object() + res.Path("$.data.name").Equal("new name") + res.Path("$.data.description").Equal("new description") + } + + // check artifacts + pl, err := T.modelSvc.GetProjectList(bson.M{"_id": bson.M{"$in": ids}}, nil) + require.Nil(t, err) + for _, p := range pl { + a, err := delegate.NewModelDelegate(&p).GetArtifact() + require.Nil(t, err) + require.True(t, a.GetSys().GetUpdateTs().After(now)) + require.True(t, a.GetSys().GetUpdateTs().After(a.GetSys().GetCreateTs())) + } +} diff --git a/core/controllers/test/spider_test.go b/core/controllers/test/spider_test.go new file mode 100644 index 000000000..b8c4dc06b --- /dev/null +++ b/core/controllers/test/spider_test.go @@ -0,0 +1,183 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "net/http" + "testing" +) + +func TestSpiderController_Delete(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + s := models.Spider{ + Name: "test spider", + Description: "this is a test spider", + ColName: "test col name", + } + + // add spider + res := T.WithAuth(e.POST("/spiders")). + WithJSON(s). + Expect().Status(http.StatusOK). + JSON().Object() + res.Path("$.data._id").NotNull() + id := res.Path("$.data._id").String().Raw() + oid, err := primitive.ObjectIDFromHex(id) + require.Nil(t, err) + require.False(t, oid.IsZero()) + + // add tasks + var taskIds []primitive.ObjectID + tasks := []models.Task{ + { + Id: primitive.NewObjectID(), + SpiderId: oid, + }, + { + Id: primitive.NewObjectID(), + SpiderId: oid, + }, + } + for _, task := range tasks { + // add task + err := delegate.NewModelDelegate(&task).Add() + require.Nil(t, err) + + // add task stat + err = delegate.NewModelDelegate(&models.TaskStat{ + Id: task.Id, + }).Add() + require.Nil(t, err) + + taskIds = append(taskIds, task.Id) + } + + // delete + T.WithAuth(e.DELETE("/spiders/" + id)). + Expect().Status(http.StatusOK) + + // get + T.WithAuth(e.GET("/spiders/" + id)). + Expect().Status(http.StatusNotFound) + + // get tasks + for _, task := range tasks { + T.WithAuth(e.GET("/tasks/" + task.Id.Hex())). + Expect().Status(http.StatusNotFound) + } + + // spider stat + modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat) + spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{ + "_id": oid, + }) + require.Nil(t, err) + require.Zero(t, spiderStatCount) + + // task stats + modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) + taskStatCount, err := modelTaskStatSvc.Count(bson.M{ + "_id": bson.M{ + "$in": taskIds, + }, + }) + require.Nil(t, err) + require.Zero(t, taskStatCount) +} + +func TestSpiderController_DeleteList(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + spiders := []models.Spider{ + { + Id: primitive.NewObjectID(), + Name: "test spider 1", + Description: "this is a test spider 1", + ColName: "test col name 1", + }, + { + Id: primitive.NewObjectID(), + Name: "test spider 2", + Description: "this is a test spider 2", + ColName: "test col name 2", + }, + } + + // add spiders + for _, spider := range spiders { + T.WithAuth(e.POST("/spiders")). + WithJSON(spider). + Expect().Status(http.StatusOK) + } + + var spiderIds []primitive.ObjectID + var taskIds []primitive.ObjectID + for _, spider := range spiders { + // task id + taskId := primitive.NewObjectID() + + // add task + err := delegate.NewModelDelegate(&models.Task{ + Id: taskId, + SpiderId: spider.Id, + }).Add() + require.Nil(t, err) + + // add task stats + err = delegate.NewModelDelegate(&models.TaskStat{ + Id: taskId, + }).Add() + require.Nil(t, err) + + spiderIds = append(spiderIds, spider.Id) + taskIds = append(taskIds, taskId) + } + + // delete spiders + T.WithAuth(e.DELETE("/spiders")). + WithJSON(entity.BatchRequestPayload{ + Ids: spiderIds, + }).Expect().Status(http.StatusOK) + + // get spiders + for _, spider := range spiders { + // get + T.WithAuth(e.GET("/spiders/" + spider.Id.Hex())). + Expect().Status(http.StatusNotFound) + } + + // get tasks + for _, taskId := range taskIds { + T.WithAuth(e.GET("/tasks/" + taskId.Hex())). + Expect().Status(http.StatusNotFound) + } + + // spider stat + modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat) + spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{ + "_id": bson.M{ + "$in": spiderIds, + }, + }) + require.Nil(t, err) + require.Zero(t, spiderStatCount) + + // task stats + modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) + taskStatCount, err := modelTaskStatSvc.Count(bson.M{ + "_id": bson.M{ + "$in": taskIds, + }, + }) + require.Nil(t, err) + require.Zero(t, taskStatCount) +} diff --git a/core/controllers/test/task_test.go b/core/controllers/test/task_test.go new file mode 100644 index 000000000..4d81738b8 --- /dev/null +++ b/core/controllers/test/task_test.go @@ -0,0 +1,102 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "net/http" + "testing" +) + +func TestTaskController_Delete(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + task := models.Task{ + Id: primitive.NewObjectID(), + } + + // add task + err := delegate.NewModelDelegate(&task).Add() + require.Nil(t, err) + + // add task stat + err = delegate.NewModelDelegate(&models.TaskStat{ + Id: task.Id, + }).Add() + require.Nil(t, err) + + // delete + T.WithAuth(e.DELETE("/tasks/" + task.Id.Hex())). + Expect().Status(http.StatusOK) + + // get + T.WithAuth(e.GET("/tasks/" + task.Id.Hex())). + Expect().Status(http.StatusNotFound) + + // task stats + modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) + taskStatCount, err := modelTaskStatSvc.Count(bson.M{ + "_id": task.Id, + }) + require.Nil(t, err) + require.Zero(t, taskStatCount) +} + +func TestTaskController_DeleteList(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + + tasks := []models.Task{ + { + Id: primitive.NewObjectID(), + }, + { + Id: primitive.NewObjectID(), + }, + } + + // add spiders + var taskIds []primitive.ObjectID + for _, task := range tasks { + // add task + err := delegate.NewModelDelegate(&task).Add() + require.Nil(t, err) + + // add task stat + err = delegate.NewModelDelegate(&models.TaskStat{ + Id: task.Id, + }).Add() + require.Nil(t, err) + + taskIds = append(taskIds, task.Id) + } + + // delete tasks + T.WithAuth(e.DELETE("/tasks")). + WithJSON(entity.BatchRequestPayload{ + Ids: taskIds, + }).Expect().Status(http.StatusOK) + + // get tasks + for _, task := range tasks { + // get + T.WithAuth(e.GET("/tasks/" + task.Id.Hex())). + Expect().Status(http.StatusNotFound) + } + + // task stats + modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) + taskStatCount, err := modelTaskStatSvc.Count(bson.M{ + "_id": bson.M{ + "$in": taskIds, + }, + }) + require.Nil(t, err) + require.Zero(t, taskStatCount) +} diff --git a/core/controllers/token.go b/core/controllers/token.go new file mode 100644 index 000000000..f94f3b688 --- /dev/null +++ b/core/controllers/token.go @@ -0,0 +1,84 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/gin-gonic/gin" +) + +var TokenController *tokenController + +var TokenActions []Action + +type tokenController struct { + ListActionControllerDelegate + d ListActionControllerDelegate + ctx *tokenContext +} + +func (ctr *tokenController) Post(c *gin.Context) { + var err error + var t models.Token + if err := c.ShouldBindJSON(&t); err != nil { + HandleErrorBadRequest(c, err) + return + } + u, err := ctr.ctx.userSvc.GetCurrentUser(c) + if err != nil { + HandleErrorUnauthorized(c, err) + return + } + t.Token, err = ctr.ctx.userSvc.MakeToken(u) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + if err := delegate.NewModelDelegate(&t, GetUserFromContext(c)).Add(); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +type tokenContext struct { + modelSvc service.ModelService + userSvc interfaces.UserService +} + +func newTokenContext() *tokenContext { + // context + ctx := &tokenContext{} + + // dependency injection + if err := container.GetContainer().Invoke(func( + modelSvc service.ModelService, + userSvc interfaces.UserService, + ) { + ctx.modelSvc = modelSvc + ctx.userSvc = userSvc + }); err != nil { + panic(err) + } + + return ctx +} + +func newTokenController() *tokenController { + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + + ctr := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions) + d := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions) + ctx := newTokenContext() + + return &tokenController{ + ListActionControllerDelegate: *ctr, + d: *d, + ctx: ctx, + } +} diff --git a/core/controllers/token_v2.go b/core/controllers/token_v2.go new file mode 100644 index 000000000..bac880b2c --- /dev/null +++ b/core/controllers/token_v2.go @@ -0,0 +1,35 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/user" + "github.com/gin-gonic/gin" +) + +func PostToken(c *gin.Context) { + var t models.TokenV2 + if err := c.ShouldBindJSON(&t); err != nil { + HandleErrorBadRequest(c, err) + return + } + svc, err := user.GetUserServiceV2() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + u := GetUserFromContextV2(c) + t.SetCreated(u.Id) + t.SetUpdated(u.Id) + t.Token, err = svc.MakeToken(u) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + _, err = service.NewModelServiceV2[models.TokenV2]().InsertOne(t) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} diff --git a/core/controllers/user.go b/core/controllers/user.go new file mode 100644 index 000000000..e243b02c7 --- /dev/null +++ b/core/controllers/user.go @@ -0,0 +1,244 @@ +package controllers + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "net/http" +) + +var UserController *userController + +func getUserActions() []Action { + userCtx := newUserContext() + return []Action{ + { + Method: http.MethodPost, + Path: "/:id/change-password", + HandlerFunc: userCtx.changePassword, + }, + { + Method: http.MethodGet, + Path: "/me", + HandlerFunc: userCtx.getMe, + }, + { + Method: http.MethodPut, + Path: "/me", + HandlerFunc: userCtx.putMe, + }, + } +} + +type userController struct { + ListActionControllerDelegate + d ListActionControllerDelegate + ctx *userContext +} + +func (ctr *userController) Post(c *gin.Context) { + var u models.User + if err := c.ShouldBindJSON(&u); err != nil { + HandleErrorBadRequest(c, err) + return + } + if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{ + Username: u.Username, + Password: u.Password, + Email: u.Email, + Role: u.Role, + }); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +func (ctr *userController) PostList(c *gin.Context) { + // users + var users []models.User + if err := c.ShouldBindJSON(&users); err != nil { + HandleErrorBadRequest(c, err) + return + } + + for _, u := range users { + if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{ + Username: u.Username, + Password: u.Password, + Email: u.Email, + Role: u.Role, + }); err != nil { + trace.PrintError(err) + } + } + + HandleSuccess(c) +} + +func (ctr *userController) PutList(c *gin.Context) { + // payload + var payload entity.BatchRequestPayloadWithStringData + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // doc to update + var doc models.User + if err := json.Unmarshal([]byte(payload.Data), &doc); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // query + query := bson.M{ + "_id": bson.M{ + "$in": payload.Ids, + }, + } + + // update users + if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdUser).UpdateDoc(query, &doc, payload.Fields); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // update passwords + if utils.Contains(payload.Fields, "password") { + for _, id := range payload.Ids { + if err := ctr.ctx.userSvc.ChangePassword(id, doc.Password); err != nil { + trace.PrintError(err) + } + } + } + + HandleSuccess(c) +} + +type userContext struct { + modelSvc service.ModelService + userSvc interfaces.UserService +} + +func (ctx *userContext) changePassword(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + var payload map[string]string + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + password, ok := payload["password"] + if !ok { + HandleErrorBadRequest(c, errors.ErrorUserMissingRequiredFields) + return + } + if len(password) < 5 { + HandleErrorBadRequest(c, errors.ErrorUserInvalidPassword) + return + } + if err := ctx.userSvc.ChangePassword(id, password); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +func (ctx *userContext) getMe(c *gin.Context) { + u, err := ctx._getMe(c) + if err != nil { + HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) + return + } + HandleSuccessWithData(c, u) +} + +func (ctx *userContext) putMe(c *gin.Context) { + // current user + u, err := ctx._getMe(c) + if err != nil { + HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) + return + } + + // payload + doc, err := NewJsonBinder(ControllerIdUser).Bind(c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + if doc.GetId() != u.GetId() { + HandleErrorBadRequest(c, errors.ErrorHttpBadRequest) + return + } + + // save to db + if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Save(); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, doc) +} + +func (ctx *userContext) _getMe(c *gin.Context) (u interfaces.User, err error) { + res, ok := c.Get(constants.UserContextKey) + if !ok { + return nil, trace.TraceError(errors.ErrorUserNotExistsInContext) + } + u, ok = res.(interfaces.User) + if !ok { + return nil, trace.TraceError(errors.ErrorUserInvalidType) + } + return u, nil +} + +func newUserContext() *userContext { + // context + ctx := &userContext{} + + // dependency injection + if err := container.GetContainer().Invoke(func( + modelSvc service.ModelService, + userSvc interfaces.UserService, + ) { + ctx.modelSvc = modelSvc + ctx.userSvc = userSvc + }); err != nil { + panic(err) + } + + return ctx +} + +func newUserController() *userController { + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + + ctr := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions()) + d := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions()) + ctx := newUserContext() + + return &userController{ + ListActionControllerDelegate: *ctr, + d: *d, + ctx: ctx, + } +} diff --git a/core/controllers/user_v2.go b/core/controllers/user_v2.go new file mode 100644 index 000000000..434b20614 --- /dev/null +++ b/core/controllers/user_v2.go @@ -0,0 +1,126 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func PostUser(c *gin.Context) { + var payload struct { + Username string `json:"username"` + Password string `json:"password"` + Role string `json:"role"` + Email string `json:"email"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + u := GetUserFromContextV2(c) + model := models.UserV2{ + Username: payload.Username, + Password: utils.EncryptMd5(payload.Password), + Role: payload.Role, + Email: payload.Email, + } + model.SetCreated(u.Id) + model.SetUpdated(u.Id) + id, err := service.NewModelServiceV2[models.UserV2]().InsertOne(model) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + result, err := service.NewModelServiceV2[models.UserV2]().GetById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, result) + +} + +func PostUserChangePassword(c *gin.Context) { + // get id + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // get payload + var payload struct { + Password string `json:"password"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // get user + u := GetUserFromContextV2(c) + modelSvc := service.NewModelServiceV2[models.UserV2]() + + // update password + user, err := modelSvc.GetById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + user.SetUpdated(u.Id) + user.Password = utils.EncryptMd5(payload.Password) + if err := modelSvc.ReplaceById(user.Id, *user); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // handle success + HandleSuccess(c) +} + +func GetUserMe(c *gin.Context) { + u := GetUserFromContextV2(c) + _u, err := service.NewModelServiceV2[models.UserV2]().GetById(u.Id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccessWithData(c, _u) +} + +func PutUserById(c *gin.Context) { + // get payload + var user models.UserV2 + if err := c.ShouldBindJSON(&user); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // get user + u := GetUserFromContextV2(c) + + modelSvc := service.NewModelServiceV2[models.UserV2]() + + // update user + userDb, err := modelSvc.GetById(u.Id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + user.Password = userDb.Password + user.SetUpdated(u.Id) + if user.Id.IsZero() { + user.Id = u.Id + } + if err := modelSvc.ReplaceById(u.Id, user); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // handle success + HandleSuccess(c) +} diff --git a/core/controllers/user_v2_test.go b/core/controllers/user_v2_test.go new file mode 100644 index 000000000..c86739fc6 --- /dev/null +++ b/core/controllers/user_v2_test.go @@ -0,0 +1,89 @@ +package controllers_test + +import ( + "github.com/crawlab-team/crawlab/core/controllers" + "github.com/crawlab-team/crawlab/core/middlewares" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestPostUserChangePassword_Success(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + modelSvc := service.NewModelServiceV2[models.UserV2]() + u := models.UserV2{} + id, err := modelSvc.InsertOne(u) + assert.Nil(t, err) + u.SetId(id) + + router := gin.Default() + router.Use(middlewares.AuthorizationMiddlewareV2()) + router.POST("/users/:id/change-password", controllers.PostUserChangePassword) + + password := "newPassword" + reqBody := strings.NewReader(`{"password":"` + password + `"}`) + req, _ := http.NewRequest(http.MethodPost, "/users/"+id.Hex()+"/change-password", reqBody) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", TestToken) + + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) +} + +func TestGetUserMe_Success(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + modelSvc := service.NewModelServiceV2[models.UserV2]() + u := models.UserV2{} + id, err := modelSvc.InsertOne(u) + assert.Nil(t, err) + u.SetId(id) + + router := gin.Default() + router.Use(middlewares.AuthorizationMiddlewareV2()) + router.GET("/users/me", controllers.GetUserMe) + + req, _ := http.NewRequest(http.MethodGet, "/users/me", nil) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", TestToken) + + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) +} + +func TestPutUserById_Success(t *testing.T) { + SetupTestDB() + defer CleanupTestDB() + + modelSvc := service.NewModelServiceV2[models.UserV2]() + u := models.UserV2{} + id, err := modelSvc.InsertOne(u) + assert.Nil(t, err) + u.SetId(id) + + router := gin.Default() + router.Use(middlewares.AuthorizationMiddlewareV2()) + router.PUT("/users/me", controllers.PutUserById) + + reqBody := strings.NewReader(`{"id":"` + id.Hex() + `","username":"newUsername","email":"newEmail@test.com"}`) + req, _ := http.NewRequest(http.MethodPut, "/users/me", reqBody) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", TestToken) + + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) +} diff --git a/core/controllers/utils_context.go b/core/controllers/utils_context.go new file mode 100644 index 000000000..9365bd0de --- /dev/null +++ b/core/controllers/utils_context.go @@ -0,0 +1,32 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/gin-gonic/gin" +) + +func GetUserFromContext(c *gin.Context) (u interfaces.User) { + value, ok := c.Get(constants.UserContextKey) + if !ok { + return nil + } + u, ok = value.(interfaces.User) + if !ok { + return nil + } + return u +} + +func GetUserFromContextV2(c *gin.Context) (u *models.UserV2) { + value, ok := c.Get(constants.UserContextKey) + if !ok { + return nil + } + u, ok = value.(*models.UserV2) + if !ok { + return nil + } + return u +} diff --git a/core/controllers/utils_filter.go b/core/controllers/utils_filter.go new file mode 100644 index 000000000..9a7b619cf --- /dev/null +++ b/core/controllers/utils_filter.go @@ -0,0 +1,123 @@ +package controllers + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "reflect" + "strings" +) + +// GetFilter Get entity.Filter from gin.Context +func GetFilter(c *gin.Context) (f *entity.Filter, err error) { + // bind + condStr := c.Query(constants.FilterQueryFieldConditions) + var conditions []*entity.Condition + if err := json.Unmarshal([]byte(condStr), &conditions); err != nil { + return nil, err + } + + // attempt to convert object id + for i, cond := range conditions { + v := reflect.ValueOf(cond.Value) + switch v.Kind() { + case reflect.String: + item := cond.Value.(string) + id, err := primitive.ObjectIDFromHex(item) + if err == nil { + conditions[i].Value = id + } else { + conditions[i].Value = item + } + case reflect.Slice, reflect.Array: + var items []interface{} + for i := 0; i < v.Len(); i++ { + vItem := v.Index(i) + item := vItem.Interface() + + // string + stringItem, ok := item.(string) + if ok { + id, err := primitive.ObjectIDFromHex(stringItem) + if err == nil { + items = append(items, id) + } else { + items = append(items, stringItem) + } + continue + } + + // default + items = append(items, item) + } + conditions[i].Value = items + } + } + + return &entity.Filter{ + IsOr: false, + Conditions: conditions, + }, nil +} + +// GetFilterQuery Get bson.M from gin.Context +func GetFilterQuery(c *gin.Context) (q bson.M, err error) { + f, err := GetFilter(c) + if err != nil { + return nil, err + } + + if f == nil { + return nil, nil + } + + // TODO: implement logic OR + + return utils.FilterToQuery(f) +} + +func MustGetFilterQuery(c *gin.Context) (q bson.M) { + q, err := GetFilterQuery(c) + if err != nil { + return nil + } + return q +} + +// GetFilterAll Get all from gin.Context +func GetFilterAll(c *gin.Context) (res bool, err error) { + resStr := c.Query(constants.FilterQueryFieldAll) + switch strings.ToUpper(resStr) { + case "1": + return true, nil + case "0": + return false, nil + case "Y": + return true, nil + case "N": + return false, nil + case "T": + return true, nil + case "F": + return false, nil + case "TRUE": + return true, nil + case "FALSE": + return false, nil + default: + return false, errors.ErrorFilterInvalidOperation + } +} + +func MustGetFilterAll(c *gin.Context) (res bool) { + res, err := GetFilterAll(c) + if err != nil { + return false + } + return res +} diff --git a/core/controllers/utils_http.go b/core/controllers/utils_http.go new file mode 100644 index 000000000..60ce95eba --- /dev/null +++ b/core/controllers/utils_http.go @@ -0,0 +1,72 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "net/http" +) + +func handleError(statusCode int, c *gin.Context, err error, print bool) { + if print { + trace.PrintError(err) + } + c.AbortWithStatusJSON(statusCode, entity.Response{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageError, + Error: err.Error(), + }) +} + +func HandleError(statusCode int, c *gin.Context, err error) { + handleError(statusCode, c, err, true) +} + +func HandleErrorNoPrint(statusCode int, c *gin.Context, err error) { + handleError(statusCode, c, err, false) +} + +func HandleErrorBadRequest(c *gin.Context, err error) { + HandleError(http.StatusBadRequest, c, err) +} + +func HandleErrorUnauthorized(c *gin.Context, err error) { + HandleError(http.StatusUnauthorized, c, err) +} + +func HandleErrorNotFound(c *gin.Context, err error) { + HandleError(http.StatusNotFound, c, err) +} + +func HandleErrorNotFoundNoPrint(c *gin.Context, err error) { + HandleErrorNoPrint(http.StatusNotFound, c, err) +} + +func HandleErrorInternalServerError(c *gin.Context, err error) { + HandleError(http.StatusInternalServerError, c, err) +} + +func HandleSuccess(c *gin.Context) { + c.AbortWithStatusJSON(http.StatusOK, entity.Response{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageSuccess, + }) +} + +func HandleSuccessWithData(c *gin.Context, data interface{}) { + c.AbortWithStatusJSON(http.StatusOK, entity.Response{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageSuccess, + Data: data, + }) +} + +func HandleSuccessWithListData(c *gin.Context, data interface{}, total int) { + c.AbortWithStatusJSON(http.StatusOK, entity.ListResponse{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageSuccess, + Data: data, + Total: total, + }) +} diff --git a/core/controllers/utils_pagination.go b/core/controllers/utils_pagination.go new file mode 100644 index 000000000..d16278af8 --- /dev/null +++ b/core/controllers/utils_pagination.go @@ -0,0 +1,30 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/gin-gonic/gin" +) + +func GetDefaultPagination() (p *entity.Pagination) { + return &entity.Pagination{ + Page: constants.PaginationDefaultPage, + Size: constants.PaginationDefaultSize, + } +} + +func GetPagination(c *gin.Context) (p *entity.Pagination, err error) { + var _p entity.Pagination + if err := c.ShouldBindQuery(&_p); err != nil { + return GetDefaultPagination(), err + } + return &_p, nil +} + +func MustGetPagination(c *gin.Context) (p *entity.Pagination) { + p, err := GetPagination(c) + if err != nil || p == nil { + return GetDefaultPagination() + } + return p +} diff --git a/core/controllers/utils_sort.go b/core/controllers/utils_sort.go new file mode 100644 index 000000000..86e68e96d --- /dev/null +++ b/core/controllers/utils_sort.go @@ -0,0 +1,58 @@ +package controllers + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" +) + +// GetSorts Get entity.Sort from gin.Context +func GetSorts(c *gin.Context) (sorts []entity.Sort, err error) { + // bind + sortStr := c.Query(constants.SortQueryField) + if err := json.Unmarshal([]byte(sortStr), &sorts); err != nil { + return nil, err + } + return sorts, nil +} + +// GetSortsOption Get entity.Sort from gin.Context +func GetSortsOption(c *gin.Context) (sort bson.D, err error) { + sorts, err := GetSorts(c) + if err != nil { + return nil, err + } + + if sorts == nil || len(sorts) == 0 { + return bson.D{{"_id", -1}}, nil + } + + return SortsToOption(sorts) +} + +func MustGetSortOption(c *gin.Context) (sort bson.D) { + sort, err := GetSortsOption(c) + if err != nil { + return nil + } + return sort +} + +// SortsToOption Translate entity.Sort to bson.D +func SortsToOption(sorts []entity.Sort) (sort bson.D, err error) { + sort = bson.D{} + for _, s := range sorts { + switch s.Direction { + case constants.ASCENDING: + sort = append(sort, bson.E{Key: s.Key, Value: 1}) + case constants.DESCENDING: + sort = append(sort, bson.E{Key: s.Key, Value: -1}) + } + } + if len(sort) == 0 { + sort = bson.D{{"_id", -1}} + } + return sort, nil +} diff --git a/core/controllers/version.go b/core/controllers/version.go new file mode 100644 index 000000000..83e3fe951 --- /dev/null +++ b/core/controllers/version.go @@ -0,0 +1,23 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/config" + "github.com/gin-gonic/gin" + "net/http" +) + +func GetVersion(c *gin.Context) { + HandleSuccessWithData(c, config.GetVersion()) +} + +func getVersionActions() []Action { + return []Action{ + { + Method: http.MethodGet, + Path: "", + HandlerFunc: GetVersion, + }, + } +} + +var VersionController ActionController diff --git a/core/data/colors.go b/core/data/colors.go new file mode 100644 index 000000000..6c97fb873 --- /dev/null +++ b/core/data/colors.go @@ -0,0 +1,5212 @@ +package data + +const ColorsDataText = `[ + { + "name": "Absolute Zero", + "hex": "#0048BA" + }, + { + "name": "Acid Green", + "hex": "#B0BF1A" + }, + { + "name": "Aero", + "hex": "#7CB9E8" + }, + { + "name": "Aero Blue", + "hex": "#C9FFE5" + }, + { + "name": "African Violet", + "hex": "#B284BE" + }, + { + "name": "Air Force Blue (RAF)", + "hex": "#5D8AA8" + }, + { + "name": "Air Force Blue (USAF)", + "hex": "#00308F" + }, + { + "name": "Air Superiority Blue", + "hex": "#72A0C1" + }, + { + "name": "Alabama Crimson", + "hex": "#AF002A" + }, + { + "name": "Alabaster", + "hex": "#F2F0E6" + }, + { + "name": "Alice Blue", + "hex": "#F0F8FF" + }, + { + "name": "Alien Armpit", + "hex": "#84DE02" + }, + { + "name": "Alizarin Crimson", + "hex": "#E32636" + }, + { + "name": "Alloy Orange", + "hex": "#C46210" + }, + { + "name": "Almond", + "hex": "#EFDECD" + }, + { + "name": "Amaranth", + "hex": "#E52B50" + }, + { + "name": "Amaranth Deep Purple", + "hex": "#9F2B68" + }, + { + "name": "Amaranth Pink", + "hex": "#F19CBB" + }, + { + "name": "Amaranth Purple", + "hex": "#AB274F" + }, + { + "name": "Amaranth Red", + "hex": "#D3212D" + }, + { + "name": "Amazon Store", + "hex": "#3B7A57" + }, + { + "name": "Amazonite", + "hex": "#00C4B0" + }, + { + "name": "Amber", + "hex": "#FFBF00" + }, + { + "name": "Amber (SAE/ECE)", + "hex": "#FF7E00" + }, + { + "name": "American Rose", + "hex": "#FF033E" + }, + { + "name": "Amethyst", + "hex": "#9966CC" + }, + { + "name": "Android Green", + "hex": "#A4C639" + }, + { + "name": "Anti-Flash White", + "hex": "#F2F3F4" + }, + { + "name": "Antique Brass", + "hex": "#CD9575" + }, + { + "name": "Antique Bronze", + "hex": "#665D1E" + }, + { + "name": "Antique Fuchsia", + "hex": "#915C83" + }, + { + "name": "Antique Ruby", + "hex": "#841B2D" + }, + { + "name": "Antique White", + "hex": "#FAEBD7" + }, + { + "name": "Ao (English)", + "hex": "#008000" + }, + { + "name": "Apple Green", + "hex": "#8DB600" + }, + { + "name": "Apricot", + "hex": "#FBCEB1" + }, + { + "name": "Aqua", + "hex": "#00FFFF" + }, + { + "name": "Aquamarine", + "hex": "#7FFFD4" + }, + { + "name": "Arctic Lime", + "hex": "#D0FF14" + }, + { + "name": "Army Green", + "hex": "#4B5320" + }, + { + "name": "Arsenic", + "hex": "#3B444B" + }, + { + "name": "Artichoke", + "hex": "#8F9779" + }, + { + "name": "Arylide Yellow", + "hex": "#E9D66B" + }, + { + "name": "Ash Gray", + "hex": "#B2BEB5" + }, + { + "name": "Asparagus", + "hex": "#87A96B" + }, + { + "name": "Atomic Tangerine", + "hex": "#FF9966" + }, + { + "name": "Auburn", + "hex": "#A52A2A" + }, + { + "name": "Aureolin", + "hex": "#FDEE00" + }, + { + "name": "AuroMetalSaurus", + "hex": "#6E7F80" + }, + { + "name": "Avocado", + "hex": "#568203" + }, + { + "name": "Awesome", + "hex": "#FF2052" + }, + { + "name": "Aztec Gold", + "hex": "#C39953" + }, + { + "name": "Azure", + "hex": "#007FFF" + }, + { + "name": "Azure (Web Color)", + "hex": "#F0FFFF" + }, + { + "name": "Azure Mist", + "hex": "#F0FFFF" + }, + { + "name": "Azureish White", + "hex": "#DBE9F4" + }, + { + "name": "Baby Blue", + "hex": "#89CFF0" + }, + { + "name": "Baby Blue Eyes", + "hex": "#A1CAF1" + }, + { + "name": "Baby Pink", + "hex": "#F4C2C2" + }, + { + "name": "Baby Powder", + "hex": "#FEFEFA" + }, + { + "name": "Baker-Miller Pink", + "hex": "#FF91AF" + }, + { + "name": "Ball Blue", + "hex": "#21ABCD" + }, + { + "name": "Banana Mania", + "hex": "#FAE7B5" + }, + { + "name": "Banana Yellow", + "hex": "#FFE135" + }, + { + "name": "Bangladesh Green", + "hex": "#006A4E" + }, + { + "name": "Barbie Pink", + "hex": "#E0218A" + }, + { + "name": "Barn Red", + "hex": "#7C0A02" + }, + { + "name": "Battery Charged Blue", + "hex": "#1DACD6" + }, + { + "name": "Battleship Grey", + "hex": "#848482" + }, + { + "name": "Bazaar", + "hex": "#98777B" + }, + { + "name": "Beau Blue", + "hex": "#BCD4E6" + }, + { + "name": "Beaver", + "hex": "#9F8170" + }, + { + "name": "Begonia", + "hex": "#FA6E79" + }, + { + "name": "Beige", + "hex": "#F5F5DC" + }, + { + "name": "B'dazzled Blue", + "hex": "#2E5894" + }, + { + "name": "Big Dip O'ruby", + "hex": "#9C2542" + }, + { + "name": "Big Foot Feet", + "hex": "#E88E5A" + }, + { + "name": "Bisque", + "hex": "#FFE4C4" + }, + { + "name": "Bistre", + "hex": "#3D2B1F" + }, + { + "name": "Bistre Brown", + "hex": "#967117" + }, + { + "name": "Bitter Lemon", + "hex": "#CAE00D" + }, + { + "name": "Bitter Lime", + "hex": "#BFFF00" + }, + { + "name": "Bittersweet", + "hex": "#FE6F5E" + }, + { + "name": "Bittersweet Shimmer", + "hex": "#BF4F51" + }, + { + "name": "Black", + "hex": "#000000" + }, + { + "name": "Black Bean", + "hex": "#3D0C02" + }, + { + "name": "Black Coral", + "hex": "#54626F" + }, + { + "name": "Black Leather Jacket", + "hex": "#253529" + }, + { + "name": "Black Olive", + "hex": "#3B3C36" + }, + { + "name": "Black Shadows", + "hex": "#BFAFB2" + }, + { + "name": "Blanched Almond", + "hex": "#FFEBCD" + }, + { + "name": "Blast-Off Bronze", + "hex": "#A57164" + }, + { + "name": "Bleu De France", + "hex": "#318CE7" + }, + { + "name": "Blizzard Blue", + "hex": "#ACE5EE" + }, + { + "name": "Blond", + "hex": "#FAF0BE" + }, + { + "name": "Blue", + "hex": "#0000FF" + }, + { + "name": "Blue (Crayola)", + "hex": "#1F75FE" + }, + { + "name": "Blue (Munsell)", + "hex": "#0093AF" + }, + { + "name": "Blue (NCS)", + "hex": "#0087BD" + }, + { + "name": "Blue (Pantone)", + "hex": "#0018A8" + }, + { + "name": "Blue (Pigment)", + "hex": "#333399" + }, + { + "name": "Blue (RYB)", + "hex": "#0247FE" + }, + { + "name": "Blue Bell", + "hex": "#A2A2D0" + }, + { + "name": "Blue Bolt", + "hex": "#00B9FB" + }, + { + "name": "Blue-Gray", + "hex": "#6699CC" + }, + { + "name": "Blue-Green", + "hex": "#0D98BA" + }, + { + "name": "Blue Jeans", + "hex": "#5DADEC" + }, + { + "name": "Blue Lagoon", + "hex": "#ACE5EE" + }, + { + "name": "Blue-Magenta Violet", + "hex": "#553592" + }, + { + "name": "Blue Sapphire", + "hex": "#126180" + }, + { + "name": "Blue-Violet", + "hex": "#8A2BE2" + }, + { + "name": "Blue Yonder", + "hex": "#5072A7" + }, + { + "name": "Blueberry", + "hex": "#4F86F7" + }, + { + "name": "Bluebonnet", + "hex": "#1C1CF0" + }, + { + "name": "Blush", + "hex": "#DE5D83" + }, + { + "name": "Bole", + "hex": "#79443B" + }, + { + "name": "Bondi Blue", + "hex": "#0095B6" + }, + { + "name": "Bone", + "hex": "#E3DAC9" + }, + { + "name": "Booger Buster", + "hex": "#DDE26A" + }, + { + "name": "Boston University Red", + "hex": "#CC0000" + }, + { + "name": "Bottle Green", + "hex": "#006A4E" + }, + { + "name": "Boysenberry", + "hex": "#873260" + }, + { + "name": "Brandeis Blue", + "hex": "#0070FF" + }, + { + "name": "Brass", + "hex": "#B5A642" + }, + { + "name": "Brick Red", + "hex": "#CB4154" + }, + { + "name": "Bright Cerulean", + "hex": "#1DACD6" + }, + { + "name": "Bright Green", + "hex": "#66FF00" + }, + { + "name": "Bright Lavender", + "hex": "#BF94E4" + }, + { + "name": "Bright Lilac", + "hex": "#D891EF" + }, + { + "name": "Bright Maroon", + "hex": "#C32148" + }, + { + "name": "Bright Navy Blue", + "hex": "#1974D2" + }, + { + "name": "Bright Pink", + "hex": "#FF007F" + }, + { + "name": "Bright Turquoise", + "hex": "#08E8DE" + }, + { + "name": "Bright Ube", + "hex": "#D19FE8" + }, + { + "name": "Bright Yellow (Crayola)", + "hex": "#FFAA1D" + }, + { + "name": "Brilliant Azure", + "hex": "#3399FF" + }, + { + "name": "Brilliant Lavender", + "hex": "#F4BBFF" + }, + { + "name": "Brilliant Rose", + "hex": "#FF55A3" + }, + { + "name": "Brink Pink", + "hex": "#FB607F" + }, + { + "name": "British Racing Green", + "hex": "#004225" + }, + { + "name": "Bronze", + "hex": "#CD7F32" + }, + { + "name": "Bronze Yellow", + "hex": "#737000" + }, + { + "name": "Brown (Traditional)", + "hex": "#964B00" + }, + { + "name": "Brown (Web)", + "hex": "#A52A2A" + }, + { + "name": "Brown-Nose", + "hex": "#6B4423" + }, + { + "name": "Brown Sugar", + "hex": "#AF6E4D" + }, + { + "name": "Brown Yellow", + "hex": "#cc9966" + }, + { + "name": "Brunswick Green", + "hex": "#1B4D3E" + }, + { + "name": "Bubble Gum", + "hex": "#FFC1CC" + }, + { + "name": "Bubbles", + "hex": "#E7FEFF" + }, + { + "name": "Bud Green", + "hex": "#7BB661" + }, + { + "name": "Buff", + "hex": "#F0DC82" + }, + { + "name": "Bulgarian Rose", + "hex": "#480607" + }, + { + "name": "Burgundy", + "hex": "#800020" + }, + { + "name": "Burlywood", + "hex": "#DEB887" + }, + { + "name": "Burnished Brown", + "hex": "#A17A74" + }, + { + "name": "Burnt Orange", + "hex": "#CC5500" + }, + { + "name": "Burnt Sienna", + "hex": "#E97451" + }, + { + "name": "Burnt Umber", + "hex": "#8A3324" + }, + { + "name": "Button Blue", + "hex": "#24A0ED" + }, + { + "name": "Byzantine", + "hex": "#BD33A4" + }, + { + "name": "Byzantium", + "hex": "#702963" + }, + { + "name": "Cadet", + "hex": "#536872" + }, + { + "name": "Cadet Blue", + "hex": "#5F9EA0" + }, + { + "name": "Cadet Grey", + "hex": "#91A3B0" + }, + { + "name": "Cadmium Green", + "hex": "#006B3C" + }, + { + "name": "Cadmium Orange", + "hex": "#ED872D" + }, + { + "name": "Cadmium Red", + "hex": "#E30022" + }, + { + "name": "Cadmium Yellow", + "hex": "#FFF600" + }, + { + "name": "Cafe Au Lait", + "hex": "#A67B5B" + }, + { + "name": "Cafe Noir", + "hex": "#4B3621" + }, + { + "name": "Cal Poly Pomona Green", + "hex": "#1E4D2B" + }, + { + "name": "Cambridge Blue", + "hex": "#A3C1AD" + }, + { + "name": "Camel", + "hex": "#C19A6B" + }, + { + "name": "Cameo Pink", + "hex": "#EFBBCC" + }, + { + "name": "Camouflage Green", + "hex": "#78866B" + }, + { + "name": "Canary", + "hex": "#FFFF99" + }, + { + "name": "Canary Yellow", + "hex": "#FFEF00" + }, + { + "name": "Candy Apple Red", + "hex": "#FF0800" + }, + { + "name": "Candy Pink", + "hex": "#E4717A" + }, + { + "name": "Capri", + "hex": "#00BFFF" + }, + { + "name": "Caput Mortuum", + "hex": "#592720" + }, + { + "name": "Cardinal", + "hex": "#C41E3A" + }, + { + "name": "Caribbean Green", + "hex": "#00CC99" + }, + { + "name": "Carmine", + "hex": "#960018" + }, + { + "name": "Carmine (M&P)", + "hex": "#D70040" + }, + { + "name": "Carmine Pink", + "hex": "#EB4C42" + }, + { + "name": "Carmine Red", + "hex": "#FF0038" + }, + { + "name": "Carnation Pink", + "hex": "#FFA6C9" + }, + { + "name": "Carnelian", + "hex": "#B31B1B" + }, + { + "name": "Carolina Blue", + "hex": "#56A0D3" + }, + { + "name": "Carrot Orange", + "hex": "#ED9121" + }, + { + "name": "Castleton Green", + "hex": "#00563F" + }, + { + "name": "Catalina Blue", + "hex": "#062A78" + }, + { + "name": "Catawba", + "hex": "#703642" + }, + { + "name": "Cedar Chest", + "hex": "#C95A49" + }, + { + "name": "Ceil", + "hex": "#92A1CF" + }, + { + "name": "Celadon", + "hex": "#ACE1AF" + }, + { + "name": "Celadon Blue", + "hex": "#007BA7" + }, + { + "name": "Celadon Green", + "hex": "#2F847C" + }, + { + "name": "Celeste", + "hex": "#B2FFFF" + }, + { + "name": "Celestial Blue", + "hex": "#4997D0" + }, + { + "name": "Cerise", + "hex": "#DE3163" + }, + { + "name": "Cerise Pink", + "hex": "#EC3B83" + }, + { + "name": "Cerulean", + "hex": "#007BA7" + }, + { + "name": "Cerulean Blue", + "hex": "#2A52BE" + }, + { + "name": "Cerulean Frost", + "hex": "#6D9BC3" + }, + { + "name": "CG Blue", + "hex": "#007AA5" + }, + { + "name": "CG Red", + "hex": "#E03C31" + }, + { + "name": "Chamoisee", + "hex": "#A0785A" + }, + { + "name": "Champagne", + "hex": "#F7E7CE" + }, + { + "name": "Champagne Pink", + "hex": "#F1DDCF" + }, + { + "name": "Charcoal", + "hex": "#36454F" + }, + { + "name": "Charleston Green", + "hex": "#232B2B" + }, + { + "name": "Charm Pink", + "hex": "#E68FAC" + }, + { + "name": "Chartreuse (Traditional)", + "hex": "#DFFF00" + }, + { + "name": "Chartreuse (Web)", + "hex": "#7FFF00" + }, + { + "name": "Cherry", + "hex": "#DE3163" + }, + { + "name": "Cherry Blossom Pink", + "hex": "#FFB7C5" + }, + { + "name": "Chestnut", + "hex": "#954535" + }, + { + "name": "China Pink", + "hex": "#DE6FA1" + }, + { + "name": "China Rose", + "hex": "#A8516E" + }, + { + "name": "Chinese Red", + "hex": "#AA381E" + }, + { + "name": "Chinese Violet", + "hex": "#856088" + }, + { + "name": "Chlorophyll Green", + "hex": "#4AFF00" + }, + { + "name": "Chocolate (Traditional)", + "hex": "#7B3F00" + }, + { + "name": "Chocolate (Web)", + "hex": "#D2691E" + }, + { + "name": "Chrome Yellow", + "hex": "#FFA700" + }, + { + "name": "Cinereous", + "hex": "#98817B" + }, + { + "name": "Cinnabar", + "hex": "#E34234" + }, + { + "name": "Cinnamon", + "hex": "#D2691E" + }, + { + "name": "Cinnamon Satin", + "hex": "#CD607E" + }, + { + "name": "Citrine", + "hex": "#E4D00A" + }, + { + "name": "Citron", + "hex": "#9FA91F" + }, + { + "name": "Claret", + "hex": "#7F1734" + }, + { + "name": "Classic Rose", + "hex": "#FBCCE7" + }, + { + "name": "Cobalt Blue", + "hex": "#0047AB" + }, + { + "name": "Cocoa Brown", + "hex": "#D2691E" + }, + { + "name": "Coconut", + "hex": "#965A3E" + }, + { + "name": "Coffee", + "hex": "#6F4E37" + }, + { + "name": "Columbia Blue", + "hex": "#C4D8E2" + }, + { + "name": "Congo Pink", + "hex": "#F88379" + }, + { + "name": "Cool Black", + "hex": "#002E63" + }, + { + "name": "Cool Grey", + "hex": "#8C92AC" + }, + { + "name": "Copper", + "hex": "#B87333" + }, + { + "name": "Copper (Crayola)", + "hex": "#DA8A67" + }, + { + "name": "Copper Penny", + "hex": "#AD6F69" + }, + { + "name": "Copper Red", + "hex": "#CB6D51" + }, + { + "name": "Copper Rose", + "hex": "#996666" + }, + { + "name": "Coquelicot", + "hex": "#FF3800" + }, + { + "name": "Coral", + "hex": "#FF7F50" + }, + { + "name": "Coral Pink", + "hex": "#F88379" + }, + { + "name": "Coral Red", + "hex": "#FF4040" + }, + { + "name": "Coral Reef", + "hex": "#FD7C6E" + }, + { + "name": "Cordovan", + "hex": "#893F45" + }, + { + "name": "Corn", + "hex": "#FBEC5D" + }, + { + "name": "Cornell Red", + "hex": "#B31B1B" + }, + { + "name": "Cornflower Blue", + "hex": "#6495ED" + }, + { + "name": "Cornsilk", + "hex": "#FFF8DC" + }, + { + "name": "Cosmic Cobalt", + "hex": "#2E2D88" + }, + { + "name": "Cosmic Latte", + "hex": "#FFF8E7" + }, + { + "name": "Coyote Brown", + "hex": "#81613C" + }, + { + "name": "Cotton Candy", + "hex": "#FFBCD9" + }, + { + "name": "Cream", + "hex": "#FFFDD0" + }, + { + "name": "Crimson", + "hex": "#DC143C" + }, + { + "name": "Crimson Glory", + "hex": "#BE0032" + }, + { + "name": "Crimson Red", + "hex": "#990000" + }, + { + "name": "Cultured", + "hex": "#F5F5F5" + }, + { + "name": "Cyan", + "hex": "#00FFFF" + }, + { + "name": "Cyan Azure", + "hex": "#4E82B4" + }, + { + "name": "Cyan-Blue Azure", + "hex": "#4682BF" + }, + { + "name": "Cyan Cobalt Blue", + "hex": "#28589C" + }, + { + "name": "Cyan Cornflower Blue", + "hex": "#188BC2" + }, + { + "name": "Cyan (Process)", + "hex": "#00B7EB" + }, + { + "name": "Cyber Grape", + "hex": "#58427C" + }, + { + "name": "Cyber Yellow", + "hex": "#FFD300" + }, + { + "name": "Cyclamen", + "hex": "#F56FA1" + }, + { + "name": "Daffodil", + "hex": "#FFFF31" + }, + { + "name": "Dandelion", + "hex": "#F0E130" + }, + { + "name": "Dark Blue", + "hex": "#00008B" + }, + { + "name": "Dark Blue-Gray", + "hex": "#666699" + }, + { + "name": "Dark Brown", + "hex": "#654321" + }, + { + "name": "Dark Brown-Tangelo", + "hex": "#88654E" + }, + { + "name": "Dark Byzantium", + "hex": "#5D3954" + }, + { + "name": "Dark Candy Apple Red", + "hex": "#A40000" + }, + { + "name": "Dark Cerulean", + "hex": "#08457E" + }, + { + "name": "Dark Chestnut", + "hex": "#986960" + }, + { + "name": "Dark Coral", + "hex": "#CD5B45" + }, + { + "name": "Dark Cyan", + "hex": "#008B8B" + }, + { + "name": "Dark Electric Blue", + "hex": "#536878" + }, + { + "name": "Dark Goldenrod", + "hex": "#B8860B" + }, + { + "name": "Dark Gray (X11)", + "hex": "#A9A9A9" + }, + { + "name": "Dark Green", + "hex": "#013220" + }, + { + "name": "Dark Green (X11)", + "hex": "#006400" + }, + { + "name": "Dark Gunmetal", + "hex": "#1F262A" + }, + { + "name": "Dark Imperial Blue", + "hex": "#00416A" + }, + { + "name": "Dark Imperial Blue", + "hex": "#00147E" + }, + { + "name": "Dark Jungle Green", + "hex": "#1A2421" + }, + { + "name": "Dark Khaki", + "hex": "#BDB76B" + }, + { + "name": "Dark Lava", + "hex": "#483C32" + }, + { + "name": "Dark Lavender", + "hex": "#734F96" + }, + { + "name": "Dark Liver", + "hex": "#534B4F" + }, + { + "name": "Dark Liver (Horses)", + "hex": "#543D37" + }, + { + "name": "Dark Magenta", + "hex": "#8B008B" + }, + { + "name": "Dark Medium Gray", + "hex": "#A9A9A9" + }, + { + "name": "Dark Midnight Blue", + "hex": "#003366" + }, + { + "name": "Dark Moss Green", + "hex": "#4A5D23" + }, + { + "name": "Dark Olive Green", + "hex": "#556B2F" + }, + { + "name": "Dark Orange", + "hex": "#FF8C00" + }, + { + "name": "Dark Orchid", + "hex": "#9932CC" + }, + { + "name": "Dark Pastel Blue", + "hex": "#779ECB" + }, + { + "name": "Dark Pastel Green", + "hex": "#03C03C" + }, + { + "name": "Dark Pastel Purple", + "hex": "#966FD6" + }, + { + "name": "Dark Pastel Red", + "hex": "#C23B22" + }, + { + "name": "Dark Pink", + "hex": "#E75480" + }, + { + "name": "Dark Powder Blue", + "hex": "#003399" + }, + { + "name": "Dark Puce", + "hex": "#4F3A3C" + }, + { + "name": "Dark Purple", + "hex": "#301934" + }, + { + "name": "Dark Raspberry", + "hex": "#872657" + }, + { + "name": "Dark Red", + "hex": "#8B0000" + }, + { + "name": "Dark Salmon", + "hex": "#E9967A" + }, + { + "name": "Dark Scarlet", + "hex": "#560319" + }, + { + "name": "Dark Sea Green", + "hex": "#8FBC8F" + }, + { + "name": "Dark Sienna", + "hex": "#3C1414" + }, + { + "name": "Dark Sky Blue", + "hex": "#8CBED6" + }, + { + "name": "Dark Slate Blue", + "hex": "#483D8B" + }, + { + "name": "Dark Slate Gray", + "hex": "#2F4F4F" + }, + { + "name": "Dark Spring Green", + "hex": "#177245" + }, + { + "name": "Dark Tan", + "hex": "#918151" + }, + { + "name": "Dark Tangerine", + "hex": "#FFA812" + }, + { + "name": "Dark Taupe", + "hex": "#483C32" + }, + { + "name": "Dark Terra Cotta", + "hex": "#CC4E5C" + }, + { + "name": "Dark Turquoise", + "hex": "#00CED1" + }, + { + "name": "Dark Vanilla", + "hex": "#D1BEA8" + }, + { + "name": "Dark Violet", + "hex": "#9400D3" + }, + { + "name": "Dark Yellow", + "hex": "#9B870C" + }, + { + "name": "Dartmouth Green", + "hex": "#00703C" + }, + { + "name": "Davy's Grey", + "hex": "#555555" + }, + { + "name": "Debian Red", + "hex": "#D70A53" + }, + { + "name": "Deep Aquamarine", + "hex": "#40826D" + }, + { + "name": "Deep Carmine", + "hex": "#A9203E" + }, + { + "name": "Deep Carmine Pink", + "hex": "#EF3038" + }, + { + "name": "Deep Carrot Orange", + "hex": "#E9692C" + }, + { + "name": "Deep Cerise", + "hex": "#DA3287" + }, + { + "name": "Deep Champagne", + "hex": "#FAD6A5" + }, + { + "name": "Deep Chestnut", + "hex": "#B94E48" + }, + { + "name": "Deep Coffee", + "hex": "#704241" + }, + { + "name": "Deep Fuchsia", + "hex": "#C154C1" + }, + { + "name": "Deep Green", + "hex": "#056608" + }, + { + "name": "Deep Green-Cyan Turquoise", + "hex": "#0E7C61" + }, + { + "name": "Deep Jungle Green", + "hex": "#004B49" + }, + { + "name": "Deep Koamaru", + "hex": "#333366" + }, + { + "name": "Deep Lemon", + "hex": "#F5C71A" + }, + { + "name": "Deep Lilac", + "hex": "#9955BB" + }, + { + "name": "Deep Magenta", + "hex": "#CC00CC" + }, + { + "name": "Deep Maroon", + "hex": "#820000" + }, + { + "name": "Deep Mauve", + "hex": "#D473D4" + }, + { + "name": "Deep Moss Green", + "hex": "#355E3B" + }, + { + "name": "Deep Peach", + "hex": "#FFCBA4" + }, + { + "name": "Deep Pink", + "hex": "#FF1493" + }, + { + "name": "Deep Puce", + "hex": "#A95C68" + }, + { + "name": "Deep Red", + "hex": "#850101" + }, + { + "name": "Deep Ruby", + "hex": "#843F5B" + }, + { + "name": "Deep Saffron", + "hex": "#FF9933" + }, + { + "name": "Deep Sky Blue", + "hex": "#00BFFF" + }, + { + "name": "Deep Space Sparkle", + "hex": "#4A646C" + }, + { + "name": "Deep Spring Bud", + "hex": "#556B2F" + }, + { + "name": "Deep Taupe", + "hex": "#7E5E60" + }, + { + "name": "Deep Tuscan Red", + "hex": "#66424D" + }, + { + "name": "Deep Violet", + "hex": "#330066" + }, + { + "name": "Deer", + "hex": "#BA8759" + }, + { + "name": "Denim", + "hex": "#1560BD" + }, + { + "name": "Denim Blue", + "hex": "#2243B6" + }, + { + "name": "Desaturated Cyan", + "hex": "#669999" + }, + { + "name": "Desert", + "hex": "#C19A6B" + }, + { + "name": "Desert Sand", + "hex": "#EDC9AF" + }, + { + "name": "Desire", + "hex": "#EA3C53" + }, + { + "name": "Diamond", + "hex": "#B9F2FF" + }, + { + "name": "Dim Gray", + "hex": "#696969" + }, + { + "name": "Dingy Dungeon", + "hex": "#C53151" + }, + { + "name": "Dirt", + "hex": "#9B7653" + }, + { + "name": "Dodger Blue", + "hex": "#1E90FF" + }, + { + "name": "Dodie Yellow", + "hex": "#FEF65B" + }, + { + "name": "Dogwood Rose", + "hex": "#D71868" + }, + { + "name": "Dollar Bill", + "hex": "#85BB65" + }, + { + "name": "Dolphin Gray", + "hex": "#828E84" + }, + { + "name": "Donkey Brown", + "hex": "#664C28" + }, + { + "name": "Drab", + "hex": "#967117" + }, + { + "name": "Duke Blue", + "hex": "#00009C" + }, + { + "name": "Dust Storm", + "hex": "#E5CCC9" + }, + { + "name": "Dutch White", + "hex": "#EFDFBB" + }, + { + "name": "Earth Yellow", + "hex": "#E1A95F" + }, + { + "name": "Ebony", + "hex": "#555D50" + }, + { + "name": "Ecru", + "hex": "#C2B280" + }, + { + "name": "Eerie Black", + "hex": "#1B1B1B" + }, + { + "name": "Eggplant", + "hex": "#614051" + }, + { + "name": "Eggshell", + "hex": "#F0EAD6" + }, + { + "name": "Egyptian Blue", + "hex": "#1034A6" + }, + { + "name": "Electric Blue", + "hex": "#7DF9FF" + }, + { + "name": "Electric Crimson", + "hex": "#FF003F" + }, + { + "name": "Electric Cyan", + "hex": "#00FFFF" + }, + { + "name": "Electric Green", + "hex": "#00FF00" + }, + { + "name": "Electric Indigo", + "hex": "#6F00FF" + }, + { + "name": "Electric Lavender", + "hex": "#F4BBFF" + }, + { + "name": "Electric Lime", + "hex": "#CCFF00" + }, + { + "name": "Electric Purple", + "hex": "#BF00FF" + }, + { + "name": "Electric Ultramarine", + "hex": "#3F00FF" + }, + { + "name": "Electric Violet", + "hex": "#8F00FF" + }, + { + "name": "Electric Yellow", + "hex": "#FFFF33" + }, + { + "name": "Emerald", + "hex": "#50C878" + }, + { + "name": "Eminence", + "hex": "#6C3082" + }, + { + "name": "English Green", + "hex": "#1B4D3E" + }, + { + "name": "English Lavender", + "hex": "#B48395" + }, + { + "name": "English Red", + "hex": "#AB4B52" + }, + { + "name": "English Vermillion", + "hex": "#CC474B" + }, + { + "name": "English Violet", + "hex": "#563C5C" + }, + { + "name": "Eton Blue", + "hex": "#96C8A2" + }, + { + "name": "Eucalyptus", + "hex": "#44D7A8" + }, + { + "name": "Fallow", + "hex": "#C19A6B" + }, + { + "name": "Falu Red", + "hex": "#801818" + }, + { + "name": "Fandango", + "hex": "#B53389" + }, + { + "name": "Fandango Pink", + "hex": "#DE5285" + }, + { + "name": "Fashion Fuchsia", + "hex": "#F400A1" + }, + { + "name": "Fawn", + "hex": "#E5AA70" + }, + { + "name": "Feldgrau", + "hex": "#4D5D53" + }, + { + "name": "Feldspar", + "hex": "#FDD5B1" + }, + { + "name": "Fern Green", + "hex": "#4F7942" + }, + { + "name": "Ferrari Red", + "hex": "#FF2800" + }, + { + "name": "Field Drab", + "hex": "#6C541E" + }, + { + "name": "Fiery Rose", + "hex": "#FF5470" + }, + { + "name": "Firebrick", + "hex": "#B22222" + }, + { + "name": "Fire Engine Red", + "hex": "#CE2029" + }, + { + "name": "Flame", + "hex": "#E25822" + }, + { + "name": "Flamingo Pink", + "hex": "#FC8EAC" + }, + { + "name": "Flattery", + "hex": "#6B4423" + }, + { + "name": "Flavescent", + "hex": "#F7E98E" + }, + { + "name": "Flax", + "hex": "#EEDC82" + }, + { + "name": "Flirt", + "hex": "#A2006D" + }, + { + "name": "Floral White", + "hex": "#FFFAF0" + }, + { + "name": "Fluorescent Orange", + "hex": "#FFBF00" + }, + { + "name": "Fluorescent Pink", + "hex": "#FF1493" + }, + { + "name": "Fluorescent Yellow", + "hex": "#CCFF00" + }, + { + "name": "Folly", + "hex": "#FF004F" + }, + { + "name": "Forest Green (Traditional)", + "hex": "#014421" + }, + { + "name": "Forest Green (Web)", + "hex": "#228B22" + }, + { + "name": "French Beige", + "hex": "#A67B5B" + }, + { + "name": "French Bistre", + "hex": "#856D4D" + }, + { + "name": "French Blue", + "hex": "#0072BB" + }, + { + "name": "French Fuchsia", + "hex": "#FD3F92" + }, + { + "name": "French Lilac", + "hex": "#86608E" + }, + { + "name": "French Lime", + "hex": "#9EFD38" + }, + { + "name": "French Mauve", + "hex": "#D473D4" + }, + { + "name": "French Pink", + "hex": "#FD6C9E" + }, + { + "name": "French Plum", + "hex": "#811453" + }, + { + "name": "French Puce", + "hex": "#4E1609" + }, + { + "name": "French Raspberry", + "hex": "#C72C48" + }, + { + "name": "French Rose", + "hex": "#F64A8A" + }, + { + "name": "French Sky Blue", + "hex": "#77B5FE" + }, + { + "name": "French Violet", + "hex": "#8806CE" + }, + { + "name": "French Wine", + "hex": "#AC1E44" + }, + { + "name": "Fresh Air", + "hex": "#A6E7FF" + }, + { + "name": "Frogert", + "hex": "#E936A7" + }, + { + "name": "Fuchsia", + "hex": "#FF00FF" + }, + { + "name": "Fuchsia (Crayola)", + "hex": "#C154C1" + }, + { + "name": "Fuchsia Pink", + "hex": "#FF77FF" + }, + { + "name": "Fuchsia Purple", + "hex": "#CC397B" + }, + { + "name": "Fuchsia Rose", + "hex": "#C74375" + }, + { + "name": "Fulvous", + "hex": "#E48400" + }, + { + "name": "Fuzzy Wuzzy", + "hex": "#CC6666" + }, + { + "name": "Gainsboro", + "hex": "#DCDCDC" + }, + { + "name": "Gamboge", + "hex": "#E49B0F" + }, + { + "name": "Gamboge Orange (Brown)", + "hex": "#996600" + }, + { + "name": "Gargoyle Gas", + "hex": "#FFDF46" + }, + { + "name": "Generic Viridian", + "hex": "#007F66" + }, + { + "name": "Ghost White", + "hex": "#F8F8FF" + }, + { + "name": "Giant's Club", + "hex": "#B05C52" + }, + { + "name": "Giants Orange", + "hex": "#FE5A1D" + }, + { + "name": "Ginger", + "hex": "#B06500" + }, + { + "name": "Glaucous", + "hex": "#6082B6" + }, + { + "name": "Glitter", + "hex": "#E6E8FA" + }, + { + "name": "Glossy Grape", + "hex": "#AB92B3" + }, + { + "name": "GO Green", + "hex": "#00AB66" + }, + { + "name": "Gold (Metallic)", + "hex": "#D4AF37" + }, + { + "name": "Gold (Web) (Golden)", + "hex": "#FFD700" + }, + { + "name": "Gold Fusion", + "hex": "#85754E" + }, + { + "name": "Golden Brown", + "hex": "#996515" + }, + { + "name": "Golden Poppy", + "hex": "#FCC200" + }, + { + "name": "Golden Yellow", + "hex": "#FFDF00" + }, + { + "name": "Goldenrod", + "hex": "#DAA520" + }, + { + "name": "Granite Gray", + "hex": "#676767" + }, + { + "name": "Granny Smith Apple", + "hex": "#A8E4A0" + }, + { + "name": "Grape", + "hex": "#6F2DA8" + }, + { + "name": "Gray", + "hex": "#808080" + }, + { + "name": "Gray (HTML/CSS Gray)", + "hex": "#808080" + }, + { + "name": "Gray (X11 Gray)", + "hex": "#BEBEBE" + }, + { + "name": "Gray-Asparagus", + "hex": "#465945" + }, + { + "name": "Gray-Blue", + "hex": "#8C92AC" + }, + { + "name": "Green (Color Wheel) (X11 Green)", + "hex": "#00FF00" + }, + { + "name": "Green (Crayola)", + "hex": "#1CAC78" + }, + { + "name": "Green (HTML/CSS Color)", + "hex": "#008000" + }, + { + "name": "Green (Munsell)", + "hex": "#00A877" + }, + { + "name": "Green (NCS)", + "hex": "#009F6B" + }, + { + "name": "Green (Pantone)", + "hex": "#00AD43" + }, + { + "name": "Green (Pigment)", + "hex": "#00A550" + }, + { + "name": "Green (RYB)", + "hex": "#66B032" + }, + { + "name": "Green-Blue", + "hex": "#1164B4" + }, + { + "name": "Green-Cyan", + "hex": "#009966" + }, + { + "name": "Green Lizard", + "hex": "#A7F432" + }, + { + "name": "Green Sheen", + "hex": "#6EAEA1" + }, + { + "name": "Green-Yellow", + "hex": "#ADFF2F" + }, + { + "name": "Grizzly", + "hex": "#885818" + }, + { + "name": "Grullo", + "hex": "#A99A86" + }, + { + "name": "Guppie Green", + "hex": "#00FF7F" + }, + { + "name": "Gunmetal", + "hex": "#2a3439" + }, + { + "name": "Halaya Ube", + "hex": "#663854" + }, + { + "name": "Han Blue", + "hex": "#446CCF" + }, + { + "name": "Han Purple", + "hex": "#5218FA" + }, + { + "name": "Hansa Yellow", + "hex": "#E9D66B" + }, + { + "name": "Harlequin", + "hex": "#3FFF00" + }, + { + "name": "Harlequin Green", + "hex": "#46CB18" + }, + { + "name": "Harvard Crimson", + "hex": "#C90016" + }, + { + "name": "Harvest Gold", + "hex": "#DA9100" + }, + { + "name": "Heart Gold", + "hex": "#808000" + }, + { + "name": "Heat Wave", + "hex": "#FF7A00" + }, + { + "name": "Heidelberg Red", + "hex": "#960018" + }, + { + "name": "Heliotrope", + "hex": "#DF73FF" + }, + { + "name": "Heliotrope Gray", + "hex": "#AA98A9" + }, + { + "name": "Heliotrope Magenta", + "hex": "#AA00BB" + }, + { + "name": "Hollywood Cerise", + "hex": "#F400A1" + }, + { + "name": "Honeydew", + "hex": "#F0FFF0" + }, + { + "name": "Honolulu Blue", + "hex": "#006DB0" + }, + { + "name": "Hooker's Green", + "hex": "#49796B" + }, + { + "name": "Hot Magenta", + "hex": "#FF1DCE" + }, + { + "name": "Hot Pink", + "hex": "#FF69B4" + }, + { + "name": "Hunter Green", + "hex": "#355E3B" + }, + { + "name": "Iceberg", + "hex": "#71A6D2" + }, + { + "name": "Icterine", + "hex": "#FCF75E" + }, + { + "name": "Iguana Green", + "hex": "#71BC78" + }, + { + "name": "Illuminating Emerald", + "hex": "#319177" + }, + { + "name": "Imperial", + "hex": "#602F6B" + }, + { + "name": "Imperial Blue", + "hex": "#002395" + }, + { + "name": "Imperial Purple", + "hex": "#66023C" + }, + { + "name": "Imperial Red", + "hex": "#ED2939" + }, + { + "name": "Inchworm", + "hex": "#B2EC5D" + }, + { + "name": "Independence", + "hex": "#4C516D" + }, + { + "name": "India Green", + "hex": "#138808" + }, + { + "name": "Indian Red", + "hex": "#CD5C5C" + }, + { + "name": "Indian Yellow", + "hex": "#E3A857" + }, + { + "name": "Indigo", + "hex": "#4B0082" + }, + { + "name": "Indigo Dye", + "hex": "#091F92" + }, + { + "name": "Indigo (Web)", + "hex": "#4B0082" + }, + { + "name": "Infra Red", + "hex": "#FF496C" + }, + { + "name": "Interdimensional Blue", + "hex": "#360CCC" + }, + { + "name": "International Klein Blue", + "hex": "#002FA7" + }, + { + "name": "International Orange (Aerospace)", + "hex": "#FF4F00" + }, + { + "name": "International Orange (Engineering)", + "hex": "#BA160C" + }, + { + "name": "International Orange (Golden Gate Bridge)", + "hex": "#C0362C" + }, + { + "name": "Iris", + "hex": "#5A4FCF" + }, + { + "name": "Irresistible", + "hex": "#B3446C" + }, + { + "name": "Isabelline", + "hex": "#F4F0EC" + }, + { + "name": "Islamic Green", + "hex": "#009000" + }, + { + "name": "Italian Sky Blue", + "hex": "#B2FFFF" + }, + { + "name": "Ivory", + "hex": "#FFFFF0" + }, + { + "name": "Jade", + "hex": "#00A86B" + }, + { + "name": "Japanese Carmine", + "hex": "#9D2933" + }, + { + "name": "Japanese Indigo", + "hex": "#264348" + }, + { + "name": "Japanese Violet", + "hex": "#5B3256" + }, + { + "name": "Jasmine", + "hex": "#F8DE7E" + }, + { + "name": "Jasper", + "hex": "#D73B3E" + }, + { + "name": "Jazzberry Jam", + "hex": "#A50B5E" + }, + { + "name": "Jelly Bean", + "hex": "#DA614E" + }, + { + "name": "Jet", + "hex": "#343434" + }, + { + "name": "Jonquil", + "hex": "#F4CA16" + }, + { + "name": "Jordy Blue", + "hex": "#8AB9F1" + }, + { + "name": "June Bud", + "hex": "#BDDA57" + }, + { + "name": "Jungle Green", + "hex": "#29AB87" + }, + { + "name": "Kelly Green", + "hex": "#4CBB17" + }, + { + "name": "Kenyan Copper", + "hex": "#7C1C05" + }, + { + "name": "Keppel", + "hex": "#3AB09E" + }, + { + "name": "Key Lime", + "hex": "#E8F48C" + }, + { + "name": "Khaki (HTML/CSS) (Khaki)", + "hex": "#C3B091" + }, + { + "name": "Khaki (X11) (Light Khaki)", + "hex": "#F0E68C" + }, + { + "name": "Kiwi", + "hex": "#8EE53F" + }, + { + "name": "Kobe", + "hex": "#882D17" + }, + { + "name": "Kobi", + "hex": "#E79FC4" + }, + { + "name": "Kobicha", + "hex": "#6B4423" + }, + { + "name": "Kombu Green", + "hex": "#354230" + }, + { + "name": "KSU Purple", + "hex": "#512888" + }, + { + "name": "KU Crimson", + "hex": "#E8000D" + }, + { + "name": "La Salle Green", + "hex": "#087830" + }, + { + "name": "Languid Lavender", + "hex": "#D6CADD" + }, + { + "name": "Lapis Lazuli", + "hex": "#26619C" + }, + { + "name": "Laser Lemon", + "hex": "#FFFF66" + }, + { + "name": "Laurel Green", + "hex": "#A9BA9D" + }, + { + "name": "Lava", + "hex": "#CF1020" + }, + { + "name": "Lavender (Floral)", + "hex": "#B57EDC" + }, + { + "name": "Lavender (Web)", + "hex": "#E6E6FA" + }, + { + "name": "Lavender Blue", + "hex": "#CCCCFF" + }, + { + "name": "Lavender Blush", + "hex": "#FFF0F5" + }, + { + "name": "Lavender Gray", + "hex": "#C4C3D0" + }, + { + "name": "Lavender Indigo", + "hex": "#9457EB" + }, + { + "name": "Lavender Magenta", + "hex": "#EE82EE" + }, + { + "name": "Lavender Mist", + "hex": "#E6E6FA" + }, + { + "name": "Lavender Pink", + "hex": "#FBAED2" + }, + { + "name": "Lavender Purple", + "hex": "#967BB6" + }, + { + "name": "Lavender Rose", + "hex": "#FBA0E3" + }, + { + "name": "Lawn Green", + "hex": "#7CFC00" + }, + { + "name": "Lemon", + "hex": "#FFF700" + }, + { + "name": "Lemon Chiffon", + "hex": "#FFFACD" + }, + { + "name": "Lemon Curry", + "hex": "#CCA01D" + }, + { + "name": "Lemon Glacier", + "hex": "#FDFF00" + }, + { + "name": "Lemon Lime", + "hex": "#E3FF00" + }, + { + "name": "Lemon Meringue", + "hex": "#F6EABE" + }, + { + "name": "Lemon Yellow", + "hex": "#FFF44F" + }, + { + "name": "Licorice", + "hex": "#1A1110" + }, + { + "name": "Liberty", + "hex": "#545AA7" + }, + { + "name": "Light Apricot", + "hex": "#FDD5B1" + }, + { + "name": "Light Blue", + "hex": "#ADD8E6" + }, + { + "name": "Light Brown", + "hex": "#B5651D" + }, + { + "name": "Light Carmine Pink", + "hex": "#E66771" + }, + { + "name": "Light Cobalt Blue", + "hex": "#88ACE0" + }, + { + "name": "Light Coral", + "hex": "#F08080" + }, + { + "name": "Light Cornflower Blue", + "hex": "#93CCEA" + }, + { + "name": "Light Crimson", + "hex": "#F56991" + }, + { + "name": "Light Cyan", + "hex": "#E0FFFF" + }, + { + "name": "Light Deep Pink", + "hex": "#FF5CCD" + }, + { + "name": "Light French Beige", + "hex": "#C8AD7F" + }, + { + "name": "Light Fuchsia Pink", + "hex": "#F984EF" + }, + { + "name": "Light Goldenrod Yellow", + "hex": "#FAFAD2" + }, + { + "name": "Light Gray", + "hex": "#D3D3D3" + }, + { + "name": "Light Grayish Magenta", + "hex": "#CC99CC" + }, + { + "name": "Light Green", + "hex": "#90EE90" + }, + { + "name": "Light Hot Pink", + "hex": "#FFB3DE" + }, + { + "name": "Light Khaki", + "hex": "#F0E68C" + }, + { + "name": "Light Medium Orchid", + "hex": "#D39BCB" + }, + { + "name": "Light Moss Green", + "hex": "#ADDFAD" + }, + { + "name": "Light Orange", + "hex": "#FED8B1" + }, + { + "name": "Light Orchid", + "hex": "#E6A8D7" + }, + { + "name": "Light Pastel Purple", + "hex": "#B19CD9" + }, + { + "name": "Light Pink", + "hex": "#FFB6C1" + }, + { + "name": "Light Red Ochre", + "hex": "#E97451" + }, + { + "name": "Light Salmon", + "hex": "#FFA07A" + }, + { + "name": "Light Salmon Pink", + "hex": "#FF9999" + }, + { + "name": "Light Sea Green", + "hex": "#20B2AA" + }, + { + "name": "Light Sky Blue", + "hex": "#87CEFA" + }, + { + "name": "Light Slate Gray", + "hex": "#778899" + }, + { + "name": "Light Steel Blue", + "hex": "#B0C4DE" + }, + { + "name": "Light Taupe", + "hex": "#B38B6D" + }, + { + "name": "Light Thulian Pink", + "hex": "#E68FAC" + }, + { + "name": "Light Yellow", + "hex": "#FFFFE0" + }, + { + "name": "Lilac", + "hex": "#C8A2C8" + }, + { + "name": "Lilac Luster", + "hex": "#AE98AA" + }, + { + "name": "Lime (Color Wheel)", + "hex": "#BFFF00" + }, + { + "name": "Lime (Web) (X11 Green)", + "hex": "#00FF00" + }, + { + "name": "Lime Green", + "hex": "#32CD32" + }, + { + "name": "Limerick", + "hex": "#9DC209" + }, + { + "name": "Lincoln Green", + "hex": "#195905" + }, + { + "name": "Linen", + "hex": "#FAF0E6" + }, + { + "name": "Loeen (Lopen) Look", + "hex": "#15F2FD" + }, + { + "name": "Liseran Purple", + "hex": "#DE6FA1" + }, + { + "name": "Little Boy Blue", + "hex": "#6CA0DC" + }, + { + "name": "Liver", + "hex": "#674C47" + }, + { + "name": "Liver (Dogs)", + "hex": "#B86D29" + }, + { + "name": "Liver (Organ)", + "hex": "#6C2E1F" + }, + { + "name": "Liver Chestnut", + "hex": "#987456" + }, + { + "name": "Livid", + "hex": "#6699CC" + }, + { + "name": "Lumber", + "hex": "#FFE4CD" + }, + { + "name": "Lust", + "hex": "#E62020" + }, + { + "name": "Maastricht Blue", + "hex": "#001C3D" + }, + { + "name": "Macaroni And Cheese", + "hex": "#FFBD88" + }, + { + "name": "Madder Lake", + "hex": "#CC3336" + }, + { + "name": "Magenta", + "hex": "#FF00FF" + }, + { + "name": "Magenta (Crayola)", + "hex": "#FF55A3" + }, + { + "name": "Magenta (Dye)", + "hex": "#CA1F7B" + }, + { + "name": "Magenta (Pantone)", + "hex": "#D0417E" + }, + { + "name": "Magenta (Process)", + "hex": "#FF0090" + }, + { + "name": "Magenta Haze", + "hex": "#9F4576" + }, + { + "name": "Magenta-Pink", + "hex": "#CC338B" + }, + { + "name": "Magic Mint", + "hex": "#AAF0D1" + }, + { + "name": "Magic Potion", + "hex": "#FF4466" + }, + { + "name": "Magnolia", + "hex": "#F8F4FF" + }, + { + "name": "Mahogany", + "hex": "#C04000" + }, + { + "name": "Maize", + "hex": "#FBEC5D" + }, + { + "name": "Majorelle Blue", + "hex": "#6050DC" + }, + { + "name": "Malachite", + "hex": "#0BDA51" + }, + { + "name": "Manatee", + "hex": "#979AAA" + }, + { + "name": "Mandarin", + "hex": "#F37A48" + }, + { + "name": "Mango Tango", + "hex": "#FF8243" + }, + { + "name": "Mantis", + "hex": "#74C365" + }, + { + "name": "Mardi Gras", + "hex": "#880085" + }, + { + "name": "Marigold", + "hex": "#EAA221" + }, + { + "name": "Maroon (Crayola)", + "hex": "#C32148" + }, + { + "name": "Maroon (HTML/CSS)", + "hex": "#800000" + }, + { + "name": "Maroon (X11)", + "hex": "#B03060" + }, + { + "name": "Mauve", + "hex": "#E0B0FF" + }, + { + "name": "Mauve Taupe", + "hex": "#915F6D" + }, + { + "name": "Mauvelous", + "hex": "#EF98AA" + }, + { + "name": "Maximum Blue", + "hex": "#47ABCC" + }, + { + "name": "Maximum Blue Green", + "hex": "#30BFBF" + }, + { + "name": "Maximum Blue Purple", + "hex": "#ACACE6" + }, + { + "name": "Maximum Green", + "hex": "#5E8C31" + }, + { + "name": "Maximum Green Yellow", + "hex": "#D9E650" + }, + { + "name": "Maximum Purple", + "hex": "#733380" + }, + { + "name": "Maximum Red", + "hex": "#D92121" + }, + { + "name": "Maximum Red Purple", + "hex": "#A63A79" + }, + { + "name": "Maximum Yellow", + "hex": "#FAFA37" + }, + { + "name": "Maximum Yellow Red", + "hex": "#F2BA49" + }, + { + "name": "May Green", + "hex": "#4C9141" + }, + { + "name": "Maya Blue", + "hex": "#73C2FB" + }, + { + "name": "Meat Brown", + "hex": "#E5B73B" + }, + { + "name": "Medium Aquamarine", + "hex": "#66DDAA" + }, + { + "name": "Medium Blue", + "hex": "#0000CD" + }, + { + "name": "Medium Candy Apple Red", + "hex": "#E2062C" + }, + { + "name": "Medium Carmine", + "hex": "#AF4035" + }, + { + "name": "Medium Champagne", + "hex": "#F3E5AB" + }, + { + "name": "Medium Electric Blue", + "hex": "#035096" + }, + { + "name": "Medium Jungle Green", + "hex": "#1C352D" + }, + { + "name": "Medium Lavender Magenta", + "hex": "#DDA0DD" + }, + { + "name": "Medium Orchid", + "hex": "#BA55D3" + }, + { + "name": "Medium Persian Blue", + "hex": "#0067A5" + }, + { + "name": "Medium Purple", + "hex": "#9370DB" + }, + { + "name": "Medium Red-Violet", + "hex": "#BB3385" + }, + { + "name": "Medium Ruby", + "hex": "#AA4069" + }, + { + "name": "Medium Sea Green", + "hex": "#3CB371" + }, + { + "name": "Medium Sky Blue", + "hex": "#80DAEB" + }, + { + "name": "Medium Slate Blue", + "hex": "#7B68EE" + }, + { + "name": "Medium Spring Bud", + "hex": "#C9DC87" + }, + { + "name": "Medium Spring Green", + "hex": "#00FA9A" + }, + { + "name": "Medium Taupe", + "hex": "#674C47" + }, + { + "name": "Medium Turquoise", + "hex": "#48D1CC" + }, + { + "name": "Medium Tuscan Red", + "hex": "#79443B" + }, + { + "name": "Medium Vermilion", + "hex": "#D9603B" + }, + { + "name": "Medium Violet-Red", + "hex": "#C71585" + }, + { + "name": "Mellow Apricot", + "hex": "#F8B878" + }, + { + "name": "Mellow Yellow", + "hex": "#F8DE7E" + }, + { + "name": "Melon", + "hex": "#FDBCB4" + }, + { + "name": "Metallic Seaweed", + "hex": "#0A7E8C" + }, + { + "name": "Metallic Sunburst", + "hex": "#9C7C38" + }, + { + "name": "Mexican Pink", + "hex": "#E4007C" + }, + { + "name": "Middle Blue", + "hex": "#7ED4E6" + }, + { + "name": "Middle Blue Green", + "hex": "#8DD9CC" + }, + { + "name": "Middle Blue Purple", + "hex": "#8B72BE" + }, + { + "name": "Middle Red Purple", + "hex": "#210837" + }, + { + "name": "Middle Green", + "hex": "#4D8C57" + }, + { + "name": "Middle Green Yellow", + "hex": "#ACBF60" + }, + { + "name": "Middle Purple", + "hex": "#D982B5" + }, + { + "name": "Middle Red", + "hex": "#E58E73" + }, + { + "name": "Middle Red Purple", + "hex": "#A55353" + }, + { + "name": "Middle Yellow", + "hex": "#FFEB00" + }, + { + "name": "Middle Yellow Red", + "hex": "#ECB176" + }, + { + "name": "Midnight", + "hex": "#702670" + }, + { + "name": "Midnight Blue", + "hex": "#191970" + }, + { + "name": "Midnight Green (Eagle Green)", + "hex": "#004953" + }, + { + "name": "Mikado Yellow", + "hex": "#FFC40C" + }, + { + "name": "Milk", + "hex": "#FDFFF5" + }, + { + "name": "Mimi Pink", + "hex": "#FFDAE9" + }, + { + "name": "Mindaro", + "hex": "#E3F988" + }, + { + "name": "Ming", + "hex": "#36747D" + }, + { + "name": "Minion Yellow", + "hex": "#F5E050" + }, + { + "name": "Mint", + "hex": "#3EB489" + }, + { + "name": "Mint Cream", + "hex": "#F5FFFA" + }, + { + "name": "Mint Green", + "hex": "#98FF98" + }, + { + "name": "Misty Moss", + "hex": "#BBB477" + }, + { + "name": "Misty Rose", + "hex": "#FFE4E1" + }, + { + "name": "Moccasin", + "hex": "#FAEBD7" + }, + { + "name": "Mode Beige", + "hex": "#967117" + }, + { + "name": "Moonstone Blue", + "hex": "#73A9C2" + }, + { + "name": "Mordant Red 19", + "hex": "#AE0C00" + }, + { + "name": "Morning Blue", + "hex": "#8DA399" + }, + { + "name": "Moss Green", + "hex": "#8A9A5B" + }, + { + "name": "Mountain Meadow", + "hex": "#30BA8F" + }, + { + "name": "Mountbatten Pink", + "hex": "#997A8D" + }, + { + "name": "MSU Green", + "hex": "#18453B" + }, + { + "name": "Mughal Green", + "hex": "#306030" + }, + { + "name": "Mulberry", + "hex": "#C54B8C" + }, + { + "name": "Mummy's Tomb", + "hex": "#828E84" + }, + { + "name": "Mustard", + "hex": "#FFDB58" + }, + { + "name": "Myrtle Green", + "hex": "#317873" + }, + { + "name": "Mystic", + "hex": "#D65282" + }, + { + "name": "Mystic Maroon", + "hex": "#AD4379" + }, + { + "name": "Nadeshiko Pink", + "hex": "#F6ADC6" + }, + { + "name": "Napier Green", + "hex": "#2A8000" + }, + { + "name": "Naples Yellow", + "hex": "#FADA5E" + }, + { + "name": "Navajo White", + "hex": "#FFDEAD" + }, + { + "name": "Navy", + "hex": "#000080" + }, + { + "name": "Navy Purple", + "hex": "#9457EB" + }, + { + "name": "Neon Carrot", + "hex": "#FFA343" + }, + { + "name": "Neon Fuchsia", + "hex": "#FE4164" + }, + { + "name": "Neon Green", + "hex": "#39FF14" + }, + { + "name": "New Car", + "hex": "#214FC6" + }, + { + "name": "New York Pink", + "hex": "#D7837F" + }, + { + "name": "Nickel", + "hex": "#727472" + }, + { + "name": "Non-Photo Blue", + "hex": "#A4DDED" + }, + { + "name": "North Texas Green", + "hex": "#059033" + }, + { + "name": "Nyanza", + "hex": "#E9FFDB" + }, + { + "name": "Ocean Blue", + "hex": "#4F42B5" + }, + { + "name": "Ocean Boat Blue", + "hex": "#0077BE" + }, + { + "name": "Ocean Green", + "hex": "#48BF91" + }, + { + "name": "Ochre", + "hex": "#CC7722" + }, + { + "name": "Office Green", + "hex": "#008000" + }, + { + "name": "Ogre Odor", + "hex": "#FD5240" + }, + { + "name": "Old Burgundy", + "hex": "#43302E" + }, + { + "name": "Old Gold", + "hex": "#CFB53B" + }, + { + "name": "Old Heliotrope", + "hex": "#563C5C" + }, + { + "name": "Old Lace", + "hex": "#FDF5E6" + }, + { + "name": "Old Lavender", + "hex": "#796878" + }, + { + "name": "Old Mauve", + "hex": "#673147" + }, + { + "name": "Old Moss Green", + "hex": "#867E36" + }, + { + "name": "Old Rose", + "hex": "#C08081" + }, + { + "name": "Old Silver", + "hex": "#848482" + }, + { + "name": "Olive", + "hex": "#808000" + }, + { + "name": "Olive Drab (#3)", + "hex": "#6B8E23" + }, + { + "name": "Olive Drab #7", + "hex": "#3C341F" + }, + { + "name": "Olivine", + "hex": "#9AB973" + }, + { + "name": "Onyx", + "hex": "#353839" + }, + { + "name": "Opera Mauve", + "hex": "#B784A7" + }, + { + "name": "Orange (Color Wheel)", + "hex": "#FF7F00" + }, + { + "name": "Orange (Crayola)", + "hex": "#FF7538" + }, + { + "name": "Orange (Pantone)", + "hex": "#FF5800" + }, + { + "name": "Orange (RYB)", + "hex": "#FB9902" + }, + { + "name": "Orange (Web)", + "hex": "#FFA500" + }, + { + "name": "Orange Peel", + "hex": "#FF9F00" + }, + { + "name": "Orange-Red", + "hex": "#FF4500" + }, + { + "name": "Orange Soda", + "hex": "#FA5B3D" + }, + { + "name": "Orange-Yellow", + "hex": "#F8D568" + }, + { + "name": "Orchid", + "hex": "#DA70D6" + }, + { + "name": "Orchid Pink", + "hex": "#F2BDCD" + }, + { + "name": "Orioles Orange", + "hex": "#FB4F14" + }, + { + "name": "Otter Brown", + "hex": "#654321" + }, + { + "name": "Outer Space", + "hex": "#414A4C" + }, + { + "name": "Outrageous Orange", + "hex": "#FF6E4A" + }, + { + "name": "Oxford Blue", + "hex": "#002147" + }, + { + "name": "OU Crimson Red", + "hex": "#990000" + }, + { + "name": "Pacific Blue", + "hex": "#1CA9C9" + }, + { + "name": "Pakistan Green", + "hex": "#006600" + }, + { + "name": "Palatinate Blue", + "hex": "#273BE2" + }, + { + "name": "Palatinate Purple", + "hex": "#682860" + }, + { + "name": "Pale Aqua", + "hex": "#BCD4E6" + }, + { + "name": "Pale Blue", + "hex": "#AFEEEE" + }, + { + "name": "Pale Brown", + "hex": "#987654" + }, + { + "name": "Pale Carmine", + "hex": "#AF4035" + }, + { + "name": "Pale Cerulean", + "hex": "#9BC4E2" + }, + { + "name": "Pale Chestnut", + "hex": "#DDADAF" + }, + { + "name": "Pale Copper", + "hex": "#DA8A67" + }, + { + "name": "Pale Cornflower Blue", + "hex": "#ABCDEF" + }, + { + "name": "Pale Cyan", + "hex": "#87D3F8" + }, + { + "name": "Pale Gold", + "hex": "#E6BE8A" + }, + { + "name": "Pale Goldenrod", + "hex": "#EEE8AA" + }, + { + "name": "Pale Green", + "hex": "#98FB98" + }, + { + "name": "Pale Lavender", + "hex": "#DCD0FF" + }, + { + "name": "Pale Magenta", + "hex": "#F984E5" + }, + { + "name": "Pale Magenta-Pink", + "hex": "#FF99CC" + }, + { + "name": "Pale Pink", + "hex": "#FADADD" + }, + { + "name": "Pale Plum", + "hex": "#DDA0DD" + }, + { + "name": "Pale Red-Violet", + "hex": "#DB7093" + }, + { + "name": "Pale Robin Egg Blue", + "hex": "#96DED1" + }, + { + "name": "Pale Silver", + "hex": "#C9C0BB" + }, + { + "name": "Pale Spring Bud", + "hex": "#ECEBBD" + }, + { + "name": "Pale Taupe", + "hex": "#BC987E" + }, + { + "name": "Pale Turquoise", + "hex": "#AFEEEE" + }, + { + "name": "Pale Violet", + "hex": "#CC99FF" + }, + { + "name": "Pale Violet-Red", + "hex": "#DB7093" + }, + { + "name": "Palm Leaf", + "hex": "#6F9940" + }, + { + "name": "Pansy Purple", + "hex": "#78184A" + }, + { + "name": "Paolo Veronese Green", + "hex": "#009B7D" + }, + { + "name": "Papaya Whip", + "hex": "#FFEFD5" + }, + { + "name": "Paradise Pink", + "hex": "#E63E62" + }, + { + "name": "Paris Green", + "hex": "#50C878" + }, + { + "name": "Parrot Pink", + "hex": "#D998A0" + }, + { + "name": "Pastel Blue", + "hex": "#AEC6CF" + }, + { + "name": "Pastel Brown", + "hex": "#836953" + }, + { + "name": "Pastel Gray", + "hex": "#CFCFC4" + }, + { + "name": "Pastel Green", + "hex": "#77DD77" + }, + { + "name": "Pastel Magenta", + "hex": "#F49AC2" + }, + { + "name": "Pastel Orange", + "hex": "#FFB347" + }, + { + "name": "Pastel Pink", + "hex": "#DEA5A4" + }, + { + "name": "Pastel Purple", + "hex": "#B39EB5" + }, + { + "name": "Pastel Red", + "hex": "#FF6961" + }, + { + "name": "Pastel Violet", + "hex": "#CB99C9" + }, + { + "name": "Pastel Yellow", + "hex": "#FDFD96" + }, + { + "name": "Patriarch", + "hex": "#800080" + }, + { + "name": "Payne's Grey", + "hex": "#536878" + }, + { + "name": "Peach", + "hex": "#FFE5B4" + }, + { + "name": "Peach", + "hex": "#FFCBA4" + }, + { + "name": "Peach-Orange", + "hex": "#FFCC99" + }, + { + "name": "Peach Puff", + "hex": "#FFDAB9" + }, + { + "name": "Peach-Yellow", + "hex": "#FADFAD" + }, + { + "name": "Pear", + "hex": "#D1E231" + }, + { + "name": "Pearl", + "hex": "#EAE0C8" + }, + { + "name": "Pearl Aqua", + "hex": "#88D8C0" + }, + { + "name": "Pearly Purple", + "hex": "#B768A2" + }, + { + "name": "Peridot", + "hex": "#E6E200" + }, + { + "name": "Periwinkle", + "hex": "#CCCCFF" + }, + { + "name": "Permanent Geranium Lake", + "hex": "#E12C2C" + }, + { + "name": "Persian Blue", + "hex": "#1C39BB" + }, + { + "name": "Persian Green", + "hex": "#00A693" + }, + { + "name": "Persian Indigo", + "hex": "#32127A" + }, + { + "name": "Persian Orange", + "hex": "#D99058" + }, + { + "name": "Persian Pink", + "hex": "#F77FBE" + }, + { + "name": "Persian Plum", + "hex": "#701C1C" + }, + { + "name": "Persian Red", + "hex": "#CC3333" + }, + { + "name": "Persian Rose", + "hex": "#FE28A2" + }, + { + "name": "Persimmon", + "hex": "#EC5800" + }, + { + "name": "Peru", + "hex": "#CD853F" + }, + { + "name": "Pewter Blue", + "hex": "#8BA8B7" + }, + { + "name": "Phlox", + "hex": "#DF00FF" + }, + { + "name": "Phthalo Blue", + "hex": "#000F89" + }, + { + "name": "Phthalo Green", + "hex": "#123524" + }, + { + "name": "Picton Blue", + "hex": "#45B1E8" + }, + { + "name": "Pictorial Carmine", + "hex": "#C30B4E" + }, + { + "name": "Piggy Pink", + "hex": "#FDDDE6" + }, + { + "name": "Pine Green", + "hex": "#01796F" + }, + { + "name": "Pineapple", + "hex": "#563C0D" + }, + { + "name": "Pink", + "hex": "#FFC0CB" + }, + { + "name": "Pink (Pantone)", + "hex": "#D74894" + }, + { + "name": "Pink Flamingo", + "hex": "#FC74FD" + }, + { + "name": "Pink Lace", + "hex": "#FFDDF4" + }, + { + "name": "Pink Lavender", + "hex": "#D8B2D1" + }, + { + "name": "Pink-Orange", + "hex": "#FF9966" + }, + { + "name": "Pink Pearl", + "hex": "#E7ACCF" + }, + { + "name": "Pink Raspberry", + "hex": "#980036" + }, + { + "name": "Pink Sherbet", + "hex": "#F78FA7" + }, + { + "name": "Pistachio", + "hex": "#93C572" + }, + { + "name": "Pixie Powder", + "hex": "#391285" + }, + { + "name": "Platinum", + "hex": "#E5E4E2" + }, + { + "name": "Plum", + "hex": "#8E4585" + }, + { + "name": "Plum (Web)", + "hex": "#DDA0DD" + }, + { + "name": "Plump Purple", + "hex": "#5946B2" + }, + { + "name": "Polished Pine", + "hex": "#5DA493" + }, + { + "name": "Pomp And Power", + "hex": "#86608E" + }, + { + "name": "Popstar", + "hex": "#BE4F62" + }, + { + "name": "Portland Orange", + "hex": "#FF5A36" + }, + { + "name": "Powder Blue", + "hex": "#B0E0E6" + }, + { + "name": "Princess Perfume", + "hex": "#FF85CF" + }, + { + "name": "Princeton Orange", + "hex": "#F58025" + }, + { + "name": "Prune", + "hex": "#701C1C" + }, + { + "name": "Prussian Blue", + "hex": "#003153" + }, + { + "name": "Psychedelic Purple", + "hex": "#DF00FF" + }, + { + "name": "Puce", + "hex": "#CC8899" + }, + { + "name": "Puce Red", + "hex": "#722F37" + }, + { + "name": "Pullman Brown (UPS Brown)", + "hex": "#644117" + }, + { + "name": "Pullman Green", + "hex": "#3B331C" + }, + { + "name": "Pumpkin", + "hex": "#FF7518" + }, + { + "name": "Purple (HTML)", + "hex": "#800080" + }, + { + "name": "Purple (Munsell)", + "hex": "#9F00C5" + }, + { + "name": "Purple (X11)", + "hex": "#A020F0" + }, + { + "name": "Purple Heart", + "hex": "#69359C" + }, + { + "name": "Purple Mountain Majesty", + "hex": "#9678B6" + }, + { + "name": "Purple Navy", + "hex": "#4E5180" + }, + { + "name": "Purple Pizzazz", + "hex": "#FE4EDA" + }, + { + "name": "Purple Plum", + "hex": "#9C51B6" + }, + { + "name": "Purple Taupe", + "hex": "#50404D" + }, + { + "name": "Purpureus", + "hex": "#9A4EAE" + }, + { + "name": "Quartz", + "hex": "#51484F" + }, + { + "name": "Queen Blue", + "hex": "#436B95" + }, + { + "name": "Queen Pink", + "hex": "#E8CCD7" + }, + { + "name": "Quick Silver", + "hex": "#A6A6A6" + }, + { + "name": "Quinacridone Magenta", + "hex": "#8E3A59" + }, + { + "name": "Rackley", + "hex": "#5D8AA8" + }, + { + "name": "Radical Red", + "hex": "#FF355E" + }, + { + "name": "Raisin Black", + "hex": "#242124" + }, + { + "name": "Rajah", + "hex": "#FBAB60" + }, + { + "name": "Raspberry", + "hex": "#E30B5D" + }, + { + "name": "Raspberry Glace", + "hex": "#915F6D" + }, + { + "name": "Raspberry Pink", + "hex": "#E25098" + }, + { + "name": "Raspberry Rose", + "hex": "#B3446C" + }, + { + "name": "Raw Sienna", + "hex": "#D68A59" + }, + { + "name": "Raw Umber", + "hex": "#826644" + }, + { + "name": "Razzle Dazzle Rose", + "hex": "#FF33CC" + }, + { + "name": "Razzmatazz", + "hex": "#E3256B" + }, + { + "name": "Razzmic Berry", + "hex": "#8D4E85" + }, + { + "name": "Rebecca Purple", + "hex": "#663399" + }, + { + "name": "Red", + "hex": "#FF0000" + }, + { + "name": "Red (Crayola)", + "hex": "#EE204D" + }, + { + "name": "Red (Munsell)", + "hex": "#F2003C" + }, + { + "name": "Red (NCS)", + "hex": "#C40233" + }, + { + "name": "Red (Pantone)", + "hex": "#ED2939" + }, + { + "name": "Red (Pigment)", + "hex": "#ED1C24" + }, + { + "name": "Red (RYB)", + "hex": "#FE2712" + }, + { + "name": "Red-Brown", + "hex": "#A52A2A" + }, + { + "name": "Red Devil", + "hex": "#860111" + }, + { + "name": "Red-Orange", + "hex": "#FF5349" + }, + { + "name": "Red-Purple", + "hex": "#E40078" + }, + { + "name": "Red Salsa", + "hex": "#FD3A4A" + }, + { + "name": "Red-Violet", + "hex": "#C71585" + }, + { + "name": "Redwood", + "hex": "#A45A52" + }, + { + "name": "Regalia", + "hex": "#522D80" + }, + { + "name": "Registration Black", + "hex": "#000000" + }, + { + "name": "Resolution Blue", + "hex": "#002387" + }, + { + "name": "Rhythm", + "hex": "#777696" + }, + { + "name": "Rich Black", + "hex": "#004040" + }, + { + "name": "Rich Black (FOGRA29)", + "hex": "#010B13" + }, + { + "name": "Rich Black (FOGRA39)", + "hex": "#010203" + }, + { + "name": "Rich Brilliant Lavender", + "hex": "#F1A7FE" + }, + { + "name": "Rich Carmine", + "hex": "#D70040" + }, + { + "name": "Rich Electric Blue", + "hex": "#0892D0" + }, + { + "name": "Rich Lavender", + "hex": "#A76BCF" + }, + { + "name": "Rich Lilac", + "hex": "#B666D2" + }, + { + "name": "Rich Maroon", + "hex": "#B03060" + }, + { + "name": "Rifle Green", + "hex": "#444C38" + }, + { + "name": "Roast Coffee", + "hex": "#704241" + }, + { + "name": "Robin Egg Blue", + "hex": "#00CCCC" + }, + { + "name": "Rocket Metallic", + "hex": "#8A7F80" + }, + { + "name": "Roman Silver", + "hex": "#838996" + }, + { + "name": "Rose", + "hex": "#FF007F" + }, + { + "name": "Rose Bonbon", + "hex": "#F9429E" + }, + { + "name": "Rose Dust", + "hex": "#9E5E6F" + }, + { + "name": "Rose Ebony", + "hex": "#674846" + }, + { + "name": "Rose Gold", + "hex": "#B76E79" + }, + { + "name": "Rose Madder", + "hex": "#E32636" + }, + { + "name": "Rose Pink", + "hex": "#FF66CC" + }, + { + "name": "Rose Quartz", + "hex": "#AA98A9" + }, + { + "name": "Rose Red", + "hex": "#C21E56" + }, + { + "name": "Rose Taupe", + "hex": "#905D5D" + }, + { + "name": "Rose Vale", + "hex": "#AB4E52" + }, + { + "name": "Rosewood", + "hex": "#65000B" + }, + { + "name": "Rosso Corsa", + "hex": "#D40000" + }, + { + "name": "Rosy Brown", + "hex": "#BC8F8F" + }, + { + "name": "Royal Azure", + "hex": "#0038A8" + }, + { + "name": "Royal Blue", + "hex": "#002366" + }, + { + "name": "Royal Blue", + "hex": "#4169E1" + }, + { + "name": "Royal Fuchsia", + "hex": "#CA2C92" + }, + { + "name": "Royal Purple", + "hex": "#7851A9" + }, + { + "name": "Royal Yellow", + "hex": "#FADA5E" + }, + { + "name": "Ruber", + "hex": "#CE4676" + }, + { + "name": "Rubine Red", + "hex": "#D10056" + }, + { + "name": "Ruby", + "hex": "#E0115F" + }, + { + "name": "Ruby Red", + "hex": "#9B111E" + }, + { + "name": "Ruddy", + "hex": "#FF0028" + }, + { + "name": "Ruddy Brown", + "hex": "#BB6528" + }, + { + "name": "Ruddy Pink", + "hex": "#E18E96" + }, + { + "name": "Rufous", + "hex": "#A81C07" + }, + { + "name": "Russet", + "hex": "#80461B" + }, + { + "name": "Russian Green", + "hex": "#679267" + }, + { + "name": "Russian Violet", + "hex": "#32174D" + }, + { + "name": "Rust", + "hex": "#B7410E" + }, + { + "name": "Rusty Red", + "hex": "#DA2C43" + }, + { + "name": "Sacramento State Green", + "hex": "#00563F" + }, + { + "name": "Saddle Brown", + "hex": "#8B4513" + }, + { + "name": "Safety Orange", + "hex": "#FF7800" + }, + { + "name": "Safety Orange (Blaze Orange)", + "hex": "#FF6700" + }, + { + "name": "Safety Yellow", + "hex": "#EED202" + }, + { + "name": "Saffron", + "hex": "#F4C430" + }, + { + "name": "Sage", + "hex": "#BCB88A" + }, + { + "name": "St. Patrick's Blue", + "hex": "#23297A" + }, + { + "name": "Salmon", + "hex": "#FA8072" + }, + { + "name": "Salmon Pink", + "hex": "#FF91A4" + }, + { + "name": "Sand", + "hex": "#C2B280" + }, + { + "name": "Sand Dune", + "hex": "#967117" + }, + { + "name": "Sandstorm", + "hex": "#ECD540" + }, + { + "name": "Sandy Brown", + "hex": "#F4A460" + }, + { + "name": "Sandy Tan", + "hex": "#FDD9B5" + }, + { + "name": "Sandy Taupe", + "hex": "#967117" + }, + { + "name": "Sangria", + "hex": "#92000A" + }, + { + "name": "Sap Green", + "hex": "#507D2A" + }, + { + "name": "Sapphire", + "hex": "#0F52BA" + }, + { + "name": "Sapphire Blue", + "hex": "#0067A5" + }, + { + "name": "Sasquatch Socks", + "hex": "#FF4681" + }, + { + "name": "Satin Sheen Gold", + "hex": "#CBA135" + }, + { + "name": "Scarlet", + "hex": "#FF2400" + }, + { + "name": "Scarlet", + "hex": "#FD0E35" + }, + { + "name": "Schauss Pink", + "hex": "#FF91AF" + }, + { + "name": "School Bus Yellow", + "hex": "#FFD800" + }, + { + "name": "Screamin' Green", + "hex": "#66FF66" + }, + { + "name": "Sea Blue", + "hex": "#006994" + }, + { + "name": "Sea Foam Green", + "hex": "#9FE2BF" + }, + { + "name": "Sea Green", + "hex": "#2E8B57" + }, + { + "name": "Sea Serpent", + "hex": "#4BC7CF" + }, + { + "name": "Seal Brown", + "hex": "#59260B" + }, + { + "name": "Seashell", + "hex": "#FFF5EE" + }, + { + "name": "Selective Yellow", + "hex": "#FFBA00" + }, + { + "name": "Sepia", + "hex": "#704214" + }, + { + "name": "Shadow", + "hex": "#8A795D" + }, + { + "name": "Shadow Blue", + "hex": "#778BA5" + }, + { + "name": "Shampoo", + "hex": "#FFCFF1" + }, + { + "name": "Shamrock Green", + "hex": "#009E60" + }, + { + "name": "Sheen Green", + "hex": "#8FD400" + }, + { + "name": "Shimmering Blush", + "hex": "#D98695" + }, + { + "name": "Shiny Shamrock", + "hex": "#5FA778" + }, + { + "name": "Shocking Pink", + "hex": "#FC0FC0" + }, + { + "name": "Shocking Pink (Crayola)", + "hex": "#FF6FFF" + }, + { + "name": "Sienna", + "hex": "#882D17" + }, + { + "name": "Silver", + "hex": "#C0C0C0" + }, + { + "name": "Silver Chalice", + "hex": "#ACACAC" + }, + { + "name": "Silver Lake Blue", + "hex": "#5D89BA" + }, + { + "name": "Silver Pink", + "hex": "#C4AEAD" + }, + { + "name": "Silver Sand", + "hex": "#BFC1C2" + }, + { + "name": "Sinopia", + "hex": "#CB410B" + }, + { + "name": "Sizzling Red", + "hex": "#FF3855" + }, + { + "name": "Sizzling Sunrise", + "hex": "#FFDB00" + }, + { + "name": "Skobeloff", + "hex": "#007474" + }, + { + "name": "Sky Blue", + "hex": "#87CEEB" + }, + { + "name": "Sky Magenta", + "hex": "#CF71AF" + }, + { + "name": "Slate Blue", + "hex": "#6A5ACD" + }, + { + "name": "Slate Gray", + "hex": "#708090" + }, + { + "name": "Smalt (Dark Powder Blue)", + "hex": "#003399" + }, + { + "name": "Slimy Green", + "hex": "#299617" + }, + { + "name": "Smashed Pumpkin", + "hex": "#FF6D3A" + }, + { + "name": "Smitten", + "hex": "#C84186" + }, + { + "name": "Smoke", + "hex": "#738276" + }, + { + "name": "Smokey Topaz", + "hex": "#832A0D" + }, + { + "name": "Smoky Black", + "hex": "#100C08" + }, + { + "name": "Smoky Topaz", + "hex": "#933D41" + }, + { + "name": "Snow", + "hex": "#FFFAFA" + }, + { + "name": "Soap", + "hex": "#CEC8EF" + }, + { + "name": "Solid Pink", + "hex": "#893843" + }, + { + "name": "Sonic Silver", + "hex": "#757575" + }, + { + "name": "Spartan Crimson", + "hex": "#9E1316" + }, + { + "name": "Space Cadet", + "hex": "#1D2951" + }, + { + "name": "Spanish Bistre", + "hex": "#807532" + }, + { + "name": "Spanish Blue", + "hex": "#0070B8" + }, + { + "name": "Spanish Carmine", + "hex": "#D10047" + }, + { + "name": "Spanish Crimson", + "hex": "#E51A4C" + }, + { + "name": "Spanish Gray", + "hex": "#989898" + }, + { + "name": "Spanish Green", + "hex": "#009150" + }, + { + "name": "Spanish Orange", + "hex": "#E86100" + }, + { + "name": "Spanish Pink", + "hex": "#F7BFBE" + }, + { + "name": "Spanish Red", + "hex": "#E60026" + }, + { + "name": "Spanish Sky Blue", + "hex": "#00FFFF" + }, + { + "name": "Spanish Violet", + "hex": "#4C2882" + }, + { + "name": "Spanish Viridian", + "hex": "#007F5C" + }, + { + "name": "Spicy Mix", + "hex": "#8B5f4D" + }, + { + "name": "Spiro Disco Ball", + "hex": "#0FC0FC" + }, + { + "name": "Spring Bud", + "hex": "#A7FC00" + }, + { + "name": "Spring Frost", + "hex": "#87FF2A" + }, + { + "name": "Spring Green", + "hex": "#00FF7F" + }, + { + "name": "Star Command Blue", + "hex": "#007BB8" + }, + { + "name": "Steel Blue", + "hex": "#4682B4" + }, + { + "name": "Steel Pink", + "hex": "#CC33CC" + }, + { + "name": "Steel Teal", + "hex": "#5F8A8B" + }, + { + "name": "Stil De Grain Yellow", + "hex": "#FADA5E" + }, + { + "name": "Stizza", + "hex": "#990000" + }, + { + "name": "Stormcloud", + "hex": "#4F666A" + }, + { + "name": "Straw", + "hex": "#E4D96F" + }, + { + "name": "Strawberry", + "hex": "#FC5A8D" + }, + { + "name": "Sugar Plum", + "hex": "#914E75" + }, + { + "name": "Sunburnt Cyclops", + "hex": "#FF404C" + }, + { + "name": "Sunglow", + "hex": "#FFCC33" + }, + { + "name": "Sunny", + "hex": "#F2F27A" + }, + { + "name": "Sunray", + "hex": "#E3AB57" + }, + { + "name": "Sunset", + "hex": "#FAD6A5" + }, + { + "name": "Sunset Orange", + "hex": "#FD5E53" + }, + { + "name": "Super Pink", + "hex": "#CF6BA9" + }, + { + "name": "Sweet Brown", + "hex": "#A83731" + }, + { + "name": "Tan", + "hex": "#D2B48C" + }, + { + "name": "Tangelo", + "hex": "#F94D00" + }, + { + "name": "Tangerine", + "hex": "#F28500" + }, + { + "name": "Tangerine Yellow", + "hex": "#FFCC00" + }, + { + "name": "Tango Pink", + "hex": "#E4717A" + }, + { + "name": "Tart Orange", + "hex": "#FB4D46" + }, + { + "name": "Taupe", + "hex": "#483C32" + }, + { + "name": "Taupe Gray", + "hex": "#8B8589" + }, + { + "name": "Tea Green", + "hex": "#D0F0C0" + }, + { + "name": "Tea Rose", + "hex": "#F88379" + }, + { + "name": "Tea Rose", + "hex": "#F4C2C2" + }, + { + "name": "Teal", + "hex": "#008080" + }, + { + "name": "Teal Blue", + "hex": "#367588" + }, + { + "name": "Teal Deer", + "hex": "#99E6B3" + }, + { + "name": "Teal Green", + "hex": "#00827F" + }, + { + "name": "Telemagenta", + "hex": "#CF3476" + }, + { + "name": "Tenne (Tawny)", + "hex": "#CD5700" + }, + { + "name": "Terra Cotta", + "hex": "#E2725B" + }, + { + "name": "Thistle", + "hex": "#D8BFD8" + }, + { + "name": "Thulian Pink", + "hex": "#DE6FA1" + }, + { + "name": "Tickle Me Pink", + "hex": "#FC89AC" + }, + { + "name": "Tiffany Blue", + "hex": "#0ABAB5" + }, + { + "name": "Tiger's Eye", + "hex": "#E08D3C" + }, + { + "name": "Timberwolf", + "hex": "#DBD7D2" + }, + { + "name": "Titanium Yellow", + "hex": "#EEE600" + }, + { + "name": "Tomato", + "hex": "#FF6347" + }, + { + "name": "Toolbox", + "hex": "#746CC0" + }, + { + "name": "Topaz", + "hex": "#FFC87C" + }, + { + "name": "Tractor Red", + "hex": "#FD0E35" + }, + { + "name": "Trolley Grey", + "hex": "#808080" + }, + { + "name": "Tropical Rain Forest", + "hex": "#00755E" + }, + { + "name": "Tropical Violet", + "hex": "#CDA4DE" + }, + { + "name": "True Blue", + "hex": "#0073CF" + }, + { + "name": "Tufts Blue", + "hex": "#3E8EDE" + }, + { + "name": "Tulip", + "hex": "#FF878D" + }, + { + "name": "Tumbleweed", + "hex": "#DEAA88" + }, + { + "name": "Turkish Rose", + "hex": "#B57281" + }, + { + "name": "Turquoise", + "hex": "#40E0D0" + }, + { + "name": "Turquoise Blue", + "hex": "#00FFEF" + }, + { + "name": "Turquoise Green", + "hex": "#A0D6B4" + }, + { + "name": "Turquoise Surf", + "hex": "#00C5CD" + }, + { + "name": "Turtle Green", + "hex": "#8A9A5B" + }, + { + "name": "Tuscan", + "hex": "#FAD6A5" + }, + { + "name": "Tuscan Brown", + "hex": "#6F4E37" + }, + { + "name": "Tuscan Red", + "hex": "#7C4848" + }, + { + "name": "Tuscan Tan", + "hex": "#A67B5B" + }, + { + "name": "Tuscany", + "hex": "#C09999" + }, + { + "name": "Twilight Lavender", + "hex": "#8A496B" + }, + { + "name": "Tyrian Purple", + "hex": "#66023C" + }, + { + "name": "UA Blue", + "hex": "#0033AA" + }, + { + "name": "UA Red", + "hex": "#D9004C" + }, + { + "name": "Ube", + "hex": "#8878C3" + }, + { + "name": "UCLA Blue", + "hex": "#536895" + }, + { + "name": "UCLA Gold", + "hex": "#FFB300" + }, + { + "name": "UFO Green", + "hex": "#3CD070" + }, + { + "name": "Ultramarine", + "hex": "#3F00FF" + }, + { + "name": "Ultramarine Blue", + "hex": "#4166F5" + }, + { + "name": "Ultra Pink", + "hex": "#FF6FFF" + }, + { + "name": "Ultra Red", + "hex": "#FC6C85" + }, + { + "name": "Umber", + "hex": "#635147" + }, + { + "name": "Unbleached Silk", + "hex": "#FFDDCA" + }, + { + "name": "United Nations Blue", + "hex": "#5B92E5" + }, + { + "name": "University Of California Gold", + "hex": "#B78727" + }, + { + "name": "Unmellow Yellow", + "hex": "#FFFF66" + }, + { + "name": "UP Forest Green", + "hex": "#014421" + }, + { + "name": "UP Maroon", + "hex": "#7B1113" + }, + { + "name": "Upsdell Red", + "hex": "#AE2029" + }, + { + "name": "Urobilin", + "hex": "#E1AD21" + }, + { + "name": "USAFA Blue", + "hex": "#004F98" + }, + { + "name": "USC Cardinal", + "hex": "#990000" + }, + { + "name": "USC Gold", + "hex": "#FFCC00" + }, + { + "name": "University Of Tennessee Orange", + "hex": "#F77F00" + }, + { + "name": "Utah Crimson", + "hex": "#D3003F" + }, + { + "name": "Van Dyke Brown", + "hex": "#664228" + }, + { + "name": "Vanilla", + "hex": "#F3E5AB" + }, + { + "name": "Vanilla Ice", + "hex": "#F38FA9" + }, + { + "name": "Vegas Gold", + "hex": "#C5B358" + }, + { + "name": "Venetian Red", + "hex": "#C80815" + }, + { + "name": "Verdigris", + "hex": "#43B3AE" + }, + { + "name": "Vermilion", + "hex": "#E34234" + }, + { + "name": "Vermilion", + "hex": "#D9381E" + }, + { + "name": "Veronica", + "hex": "#A020F0" + }, + { + "name": "Very Light Azure", + "hex": "#74BBFB" + }, + { + "name": "Very Light Blue", + "hex": "#6666FF" + }, + { + "name": "Very Light Malachite Green", + "hex": "#64E986" + }, + { + "name": "Very Light Tangelo", + "hex": "#FFB077" + }, + { + "name": "Very Pale Orange", + "hex": "#FFDFBF" + }, + { + "name": "Very Pale Yellow", + "hex": "#FFFFBF" + }, + { + "name": "Violet", + "hex": "#8F00FF" + }, + { + "name": "Violet (Color Wheel)", + "hex": "#7F00FF" + }, + { + "name": "Violet (RYB)", + "hex": "#8601AF" + }, + { + "name": "Violet (Web)", + "hex": "#EE82EE" + }, + { + "name": "Violet-Blue", + "hex": "#324AB2" + }, + { + "name": "Violet-Red", + "hex": "#F75394" + }, + { + "name": "Viridian", + "hex": "#40826D" + }, + { + "name": "Viridian Green", + "hex": "#009698" + }, + { + "name": "Vista Blue", + "hex": "#7C9ED9" + }, + { + "name": "Vivid Amber", + "hex": "#CC9900" + }, + { + "name": "Vivid Auburn", + "hex": "#922724" + }, + { + "name": "Vivid Burgundy", + "hex": "#9F1D35" + }, + { + "name": "Vivid Cerise", + "hex": "#DA1D81" + }, + { + "name": "Vivid Cerulean", + "hex": "#00AAEE" + }, + { + "name": "Vivid Crimson", + "hex": "#CC0033" + }, + { + "name": "Vivid Gamboge", + "hex": "#FF9900" + }, + { + "name": "Vivid Lime Green", + "hex": "#A6D608" + }, + { + "name": "Vivid Malachite", + "hex": "#00CC33" + }, + { + "name": "Vivid Mulberry", + "hex": "#B80CE3" + }, + { + "name": "Vivid Orange", + "hex": "#FF5F00" + }, + { + "name": "Vivid Orange Peel", + "hex": "#FFA000" + }, + { + "name": "Vivid Orchid", + "hex": "#CC00FF" + }, + { + "name": "Vivid Raspberry", + "hex": "#FF006C" + }, + { + "name": "Vivid Red", + "hex": "#F70D1A" + }, + { + "name": "Vivid Red-Tangelo", + "hex": "#DF6124" + }, + { + "name": "Vivid Sky Blue", + "hex": "#00CCFF" + }, + { + "name": "Vivid Tangelo", + "hex": "#F07427" + }, + { + "name": "Vivid Tangerine", + "hex": "#FFA089" + }, + { + "name": "Vivid Vermilion", + "hex": "#E56024" + }, + { + "name": "Vivid Violet", + "hex": "#9F00FF" + }, + { + "name": "Vivid Yellow", + "hex": "#FFE302" + }, + { + "name": "Volt", + "hex": "#CEFF00" + }, + { + "name": "Wageningen Green", + "hex": "#34B233" + }, + { + "name": "Warm Black", + "hex": "#004242" + }, + { + "name": "Waterspout", + "hex": "#A4F4F9" + }, + { + "name": "Weldon Blue", + "hex": "#7C98AB" + }, + { + "name": "Wenge", + "hex": "#645452" + }, + { + "name": "Wheat", + "hex": "#F5DEB3" + }, + { + "name": "White", + "hex": "#FFFFFF" + }, + { + "name": "White Smoke", + "hex": "#F5F5F5" + }, + { + "name": "Wild Blue Yonder", + "hex": "#A2ADD0" + }, + { + "name": "Wild Orchid", + "hex": "#D470A2" + }, + { + "name": "Wild Strawberry", + "hex": "#FF43A4" + }, + { + "name": "Wild Watermelon", + "hex": "#FC6C85" + }, + { + "name": "Willpower Orange", + "hex": "#FD5800" + }, + { + "name": "Windsor Tan", + "hex": "#A75502" + }, + { + "name": "Wine", + "hex": "#722F37" + }, + { + "name": "Wine Dregs", + "hex": "#673147" + }, + { + "name": "Winter Sky", + "hex": "#FF007C" + }, + { + "name": "Winter Wizard", + "hex": "#A0E6FF" + }, + { + "name": "Wintergreen Dream", + "hex": "#56887D" + }, + { + "name": "Wisteria", + "hex": "#C9A0DC" + }, + { + "name": "Wood Brown", + "hex": "#C19A6B" + }, + { + "name": "Xanadu", + "hex": "#738678" + }, + { + "name": "Yale Blue", + "hex": "#0F4D92" + }, + { + "name": "Yankees Blue", + "hex": "#1C2841" + }, + { + "name": "Yellow", + "hex": "#FFFF00" + }, + { + "name": "Yellow (Crayola)", + "hex": "#FCE883" + }, + { + "name": "Yellow (Munsell)", + "hex": "#EFCC00" + }, + { + "name": "Yellow (NCS)", + "hex": "#FFD300" + }, + { + "name": "Yellow (Pantone)", + "hex": "#FEDF00" + }, + { + "name": "Yellow (Process)", + "hex": "#FFEF00" + }, + { + "name": "Yellow (RYB)", + "hex": "#FEFE33" + }, + { + "name": "Yellow-Green", + "hex": "#9ACD32" + }, + { + "name": "Yellow Orange", + "hex": "#FFAE42" + }, + { + "name": "Yellow Rose", + "hex": "#FFF000" + }, + { + "name": "Yellow Sunshine", + "hex": "#FFF700" + }, + { + "name": "Zaffre", + "hex": "#0014A8" + }, + { + "name": "Zinnwaldite Brown", + "hex": "#2C1608" + }, + { + "name": "Zomp", + "hex": "#39A78E" + } +]` diff --git a/core/docker-compose.yml b/core/docker-compose.yml new file mode 100644 index 000000000..c54be4f81 --- /dev/null +++ b/core/docker-compose.yml @@ -0,0 +1,14 @@ +version: '3.3' +services: + mongo: + image: mongo:latest + container_name: mongo + restart: always + ports: + - "27017:27017" + redis: + image: redis:latest + container_name: redis + restart: always + ports: + - "6379:6379" diff --git a/core/docs/.gitignore b/core/docs/.gitignore new file mode 100644 index 000000000..2ac2692af --- /dev/null +++ b/core/docs/.gitignore @@ -0,0 +1,6 @@ +.idea +node_modules +dist +build +tmp +yarn.lock diff --git a/core/docs/api/index.html b/core/docs/api/index.html new file mode 100644 index 000000000..bb21255ef --- /dev/null +++ b/core/docs/api/index.html @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/core/docs/api/openapi.yaml b/core/docs/api/openapi.yaml new file mode 100644 index 000000000..82a35ef77 --- /dev/null +++ b/core/docs/api/openapi.yaml @@ -0,0 +1,4225 @@ +openapi: 3.0.0 +info: + title: Crawlab API + version: 0.6.0 + description: Crawlab API + license: + name: BSD-3-Clause + url: https://github.com/crawlab-team/crawlab-pro/blob/main/LICENSE + +servers: + - url: http://localhost:8000 + description: Local API + - url: http://localhost:8080/api + description: Docker API + - url: https://demo-pro.crawlab.cn/api + description: Demo API + +tags: + - name: login + description: Login related + x-displayName: Login + - name: version + description: Version related + x-displayName: Version + - name: node + description: Node related + x-displayName: Node + - name: project + description: Project related + x-displayName: Project + +x-tagGroups: + - name: Anomalous + tags: + - login + - version + - name: Auth Required + tags: + - node + - project + +paths: + # login + /login: + post: + tags: [ login ] + operationId: postLogin + summary: Login + requestBody: + content: + application/json: + schema: + type: object + properties: + username: + type: string + password: + type: string + example: + username: admin + password: admin + responses: + 200: + description: Login successful + content: + application/json: + schema: + $ref: '#/components/schemas/PostLoginResponse' + 401: + description: Login failed + content: + application/json: + schema: + $ref: '#/components/schemas/UnauthorizedErrorResponse' + /logout: + post: + tags: [ login ] + operationId: postLogout + summary: Logout + responses: + 200: + description: Logout successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + + # version + /version: + get: + tags: [ version ] + operationId: getVersion + summary: Get version + responses: + 200: + description: Get version successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetVersionResponse' + example: + status: ok + data: v0.6.0 + + # node + /nodes: + get: + tags: [ node ] + operationId: getNodes + summary: Get nodes + security: + - apiToken: [ ] + parameters: + - in: query + name: page + description: Page number + required: false + schema: + type: integer + default: 1 + - in: query + name: page_size + description: Page size + required: false + schema: + type: integer + default: 10 + responses: + 200: + description: Get nodes successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetNodesResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + put: + tags: [ node ] + operationId: putNode + summary: Update node + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/BatchRequestPayloadWithStringData' + responses: + 200: + description: Update node successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + delete: + tags: [ node ] + operationId: deleteNodes + summary: Delete nodes + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/BatchRequestPayload' + responses: + 200: + description: Delete node successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /nodes/{id}: + get: + tags: [ node ] + operationId: getNode + summary: Get node + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Node ID + required: true + schema: + type: string + responses: + 200: + description: Get node successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetNodeResponse' + 404: + description: Node not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + put: + tags: [ node ] + operationId: putNode + summary: Update node + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Node ID + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Node' + responses: + 200: + description: Update node successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Node not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + + # project + /projects: + get: + tags: [ project ] + operationId: getProjects + summary: Get projects + security: + - apiToken: [ ] + parameters: + - in: query + name: page + schema: + type: integer + - in: query + name: size + schema: + type: integer + responses: + 200: + description: Get projects successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetProjectsResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + post: + tags: [ project ] + operationId: postProject + summary: Create project + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostProjectRequest' + responses: + 200: + description: Create project successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + put: + tags: [ project ] + operationId: putProjects + summary: Update projects + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/BatchRequestPayloadWithStringData' + responses: + 200: + description: Update project successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Project not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + delete: + tags: [ project ] + operationId: deleteProjects + summary: Delete projects + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/BatchRequestPayload' + responses: + 200: + description: Delete project successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Project not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /projects/batch: + post: + tags: [ project ] + operationId: postProjects + summary: Create projects + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/PostProjectRequest' + responses: + 200: + description: Update project successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /projects/{id}: + get: + tags: [ project ] + operationId: getProject + summary: Get project + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Project ID + required: true + schema: + type: string + responses: + 200: + description: Get project successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetProjectResponse' + 404: + description: Project not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + post: + tags: [ project ] + operationId: postProject + summary: Update project + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Project ID + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Project' + responses: + 200: + description: Update project successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Project not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + delete: + tags: [ project ] + operationId: deleteProject + summary: Delete project + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Project ID + required: true + schema: + type: string + responses: + 200: + description: Delete project successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Project not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + + # spider + /spiders: + get: + tags: [ spider ] + operationId: getSpiders + summary: Get spiders + security: + - apiToken: [ ] + parameters: + - in: query + name: page + schema: + type: integer + - in: query + name: size + schema: + type: integer + responses: + 200: + description: Get spiders successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetSpidersResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + post: + tags: [ spider ] + operationId: postSpiders + summary: Create spiders + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/BatchRequestPayloadWithStringData' + responses: + 200: + description: Create spiders successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/batch: + post: + tags: [ spider ] + operationId: postSpiders + summary: Update spiders + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/PostSpiderRequest' + responses: + 200: + description: Update spiders successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}: + get: + tags: [ spider ] + operationId: getSpider + summary: Get spider + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + responses: + 200: + description: Get spider successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetSpiderResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + post: + tags: [ spider ] + operationId: postSpider + summary: Update spider + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Spider' + responses: + 200: + description: Update spider successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + delete: + tags: [ spider ] + operationId: deleteSpider + summary: Delete spider + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + /spiders/{id}/files/list: + get: + tags: [ spider ] + operationId: getSpiderFilesList + summary: Get spider files list + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + responses: + 200: + description: Get spider files list successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetSpiderFilesListResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/files/get: + get: + tags: [ spider ] + operationId: getSpiderFile + summary: Get spider file + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + - name: path + in: query + description: File path + required: true + schema: + type: string + responses: + 200: + description: Get spider file successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetSpiderFileResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/files/info: + get: + tags: [ spider ] + operationId: getSpiderFileInfo + summary: Get spider file info + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + - name: path + in: query + description: File path + required: true + schema: + type: string + responses: + 200: + description: Get spider file info successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetSpiderFileInfoResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/files/save: + post: + tags: [ spider ] + operationId: postSpiderFile + summary: Save spider file + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + - name: path + in: query + description: File path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostSpiderFileRequest' + responses: + 200: + description: Save spider file successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/files/save/dir: + post: + tags: [ spider ] + operationId: postSpiderFileDir + summary: Save spider file dir + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + - name: path + in: query + description: File path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostSpiderFileDirRequest' + responses: + 200: + description: Save spider file dir successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/files/renameFile: + post: + tags: [ spider ] + operationId: postSpiderFileRenameFile + summary: Rename spider file + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + - name: path + in: query + description: File path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostSpiderFileRenameFileRequest' + responses: + 200: + description: Rename spider file successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/files/delete: + post: + tags: [ spider ] + operationId: postSpiderFileDelete + summary: Delete spider file + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + - name: path + in: query + description: File path + required: true + schema: + type: string + responses: + 200: + description: Delete spider file successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/files/copy: + post: + tags: [ spider ] + operationId: postSpiderFileCopy + summary: Copy spider file + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + - name: path + in: query + description: File path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostSpiderFileCopyRequest' + responses: + 200: + description: Copy spider file successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/run: + post: + tags: [ spider ] + operationId: postSpiderRun + summary: Run spider + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostSpiderRunRequest' + responses: + 200: + description: Run spider successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/git: + get: + tags: [ spider ] + operationId: getSpiderGit + summary: Get spider git + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + responses: + 200: + description: Get spider git successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetSpiderGitResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/git/remote-refs: + get: + tags: [ spider ] + operationId: getSpiderGitRemoteRefs + summary: Get spider git remote refs + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + responses: + 200: + description: Get spider git remote refs successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetSpiderGitRemoteRefsResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/git/pull: + post: + tags: [ spider ] + operationId: postSpiderGitPull + summary: Pull spider git + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostSpiderGitPullRequest' + responses: + 200: + description: Pull spider git successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/git/commit: + post: + tags: [ spider ] + operationId: postSpiderGitCommit + summary: Commit spider git + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostSpiderGitCommitRequest' + responses: + 200: + description: Commit spider git successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/data-source: + get: + tags: [ spider ] + operationId: getSpiderDataSource + summary: Get spider data source + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + responses: + 200: + description: Get spider data source successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetSpiderDataSourceResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /spiders/{id}/data-source/{ds_id}: + post: + tags: [ spider ] + operationId: postSpiderDataSource + summary: Create spider data source + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Spider ID + required: true + schema: + type: string + - name: ds_id + in: path + description: Data source ID + required: true + schema: + type: string + responses: + 200: + description: Create spider data source successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Spider not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + + # schedule + /schedules: + get: + tags: [ schedule ] + operationId: getSchedules + summary: Get schedules + security: + - apiToken: [ ] + parameters: + - name: page + in: query + description: Page number + required: false + schema: + type: integer + - name: page_size + in: query + description: Page size + required: false + schema: + type: integer + responses: + 200: + description: Get schedules successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetSchedulesResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + post: + tags: [ schedule ] + operationId: postSchedule + summary: Create schedule + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostScheduleRequest' + responses: + 200: + description: Create schedule successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + put: + tags: [ schedule ] + operationId: putSchedule + summary: Update schedule + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Schedule ID + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/BatchRequestPayloadWithStringData' + responses: + 200: + description: Update schedule successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Schedule not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + delete: + tags: [ schedule ] + operationId: deleteSchedules + summary: Delete schedules + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/BatchRequestPayload' + responses: + 200: + description: Delete schedule successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Schedule not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /schedules/batch: + post: + tags: [ schedule ] + operationId: postSchedulesBatch + summary: Create schedules + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostSchedulesBatchRequest' + responses: + 200: + description: Create schedules successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /schedules/{id}: + get: + tags: [ schedule ] + operationId: getSchedule + summary: Get schedule + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Schedule ID + required: true + schema: + type: string + responses: + 200: + description: Get schedule successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetScheduleResponse' + 404: + description: Schedule not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + put: + tags: [ schedule ] + operationId: putSchedule + summary: Update schedule + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Schedule ID + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PutScheduleRequest' + responses: + 200: + description: Update schedule successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Schedule not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + delete: + tags: [ schedule ] + operationId: deleteSchedule + summary: Delete schedule + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Schedule ID + required: true + schema: + type: string + responses: + 200: + description: Delete schedule successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Schedule not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /schedules/{id}/enabled: + post: + tags: [ schedule ] + operationId: postScheduleEnabled + summary: Enable schedule + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Schedule ID + required: true + schema: + type: string + responses: + 200: + description: Enable schedule successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Schedule not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /schedules/{id}/disabled: + post: + tags: [ schedule ] + operationId: postScheduleDisabled + summary: Disable schedule + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Schedule ID + required: true + schema: + type: string + responses: + 200: + description: Disable schedule successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Schedule not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + + # task + /tasks: + get: + tags: [ task ] + operationId: getTasks + summary: Get tasks + security: + - apiToken: [ ] + parameters: + - name: page + in: query + description: Page number + required: false + schema: + type: integer + - name: size + in: query + description: Page size number + required: false + schema: + type: integer + responses: + 200: + description: Get tasks successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetTasksResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + delete: + tags: [ task ] + operationId: deleteTasks + summary: Delete tasks + security: + - apiToken: [ ] + parameters: + - name: ids + in: query + description: Task IDs + required: true + schema: + type: array + items: + type: string + responses: + 200: + description: Delete tasks successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /tasks/run: + post: + tags: [ task ] + operationId: postTaskRun + summary: Run task + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostTaskRunRequest' + responses: + 200: + description: Run task successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /tasks/{id}: + get: + tags: [ task ] + operationId: getTask + summary: Get task + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Task ID + required: true + schema: + type: string + responses: + 200: + description: Get task successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetTaskResponse' + 404: + description: Task not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + delete: + tags: [ task ] + operationId: deleteTask + summary: Delete task + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Task ID + required: true + schema: + type: string + responses: + 200: + description: Delete task successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Task not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /tasks/{id}/restart: + post: + tags: [ task ] + operationId: postTaskRestart + summary: Restart task + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Task ID + required: true + schema: + type: string + responses: + 200: + description: Restart task successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Task not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /tasks/{id}/cancel: + post: + tags: [ task ] + operationId: postTaskCancel + summary: Cancel task + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Task ID + required: true + schema: + type: string + responses: + 200: + description: Cancel task successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Task not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /tasks/{id}/logs: + get: + tags: [ task ] + operationId: getTaskLogs + summary: Get task logs + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Task ID + required: true + schema: + type: string + - name: page + in: query + description: Page number + required: false + schema: + type: integer + - name: size + in: query + description: Page size number + required: false + schema: + type: integer + responses: + 200: + description: Get task logs successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetTaskLogsResponse' + 404: + description: Task not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /tasks/{id}/data: + get: + tags: [ task ] + operationId: getTaskData + summary: Get task data + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Task ID + required: true + schema: + type: string + - name: page + in: query + description: Page number + required: false + schema: + type: integer + - name: size + in: query + description: Page size number + required: false + schema: + type: integer + responses: + 200: + description: Get task data successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetTaskDataResponse' + 404: + description: Task not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + + # user + /users: + get: + tags: [ user ] + operationId: getUsers + summary: Get users + security: + - apiToken: [ ] + parameters: + - name: page + in: query + description: Page number + required: false + schema: + type: integer + example: 1 + - name: size + in: query + description: Page size number + required: false + schema: + type: integer + example: 10 + responses: + 200: + description: Get users successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetUsersResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + post: + tags: [ user ] + operationId: postUser + summary: Create user + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostUserRequest' + responses: + 200: + description: Create user successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + put: + tags: [ user ] + operationId: putUser + summary: Update user + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: User ID + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/BatchRequestPayloadWithStringData' + responses: + 200: + description: Update user successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: User not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + delete: + tags: [ user ] + operationId: deleteUser + summary: Delete user + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: User ID + required: true + schema: + type: string + responses: + 200: + description: Delete user successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: User not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /users/{id}: + get: + tags: [ user ] + operationId: getUser + summary: Get user + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: User ID + required: true + schema: + type: string + responses: + 200: + description: Get user successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetUserResponse' + 404: + description: User not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + put: + tags: [ user ] + operationId: putUser + summary: Update user + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: User ID + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PutUserRequest' + responses: + 200: + description: Update user successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: User not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + delete: + tags: [ user ] + operationId: deleteUser + summary: Delete user + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: User ID + required: true + schema: + type: string + responses: + 200: + description: Delete user successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: User not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /users/{id}/change-password: + post: + tags: [ user ] + operationId: changePassword + summary: Change password + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: User ID + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostUserChangePasswordRequest' + responses: + 200: + description: Change password successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: User not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /users/me: + get: + tags: [ user ] + operationId: getMe + summary: Get me + security: + - apiToken: [ ] + responses: + 200: + description: Get me successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetUserResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + put: + tags: [ user ] + operationId: putMe + summary: Update me + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PutUserRequest' + responses: + 200: + description: Update me successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + + # token + /tokens: + get: + tags: [ token ] + operationId: getTokens + summary: Get tokens + security: + - apiToken: [ ] + parameters: + - name: page + in: query + description: Page number + required: false + schema: + type: integer + example: 1 + - name: size + in: query + description: Page size + required: false + schema: + type: integer + example: 10 + responses: + 200: + description: Get tokens successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetTokensResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + post: + tags: [ token ] + operationId: postToken + summary: Create token + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostTokenRequest' + responses: + 200: + description: Create token successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /tokens/{id}: + get: + tags: [ token ] + operationId: getToken + summary: Get token + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Token ID + required: true + schema: + type: string + responses: + 200: + description: Get token successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetTokenResponse' + 404: + description: Token not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + put: + tags: [ token ] + operationId: putToken + summary: Update token + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Token ID + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PutTokenRequest' + responses: + 200: + description: Update token successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Token not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + delete: + tags: [ token ] + operationId: deleteToken + summary: Delete token + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Token ID + required: true + schema: + type: string + responses: + 200: + description: Delete token successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Token not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + + # plugin + /plugins: + get: + tags: [ plugin ] + operationId: getPlugins + summary: Get plugins + security: + - apiToken: [ ] + parameters: + - name: page + in: query + description: Page number + required: false + schema: + type: integer + example: 1 + - name: size + in: query + description: Page size + required: false + schema: + type: integer + example: 10 + responses: + 200: + description: Get plugins successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetPluginsResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + post: + tags: [ plugin ] + operationId: postPlugin + summary: Create plugin + security: + - apiToken: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PostPluginRequest' + responses: + 200: + description: Create plugin successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /plugins/{id}: + get: + tags: [ plugin ] + operationId: getPlugin + summary: Get plugin + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Plugin ID + required: true + schema: + type: string + responses: + 200: + description: Get plugin successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetPluginResponse' + 404: + description: Plugin not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /plugins/{id}/start: + post: + tags: [ plugin ] + operationId: startPlugin + summary: Start plugin + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Plugin ID + required: true + schema: + type: string + responses: + 200: + description: Start plugin successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Plugin not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /plugins/{id}/stop: + post: + tags: [ plugin ] + operationId: stopPlugin + summary: Stop plugin + security: + - apiToken: [ ] + parameters: + - name: id + in: path + description: Plugin ID + required: true + schema: + type: string + responses: + 200: + description: Stop plugin successful + content: + application/json: + schema: + $ref: '#/components/schemas/EmptyResponse' + 404: + description: Plugin not found + content: + application/json: + schema: + $ref: '#/components/schemas/NotFoundErrorResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /plugins/public: + get: + tags: [ plugin ] + operationId: getPublicPlugins + summary: Get public plugins + security: + - apiToken: [ ] + responses: + 200: + description: Get public plugins successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetPublicPluginsResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + /plugins/public/info: + get: + tags: [ plugin ] + operationId: getPublicPluginsInfo + summary: Get public plugins info + security: + - apiToken: [ ] + parameters: + - in: query + name: full_name + description: Plugin full name + required: true + schema: + type: string + responses: + 200: + description: Get public plugins info successful + content: + application/json: + schema: + $ref: '#/components/schemas/GetPublicPluginsInfoResponse' + 400: + description: Bad request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequestErrorResponse' + +components: + schemas: + # base request bodies + BatchRequestPayload: + description: Batch request payload + properties: + ids: + type: array + description: Model IDs to operate + items: + type: string + BatchRequestPayloadWithStringData: + description: Batch request payload with string data + properties: + ids: + type: array + description: Model IDs to operate + items: + type: string + data: + type: string + description: JSON marshalled model fields data to operate + fields: + type: array + description: Fields to operate + items: + type: string + + # request bodies + PostProjectRequest: + description: Put project request body + properties: + name: + type: string + description: Project name + description: + type: string + description: Project description + PostSpiderRequest: + description: Put project request body + properties: + name: + type: string + description: Spider name + description: + type: string + description: Spider description + type: + type: string + description: Spider type + params: + type: string + description: Default spider task params + col_name: + type: string + description: Spider collection name + project_id: + type: string + description: Spider project ID + mode: + type: string + enum: + - random + - all-nodes + - selected-nodes + description: Default spider task mode + PostSpiderFileRequest: + description: Post spider file request body + properties: + path: + type: string + description: File path + data: + type: string + description: File data + PostSpiderFileDirRequest: + description: Post spider file dir request body + properties: + path: + type: string + description: File path + PostSpiderFileRenameFileRequest: + description: Post spider file rename file request body + properties: + path: + type: string + description: File path + new_path: + type: string + description: New file path + PostSpiderFileCopyRequest: + description: Post spider file copy request body + properties: + path: + type: string + description: File path + new_path: + type: string + description: New file path + PostSpiderRunRequest: + description: Post spider run request body + properties: + params: + type: string + description: Spider task params + mode: + type: string + enum: + - random + - all-nodes + - selected-nodes + description: Spider task mode + example: + mode: random + PostSpiderGitPullRequest: + description: Post spider git pull request body + properties: + branch: + type: string + description: Git branch + commit: + type: string + description: Git commit + remote: + type: string + description: Git remote + ref: + type: string + description: Git ref + PostSpiderGitCommitRequest: + description: Post spider git commit request body + $ref: '#/components/schemas/GitPayload' + PostScheduleRequest: + description: Post schedule request body + properties: + name: + type: string + description: Schedule name + description: + type: string + description: Schedule description + spider_id: + type: string + description: Schedule spider ID + cron: + type: string + description: Cron expression + cmd: + type: string + description: Execute Command + params: + type: string + description: Schedule params + mode: + $ref: '#/components/schemas/TaskMode' + PostSchedulesBatchRequest: + description: Post schedules batch request body + properties: + data: + type: array + description: Schedules data + items: + $ref: '#/components/schemas/PostScheduleRequest' + PutScheduleRequest: + description: Put schedule request body + properties: + name: + type: string + description: Schedule name + description: + type: string + description: Schedule description + project_id: + type: string + description: Schedule project ID + spider_id: + type: string + description: Schedule spider ID + cron: + type: string + description: Cron expression + params: + type: string + description: Schedule params + mode: + $ref: '#/components/schemas/TaskMode' + PostTaskRunRequest: + description: Post task run request body + properties: + spider_id: + type: string + description: Task spider ID + cmd: + type: string + description: Task command + params: + type: string + description: Task params + mode: + $ref: '#/components/schemas/TaskMode' + priority: + type: integer + description: Task priority + PostUserRequest: + description: Post user request body + properties: + username: + type: string + description: User username + password: + type: string + description: User password + email: + type: string + description: User email + role: + type: string + enum: + - admin + - user + description: User role + PutUserRequest: + description: Put user request body + properties: + username: + type: string + description: User username + email: + type: string + description: User email + role: + type: string + enum: + - admin + - user + description: User role + PostUserChangePasswordRequest: + description: Post user change password request body + properties: + password: + type: string + description: New password + PostTokenRequest: + description: Post token request body + properties: + name: + type: string + description: Token name + PutTokenRequest: + description: Put token request body + properties: + name: + type: string + description: Token name + PostPluginRequest: + description: Post plugin request body + properties: + name: + type: string + description: Plugin name + full_name: + type: string + description: Plugin full name + + # base responses + BaseResponse: + type: object + properties: + status: + type: string + description: Status of the response + message: + type: string + description: Message of the response + EmptyResponse: + description: Empty response body + $ref: '#/components/schemas/BaseResponse' + example: + status: ok + message: success + Response: + description: Response body + allOf: + - $ref: '#/components/schemas/BaseResponse' + - type: object + properties: + data: + type: object + ListResponse: + description: List response body + allOf: + - $ref: '#/components/schemas/BaseResponse' + - type: object + properties: + data: + type: array + total: + type: integer + ErrorResponse: + description: Error response body + allOf: + - $ref: '#/components/schemas/BaseResponse' + - type: object + properties: + error: + type: string + + # error response + NotFoundErrorResponse: + description: Not found error response body + $ref: '#/components/schemas/ErrorResponse' + example: + status: error + message: not found + BadRequestErrorResponse: + description: Bad request error response + $ref: '#/components/schemas/ErrorResponse' + example: + status: error + message: bad request + UnauthorizedErrorResponse: + description: Unauthorized error response + $ref: '#/components/schemas/ErrorResponse' + example: + status: error + message: unauthorized + + # success response + PostLoginResponse: + type: object + properties: + data: + type: string + description: API token + GetVersionResponse: + type: object + properties: + data: + type: string + description: Version + GetNodesResponse: + type: object + allOf: + - $ref: '#/components/schemas/ListResponse' + - properties: + data: + type: array + items: + $ref: '#/components/schemas/Node' + GetNodeResponse: + type: object + properties: + data: + $ref: '#/components/schemas/Node' + GetProjectsResponse: + type: object + allOf: + - $ref: '#/components/schemas/ListResponse' + - properties: + data: + type: array + items: + $ref: '#/components/schemas/Project' + GetProjectResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + $ref: '#/components/schemas/Project' + GetSpidersResponse: + type: object + allOf: + - $ref: '#/components/schemas/ListResponse' + - properties: + data: + $ref: '#/components/schemas/Spider' + GetSpiderResponse: + type: object + properties: + data: + $ref: '#/components/schemas/Spider' + GetSpiderFilesListResponse: + type: object + allOf: + - $ref: '#/components/schemas/ListResponse' + - properties: + data: + type: array + items: + $ref: '#/components/schemas/SpiderFile' + example: + { + "status": "ok", + "message": "success", + "data": [ + { + "name": "scrapy_baidu", + "path": "/scrapy_baidu", + "full_path": "/fs/62c66a8ef9dce6fe4caa58e5/scrapy_baidu", + "extension": "", + "md5": "", + "is_dir": true, + "file_size": 0, + "children": [ + { + "name": "spiders", + "path": "/scrapy_baidu/spiders", + "full_path": "/fs/62c66a8ef9dce6fe4caa58e5/scrapy_baidu/spiders", + "extension": "", + "md5": "", + "is_dir": true, + "file_size": 0, + "children": [ + { + "name": "__init__.py", + "path": "/scrapy_baidu/spiders/__init__.py", + "full_path": "/fs/62c66a8ef9dce6fe4caa58e5/scrapy_baidu/spiders/__init__.py", + "extension": "py", + "md5": "EoqQSUWYa98Uwyno4FRZYw==", + "is_dir": false, + "file_size": 161, + "children": null + }, + { + "name": "baidu.py", + "path": "/scrapy_baidu/spiders/baidu.py", + "full_path": "/fs/62c66a8ef9dce6fe4caa58e5/scrapy_baidu/spiders/baidu.py", + "extension": "py", + "md5": "AhySHuGyj88CEA1ZbUJ19g==", + "is_dir": false, + "file_size": 791, + "children": null + } + ] + }, + { + "name": "__init__.py", + "path": "/scrapy_baidu/__init__.py", + "full_path": "/fs/62c66a8ef9dce6fe4caa58e5/scrapy_baidu/__init__.py", + "extension": "py", + "md5": "chXunH2dwinSkhpA6JnsXw==", + "is_dir": false, + "file_size": 1, + "children": null + }, + { + "name": "items.py", + "path": "/scrapy_baidu/items.py", + "full_path": "/fs/62c66a8ef9dce6fe4caa58e5/scrapy_baidu/items.py", + "extension": "py", + "md5": "Zg9KIbCe1h/coLSkQcQ9Sg==", + "is_dir": false, + "file_size": 311, + "children": null + }, + { + "name": "middlewares.py", + "path": "/scrapy_baidu/middlewares.py", + "full_path": "/fs/62c66a8ef9dce6fe4caa58e5/scrapy_baidu/middlewares.py", + "extension": "py", + "md5": "4FQr2+wt4ALcuRYxzjsUdQ==", + "is_dir": false, + "file_size": 3658, + "children": null + }, + { + "name": "pipelines.py", + "path": "/scrapy_baidu/pipelines.py", + "full_path": "/fs/62c66a8ef9dce6fe4caa58e5/scrapy_baidu/pipelines.py", + "extension": "py", + "md5": "q8MuuuJwvuV0gcE42Mg2oQ==", + "is_dir": false, + "file_size": 259, + "children": null + }, + { + "name": "settings.py", + "path": "/scrapy_baidu/settings.py", + "full_path": "/fs/62c66a8ef9dce6fe4caa58e5/scrapy_baidu/settings.py", + "extension": "py", + "md5": "hh/13eHBaVrLNtB27duOZQ==", + "is_dir": false, + "file_size": 3201, + "children": null + } + ] + }, + { + "name": "__init__.py", + "path": "/__init__.py", + "full_path": "/fs/62c66a8ef9dce6fe4caa58e5/__init__.py", + "extension": "py", + "md5": "chXunH2dwinSkhpA6JnsXw==", + "is_dir": false, + "file_size": 1, + "children": null + }, + { + "name": "crawlab.json", + "path": "/crawlab.json", + "full_path": "/fs/62c66a8ef9dce6fe4caa58e5/crawlab.json", + "extension": "json", + "md5": "jQ5qto4lYSEzWNF77h11gw==", + "is_dir": false, + "file_size": 169, + "children": null + }, + { + "name": "scrapy.cfg", + "path": "/scrapy.cfg", + "full_path": "/fs/62c66a8ef9dce6fe4caa58e5/scrapy.cfg", + "extension": "cfg", + "md5": "UAALAu5OkhbTVtwQeFXpQA==", + "is_dir": false, + "file_size": 267, + "children": null + } + ], + "error": "" + } + GetSpiderFileResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + type: string + description: Spider file content + example: + { + "status": "ok", + "message": "success", + "data": "# Automatically created by: scrapy startproject\n#\n# For more information about the [deploy] section see:\n# https://scrapyd.readthedocs.io/en/latest/deploy.html\n\n[settings]\ndefault = scrapy_baidu.settings\n\n[deploy]\n#url = http://localhost:6800/\nproject = scrapy_baidu\n", + "error": "" + } + GetSpiderFileInfoResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + $ref: '#/components/schemas/SpiderFile' + example: + { + "status": "ok", + "message": "success", + "data": { + "name": "openapi.yaml", + "path": "/openapi.yaml", + "full_path": "/fs/62c66a8af9dce6fe4caa58e3/openapi.yaml", + "extension": "yaml", + "md5": "1UUumcDmYtVAUOZZNegjJA==", + "is_dir": false, + "file_size": 27698, + "children": null + }, + "error": "" + } + GetSpiderGitResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + type: object + description: Spider git content + properties: + current_branch: + type: string + description: Current git branch + branches: + type: array + description: Git branches + items: + type: string + changes: + type: array + description: Git changes + items: + $ref: '#/components/schemas/GitFileStatus' + logs: + type: array + description: Git logs + items: + $ref: '#/components/schemas/GitLog' + ignore: + type: array + description: Git ignore files + items: + type: string + git: + type: object + description: Git url + $ref: '#/components/schemas/Git' + GetSpiderGitRemoteRefsResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + type: object + description: Spider git remote refs + properties: + refs: + type: array + description: Git remote refs + items: + $ref: '#/components/schemas/GitRef' + GetSpiderDataSourceResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + type: array + description: Spider data source + items: + $ref: '#/components/schemas/DataSource' + GetScheduleResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + type: array + description: Schedules + items: + $ref: '#/components/schemas/Schedule' + GetSchedulesResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + type: array + description: Schedules + items: + $ref: '#/components/schemas/Schedule' + GetTasksResponse: + type: object + allOf: + - $ref: '#/components/schemas/ListResponse' + - properties: + data: + type: array + description: Tasks + items: + $ref: '#/components/schemas/Task' + GetTaskResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + $ref: '#/components/schemas/Task' + GetTaskLogsResponse: + type: object + allOf: + - $ref: '#/components/schemas/ListResponse' + - properties: + data: + type: array + description: Task logs + items: + type: string + GetTaskDataResponse: + type: object + allOf: + - $ref: '#/components/schemas/ListResponse' + - properties: + data: + type: array + items: + type: object + GetUsersResponse: + type: object + allOf: + - $ref: '#/components/schemas/ListResponse' + - properties: + data: + type: array + items: + $ref: '#/components/schemas/User' + example: + { + "status": "ok", + "message": "success", + "total": 1, + "data": [ + { + "_id": "62c665fb6e54352bf1b279e2", + "username": "admin", + "role": "admin", + "email": "" + } + ], + "error": "" + } + GetUserResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + $ref: '#/components/schemas/User' + example: + { + "status": "ok", + "message": "success", + "data": { + "_id": "62c665fb6e54352bf1b279e2", + "username": "admin", + "role": "admin", + "email": "" + }, + "error": "" + } + GetTokensResponse: + type: object + allOf: + - $ref: '#/components/schemas/ListResponse' + - properties: + data: + type: array + items: + $ref: '#/components/schemas/Token' + example: + { + "status": "ok", + "message": "success", + "total": 1, + "data": [ + { + "_id": "62c8e9e7f23518fd080f04aa", + "name": "my token", + "token": "xxxxxxxxxx" + } + ], + "error": "" + } + GetTokenResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + $ref: '#/components/schemas/Token' + example: + { + "status": "ok", + "message": "success", + "data": { + "_id": "62c8e9e7f23518fd080f04aa", + "name": "my token", + "token": "xxxxxxxxxx" + }, + "error": "" + } + GetPluginsResponse: + type: object + allOf: + - $ref: '#/components/schemas/ListResponse' + - properties: + data: + type: array + items: + $ref: '#/components/schemas/Plugin' + example: + { + "status": "ok", + "message": "success", + "total": 3, + "data": [ + { + "_id": "62a88b02112099e6932808a8", + "name": "spider-assistant", + "short_name": "plugin-spider-assistant", + "full_name": "crawlab-team/plugin-spider-assistant", + "description": "Spider assistant plugin for Crawlab", + "proto": "http", + "active": false, + "endpoint": "localhost:9997", + "cmd": "sh ./bin/start.sh", + "docker_cmd": "/app/plugins/bin/plugin-spider-assistant", + "docker_dir": "/app/plugins/plugin-spider-assistant", + "event_key": { + "include": "^model:", + "exclude": "artifact" + }, + "install_type": "public", + "install_url": "/app/plugins/plugin-spider-assistant", + "install_cmd": "", + "deploy_mode": "master", + "auto_start": true, + "ui_components": [ + { + "name": "assistant", + "title": "assistant.detail.tabs.title", + "src": "ui/src/AssistantDetail.vue", + "type": "tab", + "path": "assistant", + "parent_paths": [ + "/spiders/:id" + ] + } + ], + "ui_sidebar_navs": [ ], + "ui_assets": [ ], + "lang_url": "ui/lang", + "status": null + }, + { + "_id": "62a88b04112099e6932808aa", + "name": "notification", + "short_name": "plugin-notification", + "full_name": "crawlab-team/plugin-notification", + "description": "A plugin for handling notifications", + "proto": "http", + "active": false, + "endpoint": "localhost:39999", + "cmd": "sh ./bin/start.sh", + "docker_cmd": "/app/plugins/bin/plugin-notification", + "docker_dir": "/app/plugins/plugin-notification", + "event_key": { + "include": "^model:", + "exclude": "artifact" + }, + "install_type": "public", + "install_url": "/app/plugins/plugin-notification", + "install_cmd": "", + "deploy_mode": "master_only", + "auto_start": true, + "ui_components": [ + { + "name": "notification-list", + "title": "Notifications", + "src": "ui/src/NotificationList.vue", + "type": "view", + "path": "notifications", + "parent_paths": null + }, + { + "name": "notification-detail", + "title": "Notifications", + "src": "ui/src/NotificationDetail.vue", + "type": "view", + "path": "notifications/:id", + "parent_paths": null + } + ], + "ui_sidebar_navs": [ + { + "path": "/notifications", + "title": "plugins.notification.ui_sidebar_navs.title.notifications", + "icon": [ + "fa", + "envelope" + ] + } + ], + "ui_assets": [ + { + "path": "ui/public/simplemde/simplemde.js", + "type": "js" + }, + { + "path": "ui/public/simplemde/simplemde.css", + "type": "css" + }, + { + "path": "ui/public/css/style.css", + "type": "css" + } + ], + "lang_url": "ui/lang", + "status": null + }, + { + "_id": "62c390ab5853635d26cf76c7", + "name": "dependency", + "short_name": "plugin-dependency", + "full_name": "crawlab-team/plugin-dependency", + "description": "A plugin for managing dependencies", + "proto": "http", + "active": false, + "endpoint": "localhost:9998", + "cmd": "sh ./bin/start.sh", + "docker_cmd": "/app/plugins/bin/plugin-dependency", + "docker_dir": "/app/plugins/plugin-dependency", + "event_key": { + "include": "^model:", + "exclude": "artifact" + }, + "install_type": "public", + "install_url": "/app/plugins/plugin-dependency", + "install_cmd": "", + "deploy_mode": "all", + "auto_start": true, + "ui_components": [ + { + "name": "dependency-settings", + "title": "Dependencies Settings", + "src": "ui/src/setting/DependencySettings.vue", + "type": "view", + "path": "dependencies/settings", + "parent_paths": null + }, + { + "name": "dependency-python", + "title": "Dependencies Python", + "src": "ui/src/python/DependencyPython.vue", + "type": "view", + "path": "dependencies/python", + "parent_paths": null + }, + { + "name": "dependency-node", + "title": "Dependencies Node", + "src": "ui/src/node/DependencyNode.vue", + "type": "view", + "path": "dependencies/node", + "parent_paths": null + }, + { + "name": "dependencies", + "title": "ui_components.title.dependencies", + "src": "ui/src/spider/DependencySpiderTab.vue", + "type": "tab", + "path": "dependencies", + "parent_paths": [ + "/spiders/:id" + ] + } + ], + "ui_sidebar_navs": [ + { + "path": "/dependencies", + "title": "plugins.dependency.ui_sidebar_navs.title.dependencies", + "icon": [ + "fa", + "puzzle-piece" + ], + "children": [ + { + "path": "/dependencies/python", + "title": "plugins.dependency.ui_sidebar_navs.title.python", + "icon": [ + "fab", + "python" + ] + }, + { + "path": "/dependencies/node", + "title": "plugins.dependency.ui_sidebar_navs.title.node", + "icon": [ + "fab", + "node-js" + ] + }, + { + "path": "/dependencies/settings", + "title": "plugins.dependency.ui_sidebar_navs.title.settings", + "icon": [ + "fa", + "cog" + ] + } + ] + } + ], + "ui_assets": [ ], + "lang_url": "ui/lang", + "status": null + } + ], + "error": "" + } + GetPluginResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + $ref: '#/components/schemas/Plugin' + GetPublicPluginsResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + type: array + items: + $ref: '#/components/schemas/PublicPlugin' + GetPublicPluginsInfoResponse: + type: object + allOf: + - $ref: '#/components/schemas/Response' + - properties: + data: + type: array + items: + type: object + + # models + Model: + description: Base model + type: object + properties: + _id: + type: string + description: ID of the model (MongoDB ObjectID) + ModelWithNameDescription: + allOf: + - $ref: '#/components/schemas/Model' + - properties: + name: + type: string + description: Name of the model + description: + type: string + description: Description of the model + ModelWithKey: + allOf: + - $ref: '#/components/schemas/Model' + - properties: + key: + type: string + description: Key of the model + Node: + description: Node model + type: object + allOf: + - $ref: '#/components/schemas/ModelWithKey' + - $ref: '#/components/schemas/ModelWithNameDescription' + - properties: + ip: + description: Node IP + type: string + port: + description: Node port + type: string + mac: + description: Node MAC + type: string + hostname: + description: Node hostname + type: string + is_master: + description: Whether the node is master + type: boolean + status: + description: Node current status + type: string + enum: + - online + - offline + enabled: + description: Node enabled + type: boolean + active: + description: Node active + type: boolean + active_ts: + description: Node active timestamp + type: string + format: date-time + available_runners: + description: Available number of available runners of the node + type: integer + max_runners: + description: Max number of runners of the node + type: integer + Project: + description: Project model + type: object + allOf: + - $ref: '#/components/schemas/ModelWithNameDescription' + - properties: + spiders: + description: Spiders of the project + type: integer + TaskStat: + type: object + allOf: + - $ref: '#/components/schemas/Model' + - properties: + create_ts: + type: string + format: date-time + start_ts: + type: string + format: date-time + end_ts: + type: string + format: date-time + wait_duration: + type: integer + runtime_duration: + type: integer + total_duration: + type: integer + result_count: + type: integer + error_log_count: + type: integer + Task: + type: object + allOf: + - $ref: '#/components/schemas/Model' + - properties: + spider_id: + type: string + status: + type: string + node_id: + type: string + cmd: + type: string + param: + type: string + error: + type: string + pid: + description: Process ID + type: integer + schedule_id: + type: string + type: + type: string + mode: + $ref: '#/components/schemas/TaskMode' + node_ids: + type: array + items: + type: string + parent_id: + type: string + priority: + type: integer + stat: + $ref: '#/components/schemas/TaskStat' + has_sub: + type: boolean + sub_tasks: + type: array + items: + $ref: '#/components/schemas/Task' + user_id: + type: string + TaskMode: + type: string + enum: + - random + - all-nodes + - selected-nodes + SpiderStat: + description: Spider stat + type: object + allOf: + - $ref: '#/components/schemas/Model' + - properties: + type: + description: Last task ID + type: string + task: + description: Last task + $ref: '#/components/schemas/Task' + Spider: + description: Spider model + type: object + allOf: + - $ref: '#/components/schemas/ModelWithNameDescription' + - properties: + type: + description: Spider type + type: string + col_id: + description: Spider collection ID + type: string + col_name: + description: Spider collection name + type: string + data_source_id: + description: Spider data source id + type: string + data_source_name: + description: Spider data source name + type: string + project_id: + description: Spider project id + type: string + mode: + description: Spider task mode + $ref: '#/components/schemas/TaskMode' + node_ids: + description: Spider task selected node IDs + type: array + items: + type: string + stat: + description: Spider stat + $ref: '#/components/schemas/SpiderStat' + SpiderFile: + description: Spider file model + type: object + properties: + name: + description: File name + type: string + path: + description: File path + type: string + full_path: + description: Full file path + type: string + extension: + description: File extension + type: string + md5: + description: File MD5 hash + type: string + is_dir: + description: Whether the file is directory + type: string + fileSize: + description: File size + type: integer + children: + description: Children files + type: array + items: + $ref: '#/components/schemas/SpiderFile' + GitFileStatus: + description: Git file status + type: object + properties: + path: + description: File path + type: string + name: + description: File name + type: string + is_dir: + description: Whether the file is directory + type: boolean + staging: + description: Staging + type: string + worktree: + description: Worktree + type: string + extra: + description: Extra + type: string + children: + description: Children files + type: array + items: + $ref: '#/components/schemas/GitFileStatus' + GitRef: + description: Git ref + type: object + properties: + type: + description: Ref type + type: string + name: + description: Ref name + type: string + full_name: + description: Ref full name + type: string + hash: + description: Ref hash + type: string + timestamp: + description: Ref timestamp + type: string + format: date-time + GitLog: + description: Git log + type: object + properties: + hash: + description: Commit hash + type: string + msg: + description: Commit message + type: string + author_name: + description: Author name + type: string + author_email: + description: Author email + type: string + timestamp: + description: Commit timestamp + type: string + format: date-time + refs: + description: Commit refs + type: array + items: + $ref: '#/components/schemas/GitRef' + Git: + description: Git model + type: object + allOf: + - $ref: '#/components/schemas/ModelWithKey' + - properties: + url: + description: Git url + type: string + branch: + description: Git branch + type: string + changes: + description: Git changes + type: array + items: + $ref: '#/components/schemas/GitFileStatus' + logs: + description: Git logs + type: array + items: + $ref: '#/components/schemas/GitLog' + ignore: + description: Git ignore files + type: array + items: + type: string + git: + description: Git url + type: string + GitPayload: + description: Git payload + type: object + properties: + paths: + description: Git paths + type: array + items: + type: string + commit_message: + description: Commit message + type: string + branch: + description: Branch + type: string + tag: + description: Tag + type: string + DataSource: + description: Data source model + type: object + allOf: + - $ref: '#/components/schemas/ModelWithKey' + - properties: + name: + description: Data source name + type: string + description: + description: Data source description + type: string + type: + description: Data source type + type: string + host: + description: Data source host + type: string + port: + description: Data source port + type: string + url: + description: Data source url + type: string + hosts: + description: Data source hosts + type: array + items: + type: string + database: + description: Data source database + type: string + username: + description: Data source username + type: string + password: + description: Data source password + type: string + enabled: + description: Data source enabled + type: boolean + connect_type: + description: Data source connect type + type: string + status: + description: Data source status + type: string + error: + description: Data source error + type: string + extra: + description: Data source extra + type: object + Schedule: + description: Schedule model + type: object + allOf: + - $ref: '#/components/schemas/ModelWithKey' + - properties: + spider_id: + description: Schedule spider ID + type: string + cron: + description: Schedule cron + type: string + example: "* * * * *" + entry_id: + description: Schedule entry ID + type: string + example: 1 + cmd: + description: Schedule execute command + type: string + example: scrapy crawl spider + param: + description: Schedule execute command param + type: string + example: -a param1=value1 -a param2=value2 + mode: + description: Schedule task mode + type: string + example: random + node_ids: + enabled: + description: Schedule enabled + type: boolean + example: true + user_id: + description: Schedule user ID + type: string + User: + description: User model + type: object + allOf: + - $ref: '#/components/schemas/Model' + - properties: + username: + description: User username + type: string + email: + description: User email + type: string + role: + description: User role + type: string + Token: + description: Token model + type: object + allOf: + - $ref: '#/components/schemas/Model' + - properties: + name: + description: Token name + type: string + token: + description: Token + type: string + Plugin: + description: Plugin model + type: object + allOf: + - $ref: '#/components/schemas/ModelWithNameDescription' + - properties: + short_name: + description: Plugin short name + type: string + full_name: + description: Plugin full name + type: string + proto: + description: Plugin protocol + type: string + active: + description: Plugin active + type: boolean + endpoint: + description: Plugin endpoint + type: string + cmd: + description: Plugin cmd + type: string + docker_cmd: + description: Plugin docker cmd + type: string + docker_dir: + description: Plugin docker directory + type: string + event_key: + description: Plugin event key + $ref: '#/components/schemas/PluginEventKey' + install_type: + description: Plugin install type + type: string + install_url: + description: Plugin install url + type: string + install_cmd: + description: Plugin install cmd + type: string + deploy_mode: + description: Plugin deploy mode + type: string + auto_start: + description: Plugin auto start + type: boolean + ui_components: + description: Plugin ui components + type: array + items: + $ref: '#/components/schemas/PluginUIComponent' + ui_sidebar_navs: + description: Plugin ui sidebar navs + type: array + items: + $ref: '#/components/schemas/PluginUINav' + ui_assets: + description: Plugin ui assets + type: array + items: + $ref: '#/components/schemas/PluginUIAsset' + lang_url: + description: Plugin language url + type: string + status: + description: Plugin status + $ref: '#/components/schemas/PluginStatus' + PluginEventKey: + description: Plugin event key model + type: object + properties: + include: + description: Plugin included event key + type: array + items: + type: string + exclude: + description: Plugin excluded event key + type: array + items: + type: string + PluginUIComponent: + description: Plugin ui component model + type: object + properties: + name: + description: Plugin ui component name + type: string + title: + description: Plugin ui component title + type: string + src: + description: Plugin ui component src + type: string + type: + description: Plugin ui component type + type: string + path: + description: Plugin ui component path + type: string + parent_paths: + description: Plugin ui component parent paths + type: array + items: + type: string + PluginUINav: + description: Plugin ui nav model + type: object + properties: + path: + description: Plugin ui nav path + type: string + title: + description: Plugin ui nav title + type: string + icon: + description: Plugin ui nav icon + type: array + items: + type: string + children: + description: Plugin ui nav children + type: array + items: + $ref: '#/components/schemas/PluginUINav' + PluginUIAsset: + description: Plugin ui asset model + type: object + properties: + path: + description: Plugin ui asset path + type: string + type: + description: Plugin ui asset type + type: string + PluginStatus: + description: Plugin status model + type: object + allOf: + - $ref: '#/components/schemas/Model' + - properties: + plugin_id: + description: Plugin ID + type: string + node_id: + description: Node ID + type: string + status: + description: Plugin status + type: string + pid: + description: Plugin process ID + type: integer + error: + description: Plugin error + type: string + node: + description: Node model + $ref: '#/components/schemas/Node' + PublicPlugin: + description: Public plugin model + type: object + properties: + id: + description: Public plugin ID + type: string + name: + description: Public plugin name + type: string + full_name: + description: Public plugin full name + type: string + description: + description: Public plugin description + type: string + html_url: + description: Public plugin html url + type: string + pushed_at: + description: Public plugin pushed at + type: string + created_at: + description: Public plugin created at + type: string + updated_at: + description: Public plugin updated at + type: string + + securitySchemes: + apiToken: + type: apiKey + in: header + name: Authorization + description: API Token + x-displayName: API Token diff --git a/core/docs/package.json b/core/docs/package.json new file mode 100644 index 000000000..391315016 --- /dev/null +++ b/core/docs/package.json @@ -0,0 +1,16 @@ +{ + "name": "docs", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "publish": "node scripts/publish.js" + }, + "author": "", + "license": "ISC", + "devDependencies": { + "chalk": "^4.1.2", + "qiniu": "^7.4.0", + "walk-sync": "^3.0.0" + } +} diff --git a/core/docs/scripts/publish.js b/core/docs/scripts/publish.js new file mode 100644 index 000000000..decf65e77 --- /dev/null +++ b/core/docs/scripts/publish.js @@ -0,0 +1,81 @@ +const path = require('path') +const qiniu = require('qiniu') +const walkSync = require('walk-sync') +const chalk = require('chalk') + +// target directory +const targetDir = './api' + +// access key +const accessKey = process.env.QINIU_ACCESS_KEY + +// secret key +const secretKey = process.env.QINIU_SECRET_KEY + +// bucket +const bucket = process.env.QINIU_BUCKET + +// config +const config = new qiniu.conf.Config() + +// zone +config.zone = qiniu.zone[process.env.QINIU_ZONE] + +function uploadFile(localFile, key) { + // options + const options = { + scope: `${bucket}:${key}`, + } + + // mac + const mac = new qiniu.auth.digest.Mac(accessKey, secretKey) + + // put policy + const putPolicy = new qiniu.rs.PutPolicy(options) + + // upload token + const uploadToken = putPolicy.uploadToken(mac) + + return new Promise((resolve, reject) => { + const formUploader = new qiniu.form_up.FormUploader(config) + const putExtra = new qiniu.form_up.PutExtra() + formUploader.putFile(uploadToken, key, localFile, putExtra, function (respErr, respBody, respInfo) { + if (respErr) { + throw respErr + } + if (respInfo.statusCode === 200) { + console.log(`${chalk.green('uploaded')} ${localFile} => ${key}`) + resolve() + } else if (respInfo.statusCode === 614) { + console.log(`${chalk.yellow('exists')} ${localFile} => ${key}`) + resolve() + } else { + const errMsg = `${chalk.red('error[' + respInfo.statusCode + ']')} ${localFile} => ${key}` + console.error(errMsg) + reject(new Error(respBody)) + } + }) + }) +} + +async function main() { + // paths + const paths = walkSync(targetDir, { + includeBasePath: true, directories: false, + }) + + // iterate paths + for (const filePath of paths) { + const localFile = path.resolve(filePath) + const key = filePath.replace(targetDir + '/', '') + try { + await uploadFile(localFile, key) + } finally { + // do nothing + } + } +} + +(async () => { + await main() +})() diff --git a/core/ds/cockroachdb.go b/core/ds/cockroachdb.go new file mode 100644 index 000000000..c7777eea9 --- /dev/null +++ b/core/ds/cockroachdb.go @@ -0,0 +1,70 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type CockroachdbService struct { + SqlService +} + +func NewDataSourceCockroachdbService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &CockroachdbService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, trace.TraceError(err) + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, trace.TraceError(err) + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultCockroachdbPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, trace.TraceError(err) + } + + // session + svc.s, err = utils.GetCockroachdbSession(svc.ds) + if err != nil { + return nil, trace.TraceError(err) + } + + // collection + svc.col = svc.s.Collection(svc.dc.Name) + + return svc, nil +} diff --git a/core/ds/default.go b/core/ds/default.go new file mode 100644 index 000000000..e5143a2c9 --- /dev/null +++ b/core/ds/default.go @@ -0,0 +1 @@ +package ds diff --git a/core/ds/es.go b/core/ds/es.go new file mode 100644 index 000000000..bd7fddaad --- /dev/null +++ b/core/ds/es.go @@ -0,0 +1,220 @@ +package ds + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab/core/constants" + constants2 "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + entity2 "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" + "github.com/google/uuid" + "go.mongodb.org/mongo-driver/bson/primitive" + "strings" + "sync" + "time" +) + +type ElasticsearchService struct { + // dependencies + modelSvc service.ModelService + + // internals + dc *models.DataCollection // models.DataCollection + ds *models.DataSource // models.DataSource + c *elasticsearch.Client // elasticsearch.Client + t time.Time +} + +func (svc *ElasticsearchService) Insert(records ...interface{}) (err error) { + // wait group + var wg sync.WaitGroup + wg.Add(len(records)) + + // iterate records + for _, r := range records { + // async operation + go func(r interface{}) { + switch r.(type) { + case entity.Result: + // convert type to entity.Result + d := r.(entity.Result) + + // get document id + id := d.GetValue("id") + var docId string + switch id.(type) { + case string: + docId = id.(string) + } + if docId == "" { + docId = uuid.New().String() // generate new uuid if id is empty + } + + // collection + d[constants2.DataCollectionKey] = svc.dc.Name + + // index request + req := esapi.IndexRequest{ + Index: svc.getIndexName(), + DocumentID: docId, + Body: strings.NewReader(d.String()), + } + + // perform request + res, err := req.Do(context.Background(), svc.c) + if err != nil { + trace.PrintError(err) + wg.Done() + return + } + defer res.Body.Close() + if res.IsError() { + trace.PrintError(errors.New(fmt.Sprintf("[ElasticsearchService] [%s] error inserting record: %v", res.Status(), r))) + } + + // release + wg.Done() + default: + wg.Done() + return + } + }(r) + } + + // wait + wg.Wait() + + return nil +} + +func (svc *ElasticsearchService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { + data, err := svc.getListResponse(query, opts, false) + if err != nil { + return nil, err + } + for _, hit := range data.Hits.Hits { + results = append(results, hit.Source) + } + return results, nil +} + +func (svc *ElasticsearchService) Count(query generic.ListQuery) (n int, err error) { + data, err := svc.getListResponse(query, nil, true) + if err != nil { + return n, err + } + return int(data.Hits.Total.Value), nil +} + +func (svc *ElasticsearchService) getListResponse(query generic.ListQuery, opts *generic.ListOptions, trackTotalHits bool) (data *entity2.ElasticsearchResponseData, err error) { + if opts == nil { + opts = &generic.ListOptions{} + } + query = append(query, generic.ListQueryCondition{ + Key: constants2.DataCollectionKey, + Op: constants2.FilterOpEqual, + Value: svc.dc.Name, + }) + res, err := svc.c.Search( + svc.c.Search.WithContext(context.Background()), + svc.c.Search.WithIndex(svc.getIndexName()), + svc.c.Search.WithBody(utils.GetElasticsearchQueryWithOptions(query, opts)), + svc.c.Search.WithTrackTotalHits(trackTotalHits), + ) + if err != nil { + return nil, trace.TraceError(err) + } + defer res.Body.Close() + if res.IsError() { + err = errors.New(fmt.Sprintf("[ElasticsearchService] [%s] error listing records: response=%s, query=%v opts=%v", res.Status(), res.String(), query, opts)) + trace.PrintError(err) + return nil, err + } + data = &entity2.ElasticsearchResponseData{} + if err := json.NewDecoder(res.Body).Decode(data); err != nil { + return nil, trace.TraceError(err) + } + return data, nil +} + +func (svc *ElasticsearchService) getIndexName() (index string) { + if svc.ds.Database == "" { + return svc.dc.Name + } else { + return svc.ds.Name + } +} + +func NewDataSourceElasticsearchService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &ElasticsearchService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, err + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, err + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultElasticsearchPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, err + } + + // client + svc.c, err = utils.GetElasticsearchClient(svc.ds) + if err != nil { + return nil, err + } + + return svc, nil +} + +func (svc *ElasticsearchService) Index(fields []string) { + // TODO: implement me +} + +func (svc *ElasticsearchService) SetTime(t time.Time) { + svc.t = t +} + +func (svc *ElasticsearchService) GetTime() (t time.Time) { + return svc.t +} diff --git a/core/ds/kafka.go b/core/ds/kafka.go new file mode 100644 index 000000000..44a42209c --- /dev/null +++ b/core/ds/kafka.go @@ -0,0 +1,116 @@ +package ds + +import ( + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/segmentio/kafka-go" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type KafkaService struct { + // dependencies + modelSvc service.ModelService + + // internals + dc *models.DataCollection // models.DataCollection + ds *models.DataSource // models.DataSource + c *kafka.Conn // kafka.Conn + rb backoff.BackOff + t time.Time +} + +func (svc *KafkaService) Insert(records ...interface{}) (err error) { + var messages []kafka.Message + for _, r := range records { + switch r.(type) { + case entity.Result: + d := r.(entity.Result) + messages = append(messages, kafka.Message{ + Topic: svc.ds.Database, + Key: []byte(d.GetTaskId().Hex()), + Value: d.Bytes(), + }) + } + } + _, err = svc.c.WriteMessages(messages...) + if err != nil { + return trace.TraceError(err) + } + return nil +} + +func (svc *KafkaService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { + // N/A + return nil, nil +} + +func (svc *KafkaService) Count(query generic.ListQuery) (n int, err error) { + // N/A + return 0, nil +} + +func NewDataSourceKafkaService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &KafkaService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, err + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, err + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultKafkaPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, err + } + + return svc, nil +} + +func (svc *KafkaService) Index(fields []string) { + // TODO: implement me +} + +func (svc *KafkaService) SetTime(t time.Time) { + svc.t = t +} + +func (svc *KafkaService) GetTime() (t time.Time) { + return svc.t +} diff --git a/core/ds/mongo.go b/core/ds/mongo.go new file mode 100644 index 000000000..31bc8372c --- /dev/null +++ b/core/ds/mongo.go @@ -0,0 +1,118 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + utils2 "github.com/crawlab-team/crawlab/core/utils" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "time" +) + +type MongoService struct { + // dependencies + modelSvc service.ModelService + + // internals + dc *models.DataCollection // models.DataCollection + ds *models.DataSource // models.DataSource + c *mongo2.Client + db *mongo2.Database + col *mongo.Col + t time.Time +} + +func (svc *MongoService) Insert(records ...interface{}) (err error) { + _, err = svc.col.InsertMany(records) + return err +} + +func (svc *MongoService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { + var docs []models.Result + if err := svc.col.Find(utils.GetMongoQuery(query), utils.GetMongoOpts(opts)).All(&docs); err != nil { + return nil, err + } + for i := range docs { + results = append(results, &docs[i]) + } + return results, nil +} + +func (svc *MongoService) Count(query generic.ListQuery) (n int, err error) { + return svc.col.Count(utils.GetMongoQuery(query)) +} + +func NewDataSourceMongoService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &MongoService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, err + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, err + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultMongoPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, err + } + + // mongo client + svc.c, err = utils2.GetMongoClient(svc.ds) + if err != nil { + return nil, err + } + + // mongo database + svc.db = mongo.GetMongoDb(svc.ds.Database, mongo.WithDbClient(svc.c)) + + // mongo col + svc.col = mongo.GetMongoColWithDb(svc.dc.Name, svc.db) + + return svc, nil +} + +func (svc *MongoService) Index(fields []string) { + // TODO: implement me +} + +func (svc *MongoService) SetTime(t time.Time) { + svc.t = t +} + +func (svc *MongoService) GetTime() (t time.Time) { + return svc.t +} diff --git a/core/ds/mssql.go b/core/ds/mssql.go new file mode 100644 index 000000000..4adc73896 --- /dev/null +++ b/core/ds/mssql.go @@ -0,0 +1,71 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + utils2 "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type MssqlService struct { + SqlService +} + +func NewDataSourceMssqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &MssqlService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, trace.TraceError(err) + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, trace.TraceError(err) + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultMssqlPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, trace.TraceError(err) + } + + // session + svc.s, err = utils2.GetMssqlSession(svc.ds) + if err != nil { + return nil, trace.TraceError(err) + } + + // collection + svc.col = svc.s.Collection(svc.dc.Name) + + return svc, nil +} diff --git a/core/ds/mssql_test.go b/core/ds/mssql_test.go new file mode 100644 index 000000000..229197c74 --- /dev/null +++ b/core/ds/mssql_test.go @@ -0,0 +1,8 @@ +package ds + +import "testing" + +func TestNewDataSourceMssqlService(t *testing.T) { + t.Run("insert", func(t *testing.T) { + }) +} diff --git a/core/ds/mysql.go b/core/ds/mysql.go new file mode 100644 index 000000000..b9f840181 --- /dev/null +++ b/core/ds/mysql.go @@ -0,0 +1,69 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + utils2 "github.com/crawlab-team/crawlab/core/utils" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type MysqlService struct { + SqlService +} + +func NewDataSourceMysqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &MysqlService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, err + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, err + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultMysqlPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, err + } + + // session + svc.s, err = utils2.GetMysqlSession(svc.ds) + if err != nil { + return nil, err + } + + // collection + svc.col = svc.s.Collection(svc.dc.Name) + + return svc, nil +} diff --git a/core/ds/options.go b/core/ds/options.go new file mode 100644 index 000000000..58cc970e8 --- /dev/null +++ b/core/ds/options.go @@ -0,0 +1,14 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type DataSourceServiceOption func(svc interfaces.DataSourceService) + +func WithMonitorInterval(duration time.Duration) DataSourceServiceOption { + return func(svc interfaces.DataSourceService) { + svc.SetMonitorInterval(duration) + } +} diff --git a/core/ds/postgresql.go b/core/ds/postgresql.go new file mode 100644 index 000000000..06aac13f4 --- /dev/null +++ b/core/ds/postgresql.go @@ -0,0 +1,71 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + utils2 "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type PostgresqlService struct { + SqlService +} + +func NewDataSourcePostgresqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &PostgresqlService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, trace.TraceError(err) + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, trace.TraceError(err) + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultPostgresqlPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, trace.TraceError(err) + } + + // session + svc.s, err = utils2.GetPostgresqlSession(svc.ds) + if err != nil { + return nil, trace.TraceError(err) + } + + // collection + svc.col = svc.s.Collection(svc.dc.Name) + + return svc, nil +} diff --git a/core/ds/service.go b/core/ds/service.go new file mode 100644 index 000000000..cf57a92bc --- /dev/null +++ b/core/ds/service.go @@ -0,0 +1,305 @@ +package ds + +import ( + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/constants" + constants2 "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/result" + "github.com/crawlab-team/crawlab/core/utils" + utils2 "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "sync" + "time" +) + +type Service struct { + // dependencies + modelSvc service.ModelService + + // internals + timeout time.Duration + monitorInterval time.Duration + stopped bool +} + +func (svc *Service) Init() { + // result service registry + reg := result.GetResultServiceRegistry() + + // register result services + reg.Register(constants.DataSourceTypeMongo, NewDataSourceMongoService) + reg.Register(constants.DataSourceTypeMysql, NewDataSourceMysqlService) + reg.Register(constants.DataSourceTypePostgresql, NewDataSourcePostgresqlService) + reg.Register(constants.DataSourceTypeMssql, NewDataSourceMssqlService) + reg.Register(constants.DataSourceTypeSqlite, NewDataSourceSqliteService) + reg.Register(constants.DataSourceTypeCockroachdb, NewDataSourceCockroachdbService) + reg.Register(constants.DataSourceTypeElasticSearch, NewDataSourceElasticsearchService) + reg.Register(constants.DataSourceTypeKafka, NewDataSourceKafkaService) +} + +func (svc *Service) Start() { + // start monitoring + go svc.Monitor() +} + +func (svc *Service) Wait() { + utils.DefaultWait() +} + +func (svc *Service) Stop() { + svc.stopped = true +} + +func (svc *Service) ChangePassword(id primitive.ObjectID, password string) (err error) { + p, err := svc.modelSvc.GetPasswordById(id) + if err == nil { + // exists, save + encryptedPassword, err := utils.EncryptAES(password) + if err != nil { + return err + } + p.Password = encryptedPassword + if err := delegate.NewModelDelegate(p).Save(); err != nil { + return err + } + return nil + } else if err.Error() == mongo.ErrNoDocuments.Error() { + // not exists, add + encryptedPassword, err := utils.EncryptAES(password) + if err != nil { + return err + } + p = &models.Password{ + Id: id, + Password: encryptedPassword, + } + if err := delegate.NewModelDelegate(p).Add(); err != nil { + return err + } + return nil + } else { + // error + return err + } +} + +func (svc *Service) Monitor() { + for { + // return if stopped + if svc.stopped { + return + } + + // monitor + if err := svc.monitor(); err != nil { + trace.PrintError(err) + } + + // wait + time.Sleep(svc.monitorInterval) + } +} + +func (svc *Service) CheckStatus(id primitive.ObjectID) (err error) { + ds, err := svc.modelSvc.GetDataSourceById(id) + if err != nil { + return err + } + return svc.checkStatus(ds, true) +} + +func (svc *Service) SetTimeout(duration time.Duration) { + svc.timeout = duration +} + +func (svc *Service) SetMonitorInterval(duration time.Duration) { + svc.monitorInterval = duration +} + +func (svc *Service) monitor() (err error) { + // start + tic := time.Now() + log.Debugf("[DataSourceService] start monitoring") + + // data source list + dsList, err := svc.modelSvc.GetDataSourceList(nil, nil) + if err != nil { + return err + } + + // waiting group + wg := sync.WaitGroup{} + wg.Add(len(dsList)) + + // iterate data source list + for _, ds := range dsList { + // async operation + go func(ds models.DataSource) { + // check status and save + _ = svc.checkStatus(&ds, true) + + // release + wg.Done() + }(ds) + } + + // wait + wg.Wait() + + // finish + toc := time.Now() + log.Debugf("[DataSourceService] finished monitoring. elapsed: %d ms", (toc.Sub(tic)).Milliseconds()) + + return nil +} + +func (svc *Service) checkStatus(ds *models.DataSource, save bool) (err error) { + // password + if ds.Password == "" { + pwd, err := svc.modelSvc.GetPasswordById(ds.Id) + if err == nil { + ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return err + } + } else if err.Error() != mongo.ErrNoDocuments.Error() { + return trace.TraceError(err) + } + } + + // check status + if err := svc._checkStatus(ds); err != nil { + ds.Status = constants2.DataSourceStatusOffline + ds.Error = err.Error() + } else { + ds.Status = constants2.DataSourceStatusOnline + ds.Error = "" + } + + // save + if save { + return svc._save(ds) + } + + return nil +} + +func (svc *Service) _save(ds *models.DataSource) (err error) { + log.Debugf("[DataSourceService] saving data source: name=%s, type=%s, status=%s, error=%s", ds.Name, ds.Type, ds.Status, ds.Error) + return delegate.NewModelDelegate(ds).Save() +} + +func (svc *Service) _checkStatus(ds *models.DataSource) (err error) { + switch ds.Type { + case constants.DataSourceTypeMongo: + _, err := utils2.GetMongoClientWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + case constants.DataSourceTypeMysql: + s, err := utils2.GetMysqlSessionWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + s.Close() + } + case constants.DataSourceTypePostgresql: + s, err := utils2.GetPostgresqlSessionWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + s.Close() + } + case constants.DataSourceTypeMssql: + s, err := utils2.GetMssqlSessionWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + s.Close() + } + case constants.DataSourceTypeSqlite: + s, err := utils2.GetSqliteSessionWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + s.Close() + } + case constants.DataSourceTypeCockroachdb: + s, err := utils2.GetCockroachdbSessionWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + s.Close() + } + case constants.DataSourceTypeElasticSearch: + _, err := utils2.GetElasticsearchClientWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + case constants.DataSourceTypeKafka: + c, err := utils2.GetKafkaConnectionWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + if c != nil { + c.Close() + } + default: + log.Warnf("[DataSourceService] invalid data source type: %s", ds.Type) + } + return nil +} + +func NewDataSourceService(opts ...DataSourceServiceOption) (svc2 interfaces.DataSourceService, err error) { + // service + svc := &Service{ + monitorInterval: 15 * time.Second, + timeout: 10 * time.Second, + } + + // apply options + for _, opt := range opts { + opt(svc) + } + + // dependency injection + if err := container.GetContainer().Invoke(func(modelSvc service.ModelService) { + svc.modelSvc = modelSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + // initialize + svc.Init() + + // start + svc.Start() + + return svc, nil +} + +var _dsSvc interfaces.DataSourceService + +func GetDataSourceService() (svc interfaces.DataSourceService, err error) { + if _dsSvc != nil { + return _dsSvc, nil + } + svc, err = NewDataSourceService() + if err != nil { + return nil, err + } + _dsSvc = svc + return svc, nil +} diff --git a/core/ds/sql.go b/core/ds/sql.go new file mode 100644 index 000000000..4a1f1caf8 --- /dev/null +++ b/core/ds/sql.go @@ -0,0 +1,76 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + utils2 "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/upper/db/v4" + "time" +) + +type SqlService struct { + // dependencies + modelSvc service.ModelService + + // internals + ds *models.DataSource + dc *models.DataCollection + s db.Session + col db.Collection + t time.Time +} + +func (svc *SqlService) Insert(records ...interface{}) (err error) { + for _, d := range records { + var r entity.Result + switch d.(type) { + case entity.Result: + r = d.(entity.Result) + default: + continue + } + _r := r.Flatten() + if _, err = svc.col.Insert(_r); err != nil { + trace.PrintError(err) + continue + } + } + return nil +} + +func (svc *SqlService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { + var docs []entity.Result + if err := svc.col.Find(utils2.GetSqlQuery(query)). + Offset(opts.Skip). + Limit(opts.Limit).All(&docs); err != nil { + return nil, trace.TraceError(err) + } + for i := range docs { + d := docs[i].ToJSON() + results = append(results, &d) + } + return results, nil +} + +func (svc *SqlService) Count(query generic.ListQuery) (n int, err error) { + nInt64, err := svc.col.Find(utils2.GetSqlQuery(query)).Count() + if err != nil { + return n, err + } + return int(nInt64), nil +} + +func (svc *SqlService) Index(fields []string) { + // TODO: implement me +} + +func (svc *SqlService) SetTime(t time.Time) { + svc.t = t +} + +func (svc *SqlService) GetTime() (t time.Time) { + return svc.t +} diff --git a/core/ds/sql_options.go b/core/ds/sql_options.go new file mode 100644 index 000000000..e7d268764 --- /dev/null +++ b/core/ds/sql_options.go @@ -0,0 +1,6 @@ +package ds + +type SqlOptions struct { + DefaultHost string + DefaultPort string +} diff --git a/core/ds/sqlite.go b/core/ds/sqlite.go new file mode 100644 index 000000000..43fdb6f72 --- /dev/null +++ b/core/ds/sqlite.go @@ -0,0 +1,52 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + utils2 "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type SqliteService struct { + SqlService +} + +func NewDataSourceSqliteService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &SqliteService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, trace.TraceError(err) + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, trace.TraceError(err) + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, trace.TraceError(err) + } + + // session + svc.s, err = utils2.GetSqliteSession(svc.ds) + if err != nil { + return nil, trace.TraceError(err) + } + + // collection + svc.col = svc.s.Collection(svc.dc.Name) + + return svc, nil +} diff --git a/core/entity/address.go b/core/entity/address.go new file mode 100644 index 000000000..5b91bd1b2 --- /dev/null +++ b/core/entity/address.go @@ -0,0 +1,56 @@ +package entity + +import ( + "errors" + "fmt" + "strings" +) + +type Address struct { + Host string + Port string +} + +func (a *Address) String() (res string) { + return fmt.Sprintf("%s:%s", a.Host, a.Port) +} + +func (a *Address) IsEmpty() (res bool) { + return a.Host == "" || a.Port == "" +} + +func (a *Address) Value() (res interface{}) { + return a +} + +type AddressOptions struct { + Host string + Port string +} + +func NewAddress(opts *AddressOptions) (res *Address) { + if opts == nil { + opts = &AddressOptions{} + } + //if opts.Host == "" { + // opts.Host = "localhost" + //} + if opts.Port == "" { + opts.Port = "9666" + } + return &Address{ + Host: opts.Host, + Port: opts.Port, + } +} + +func NewAddressFromString(address string) (res *Address, err error) { + parts := strings.Split(address, ":") + if len(parts) == 1 { + return NewAddress(&AddressOptions{Host: parts[0]}), nil + } else if len(parts) == 2 { + return NewAddress(&AddressOptions{Host: parts[0], Port: parts[1]}), nil + } else { + return nil, errors.New(fmt.Sprintf("parsing address error: %v", err)) + } +} diff --git a/core/entity/color.go b/core/entity/color.go new file mode 100644 index 000000000..a8ed0c5b7 --- /dev/null +++ b/core/entity/color.go @@ -0,0 +1,18 @@ +package entity + +type Color struct { + Name string `json:"name"` + Hex string `json:"hex"` +} + +func (c *Color) GetHex() string { + return c.Hex +} + +func (c *Color) GetName() string { + return c.Name +} + +func (c *Color) Value() interface{} { + return c +} diff --git a/core/entity/common.go b/core/entity/common.go new file mode 100644 index 000000000..c46ae4f9b --- /dev/null +++ b/core/entity/common.go @@ -0,0 +1,17 @@ +package entity + +import "strconv" + +type Page struct { + Skip int + Limit int + PageNum int + PageSize int +} + +func (p *Page) GetPage(pageNum string, pageSize string) { + p.PageNum, _ = strconv.Atoi(pageNum) + p.PageSize, _ = strconv.Atoi(pageSize) + p.Skip = p.PageSize * (p.PageNum - 1) + p.Limit = p.PageSize +} diff --git a/core/entity/config_spider.go b/core/entity/config_spider.go new file mode 100644 index 000000000..054ee2fec --- /dev/null +++ b/core/entity/config_spider.go @@ -0,0 +1,40 @@ +package entity + +type ConfigSpiderData struct { + // 通用 + Name string `yaml:"name" json:"name"` + DisplayName string `yaml:"display_name" json:"display_name"` + Col string `yaml:"col" json:"col"` + Remark string `yaml:"remark" json:"remark"` + Type string `yaml:"type" bson:"type"` + + // 可配置爬虫 + Engine string `yaml:"engine" json:"engine"` + StartUrl string `yaml:"start_url" json:"start_url"` + StartStage string `yaml:"start_stage" json:"start_stage"` + Stages []Stage `yaml:"stages" json:"stages"` + Settings map[string]string `yaml:"settings" json:"settings"` + + // 自定义爬虫 + Cmd string `yaml:"cmd" json:"cmd"` +} + +type Stage struct { + Name string `yaml:"name" json:"name"` + IsList bool `yaml:"is_list" json:"is_list"` + ListCss string `yaml:"list_css" json:"list_css"` + ListXpath string `yaml:"list_xpath" json:"list_xpath"` + PageCss string `yaml:"page_css" json:"page_css"` + PageXpath string `yaml:"page_xpath" json:"page_xpath"` + PageAttr string `yaml:"page_attr" json:"page_attr"` + Fields []Field `yaml:"fields" json:"fields"` +} + +type Field struct { + Name string `yaml:"name" json:"name"` + Css string `yaml:"css" json:"css"` + Xpath string `yaml:"xpath" json:"xpath"` + Attr string `yaml:"attr" json:"attr"` + NextStage string `yaml:"next_stage" json:"next_stage"` + Remark string `yaml:"remark" json:"remark"` +} diff --git a/core/entity/data_field.go b/core/entity/data_field.go new file mode 100644 index 000000000..017aee2b9 --- /dev/null +++ b/core/entity/data_field.go @@ -0,0 +1,6 @@ +package entity + +type DataField struct { + Key string `json:"key" bson:"key"` + Type string `json:"type" bson:"type"` +} diff --git a/core/entity/doc.go b/core/entity/doc.go new file mode 100644 index 000000000..b356d38ad --- /dev/null +++ b/core/entity/doc.go @@ -0,0 +1,8 @@ +package entity + +type DocItem struct { + Title string `json:"title"` + Url string `json:"url"` + Path string `json:"path"` + Children []DocItem `json:"children"` +} diff --git a/core/entity/es.go b/core/entity/es.go new file mode 100644 index 000000000..d3b5614ac --- /dev/null +++ b/core/entity/es.go @@ -0,0 +1,54 @@ +package entity + +/* ElasticsearchResponseData JSON format +{ + "took" : 6, + "timed_out" : false, + "_shards" : { + "total" : 1, + "successful" : 1, + "skipped" : 0, + "failed" : 0 + }, + "hits" : { + "total" : { + "value" : 60, + "relation" : "eq" + }, + "max_score" : 1.0, + "hits" : [ + { + "_index" : "test_table", + "_id" : "c39ad9a2-9a37-49fb-b7ea-f1b55913e0af", + "_score" : 1.0, + "_source" : { + "_tid" : "62524ac7f5f99e7ef594de64", + "author" : "James Baldwin", + "tags" : [ + "love" + ], + "text" : "“Love does not begin and end the way we seem to think it does. Love is a battle, love is a war; love is a growing up.”" + } + } + ] + } +} +*/ + +type ElasticsearchResponseData struct { + Took int64 `json:"took"` + Timeout bool `json:"timeout"` + Hits struct { + Total struct { + Value int64 `json:"value"` + Relation string `json:"relation"` + } `json:"total"` + MaxScore float64 `json:"max_score"` + Hits []struct { + Index string `json:"_index"` + Id string `json:"_id"` + Score float64 `json:"_score"` + Source interface{} `json:"_source"` + } `json:"hits"` + } `json:"hits"` +} diff --git a/core/entity/event.go b/core/entity/event.go new file mode 100644 index 000000000..975cae60a --- /dev/null +++ b/core/entity/event.go @@ -0,0 +1,14 @@ +package entity + +type EventData struct { + Event string + Data interface{} +} + +func (d *EventData) GetEvent() string { + return d.Event +} + +func (d *EventData) GetData() interface{} { + return d.Data +} diff --git a/core/entity/export.go b/core/entity/export.go new file mode 100644 index 000000000..5bd06a7b5 --- /dev/null +++ b/core/entity/export.go @@ -0,0 +1,51 @@ +package entity + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type Export struct { + Id string `json:"id"` + Type string `json:"type"` + Target string `json:"target"` + Filter interfaces.Filter `json:"filter"` + Status string `json:"status"` + StartTs time.Time `json:"start_ts"` + EndTs time.Time `json:"end_ts"` + FileName string `json:"file_name"` + DownloadPath string `json:"-"` + Limit int `json:"-"` +} + +func (e *Export) GetId() string { + return e.Id +} + +func (e *Export) GetType() string { + return e.Type +} + +func (e *Export) GetTarget() string { + return e.Target +} + +func (e *Export) GetFilter() interfaces.Filter { + return e.Filter +} + +func (e *Export) GetStatus() string { + return e.Status +} + +func (e *Export) GetStartTs() time.Time { + return e.StartTs +} + +func (e *Export) GetEndTs() time.Time { + return e.EndTs +} + +func (e *Export) GetDownloadPath() string { + return e.DownloadPath +} diff --git a/core/entity/filter.go b/core/entity/filter.go new file mode 100644 index 000000000..6fcea3f35 --- /dev/null +++ b/core/entity/filter.go @@ -0,0 +1,68 @@ +package entity + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "reflect" +) + +type Condition struct { + Key string `json:"key"` + Op string `json:"op"` + Value interface{} `json:"value"` +} + +func (c *Condition) GetKey() (key string) { + return c.Key +} + +func (c *Condition) SetKey(key string) { + c.Key = key +} + +func (c *Condition) GetOp() (op string) { + return c.Op +} + +func (c *Condition) SetOp(op string) { + c.Op = op +} + +func (c *Condition) GetValue() (value interface{}) { + return c.Value +} + +func (c *Condition) SetValue(value interface{}) { + c.Value = value +} + +type Filter struct { + IsOr bool `form:"is_or" url:"is_or"` + Conditions []*Condition `json:"conditions"` +} + +func (f *Filter) GetIsOr() (isOr bool) { + return f.IsOr +} + +func (f *Filter) SetIsOr(isOr bool) { + f.IsOr = isOr +} + +func (f *Filter) GetConditions() (conditions []interfaces.FilterCondition) { + for _, c := range f.Conditions { + conditions = append(conditions, c) + } + return conditions +} + +func (f *Filter) SetConditions(conditions []interfaces.FilterCondition) { + f.Conditions = make([]*Condition, len(conditions)) + for _, c := range conditions { + f.Conditions = append(f.Conditions, c.(*Condition)) + } +} + +func (f *Filter) IsNil() (ok bool) { + val := reflect.ValueOf(f) + return val.IsNil() +} diff --git a/core/entity/filter_select_option.go b/core/entity/filter_select_option.go new file mode 100644 index 000000000..d3121683a --- /dev/null +++ b/core/entity/filter_select_option.go @@ -0,0 +1,6 @@ +package entity + +type FilterSelectOption struct { + Value interface{} `json:"value" bson:"value"` + Label string `json:"label" bson:"label"` +} diff --git a/core/entity/fs_file_info.go b/core/entity/fs_file_info.go new file mode 100644 index 000000000..826ec1316 --- /dev/null +++ b/core/entity/fs_file_info.go @@ -0,0 +1,60 @@ +package entity + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "os" + "time" +) + +type FsFileInfo struct { + Name string `json:"name"` // file name + Path string `json:"path"` // file path + FullPath string `json:"full_path"` // file full path + Extension string `json:"extension"` // file extension + IsDir bool `json:"is_dir"` // whether it is directory + FileSize int64 `json:"file_size"` // file size (bytes) + Children []interfaces.FsFileInfo `json:"children"` // children for sub-directory + ModTime time.Time `json:"mod_time"` // modification time + Mode os.FileMode `json:"mode"` // file mode + Hash string `json:"hash"` // file hash +} + +func (f *FsFileInfo) GetName() string { + return f.Name +} + +func (f *FsFileInfo) GetPath() string { + return f.Path +} + +func (f *FsFileInfo) GetFullPath() string { + return f.FullPath +} + +func (f *FsFileInfo) GetExtension() string { + return f.Extension +} + +func (f *FsFileInfo) GetIsDir() bool { + return f.IsDir +} + +func (f *FsFileInfo) GetFileSize() int64 { + return f.FileSize +} + +func (f *FsFileInfo) GetModTime() time.Time { + return f.ModTime +} + +func (f *FsFileInfo) GetMode() os.FileMode { + return f.Mode +} + +func (f *FsFileInfo) GetHash() string { + return f.Hash +} + +func (f *FsFileInfo) GetChildren() []interfaces.FsFileInfo { + return f.Children +} diff --git a/core/entity/git.go b/core/entity/git.go new file mode 100644 index 000000000..f11e2306d --- /dev/null +++ b/core/entity/git.go @@ -0,0 +1,12 @@ +package entity + +type GitPayload struct { + Paths []string `json:"paths"` + CommitMessage string `json:"commit_message"` + Branch string `json:"branch"` + Tag string `json:"tag"` +} + +type GitConfig struct { + Url string `json:"url" bson:"url"` +} diff --git a/core/entity/grpc_base_service_message.go b/core/entity/grpc_base_service_message.go new file mode 100644 index 000000000..6d51b1eff --- /dev/null +++ b/core/entity/grpc_base_service_message.go @@ -0,0 +1,29 @@ +package entity + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/go-trace" +) + +type GrpcBaseServiceMessage struct { + ModelId interfaces.ModelId `json:"id"` + Data []byte `json:"d"` +} + +func (msg *GrpcBaseServiceMessage) GetModelId() interfaces.ModelId { + return msg.ModelId +} + +func (msg *GrpcBaseServiceMessage) GetData() []byte { + return msg.Data +} + +func (msg *GrpcBaseServiceMessage) ToBytes() (data []byte) { + data, err := json.Marshal(*msg) + if err != nil { + _ = trace.TraceError(err) + return data + } + return data +} diff --git a/core/entity/grpc_base_service_params.go b/core/entity/grpc_base_service_params.go new file mode 100644 index 000000000..4d2fa9a0f --- /dev/null +++ b/core/entity/grpc_base_service_params.go @@ -0,0 +1,23 @@ +package entity + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type GrpcBaseServiceParams struct { + Query bson.M `json:"q"` + Id primitive.ObjectID `json:"id"` + Update bson.M `json:"u"` + Doc interfaces.Model `json:"d"` + Fields []string `json:"f"` + FindOptions *mongo.FindOptions `json:"o"` + Docs []interface{} `json:"dl"` + User interfaces.User `json:"U"` +} + +func (params *GrpcBaseServiceParams) Value() interface{} { + return params +} diff --git a/core/entity/grpc_delegate_message.go b/core/entity/grpc_delegate_message.go new file mode 100644 index 000000000..2760e7309 --- /dev/null +++ b/core/entity/grpc_delegate_message.go @@ -0,0 +1,34 @@ +package entity + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/go-trace" +) + +type GrpcDelegateMessage struct { + ModelId interfaces.ModelId `json:"id"` + Method interfaces.ModelDelegateMethod `json:"m"` + Data []byte `json:"d"` +} + +func (msg *GrpcDelegateMessage) GetModelId() interfaces.ModelId { + return msg.ModelId +} + +func (msg *GrpcDelegateMessage) GetMethod() interfaces.ModelDelegateMethod { + return msg.Method +} + +func (msg *GrpcDelegateMessage) GetData() []byte { + return msg.Data +} + +func (msg *GrpcDelegateMessage) ToBytes() (data []byte) { + data, err := json.Marshal(*msg) + if err != nil { + _ = trace.TraceError(err) + return data + } + return data +} diff --git a/core/entity/grpc_event_service_message.go b/core/entity/grpc_event_service_message.go new file mode 100644 index 000000000..93b41de18 --- /dev/null +++ b/core/entity/grpc_event_service_message.go @@ -0,0 +1,8 @@ +package entity + +type GrpcEventServiceMessage struct { + Type string `json:"type"` + Events []string `json:"events"` + Key string `json:"key"` + Data []byte `json:"data"` +} diff --git a/core/entity/grpc_subscribe.go b/core/entity/grpc_subscribe.go new file mode 100644 index 000000000..79ae46eeb --- /dev/null +++ b/core/entity/grpc_subscribe.go @@ -0,0 +1,26 @@ +package entity + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" +) + +type GrpcSubscribe struct { + Stream interfaces.GrpcStream + Finished chan bool +} + +func (sub *GrpcSubscribe) GetStream() interfaces.GrpcStream { + return sub.Stream +} + +func (sub *GrpcSubscribe) GetStreamBidirectional() interfaces.GrpcStreamBidirectional { + stream, ok := sub.Stream.(interfaces.GrpcStreamBidirectional) + if !ok { + return nil + } + return stream +} + +func (sub *GrpcSubscribe) GetFinished() chan bool { + return sub.Finished +} diff --git a/core/entity/http.go b/core/entity/http.go new file mode 100644 index 000000000..a958b74bd --- /dev/null +++ b/core/entity/http.go @@ -0,0 +1,42 @@ +package entity + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type Response struct { + Status string `json:"status"` + Message string `json:"message"` + Data interface{} `json:"data"` + Error string `json:"error"` +} + +type ListResponse struct { + Status string `json:"status"` + Message string `json:"message"` + Total int `json:"total"` + Data interface{} `json:"data"` + Error string `json:"error"` +} + +type ListRequestData struct { + PageNum int `form:"page_num" json:"page_num"` + PageSize int `form:"page_size" json:"page_size"` + SortKey string `form:"sort_key" json:"sort_key"` + Status string `form:"status" json:"status"` + Keyword string `form:"keyword" json:"keyword"` +} + +type BatchRequestPayload struct { + Ids []primitive.ObjectID `form:"ids" json:"ids"` +} + +type BatchRequestPayloadWithStringData struct { + Ids []primitive.ObjectID `form:"ids" json:"ids"` + Data string `form:"data" json:"data"` + Fields []string `form:"fields" json:"fields"` +} + +type FileRequestPayload struct { + Path string `json:"path" form:"path"` + NewPath string `json:"new_path" form:"new_path"` + Data string `json:"data" form:"data"` +} diff --git a/core/entity/model_delegate.go b/core/entity/model_delegate.go new file mode 100644 index 000000000..533487a05 --- /dev/null +++ b/core/entity/model_delegate.go @@ -0,0 +1,11 @@ +package entity + +import "github.com/crawlab-team/crawlab/core/interfaces" + +type ModelDelegate struct { + Id interfaces.ModelId `json:"id"` + ColName string `json:"col_name"` + Doc interfaces.Model `json:"doc"` + Artifact interfaces.ModelArtifact `json:"a"` + User interfaces.User `json:"u"` +} diff --git a/core/entity/model_info.go b/core/entity/model_info.go new file mode 100644 index 000000000..e7c9e06fc --- /dev/null +++ b/core/entity/model_info.go @@ -0,0 +1,8 @@ +package entity + +import "github.com/crawlab-team/crawlab/core/interfaces" + +type ModelInfo struct { + Id interfaces.ModelId + ColName string +} diff --git a/core/entity/node.go b/core/entity/node.go new file mode 100644 index 000000000..6289c089b --- /dev/null +++ b/core/entity/node.go @@ -0,0 +1,17 @@ +package entity + +type NodeInfo struct { + Key string `json:"key"` + IsMaster bool `json:"is_master"` + Name string `json:"name"` + Ip string `json:"ip"` + Mac string `json:"mac"` + Hostname string `json:"hostname"` + Description string `json:"description"` + AuthKey string `json:"auth_key"` + MaxRunners int `json:"max_runners"` +} + +func (n NodeInfo) Value() interface{} { + return n +} diff --git a/core/entity/pagination.go b/core/entity/pagination.go new file mode 100644 index 000000000..ba891f54f --- /dev/null +++ b/core/entity/pagination.go @@ -0,0 +1,18 @@ +package entity + +import "github.com/crawlab-team/crawlab/core/constants" + +type Pagination struct { + Page int `form:"page" url:"page"` + Size int `form:"size" url:"size"` +} + +func (p *Pagination) IsZero() (ok bool) { + return p.Page == 0 && + p.Size == 0 +} + +func (p *Pagination) IsDefault() (ok bool) { + return p.Page == constants.PaginationDefaultPage && + p.Size == constants.PaginationDefaultSize +} diff --git a/core/entity/result.go b/core/entity/result.go new file mode 100644 index 000000000..db0970c7f --- /dev/null +++ b/core/entity/result.go @@ -0,0 +1,100 @@ +package entity + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Result map[string]interface{} + +func (r Result) Value() map[string]interface{} { + return r +} + +func (r Result) SetValue(key string, value interface{}) { + r[key] = value +} + +func (r Result) GetValue(key string) (value interface{}) { + value, _ = r[key] + return value +} + +func (r Result) GetTaskId() (id primitive.ObjectID) { + _tid, ok := r[constants.TaskKey] + if !ok { + return id + } + switch _tid.(type) { + case string: + oid, err := primitive.ObjectIDFromHex(_tid.(string)) + if err != nil { + return id + } + return oid + default: + return id + } +} + +func (r Result) SetTaskId(id primitive.ObjectID) { + r[constants.TaskKey] = id +} + +func (r Result) DenormalizeObjectId() (res Result) { + for k, v := range r { + switch v.(type) { + case primitive.ObjectID: + r[k] = v.(primitive.ObjectID).Hex() + case Result: + r[k] = v.(Result).DenormalizeObjectId() + } + } + return r +} + +func (r Result) ToJSON() (res Result) { + r = r.DenormalizeObjectId() + for k, v := range r { + switch v.(type) { + case []byte: + r[k] = string(v.([]byte)) + } + } + return r +} + +func (r Result) Flatten() (res Result) { + r = r.ToJSON() + for k, v := range r { + switch v.(type) { + case string, + bool, + uint, uint8, uint16, uint32, uint64, + int, int8, int16, int32, int64, + float32, float64: + default: + bytes, err := json.Marshal(v) + if err != nil { + trace.PrintError(err) + return nil + } + r[k] = string(bytes) + } + } + return r +} + +func (r Result) String() (s string) { + return string(r.Bytes()) +} + +func (r Result) Bytes() (bytes []byte) { + bytes, err := json.Marshal(r.ToJSON()) + if err != nil { + return bytes + } + return bytes +} diff --git a/core/entity/rpc.go b/core/entity/rpc.go new file mode 100644 index 000000000..48f14b266 --- /dev/null +++ b/core/entity/rpc.go @@ -0,0 +1,11 @@ +package entity + +type RpcMessage struct { + Id string `json:"id"` // 消息ID + Method string `json:"method"` // 消息方法 + NodeId string `json:"node_id"` // 节点ID + Params map[string]string `json:"params"` // 参数 + Timeout int `json:"timeout"` // 超时 + Result string `json:"result"` // 结果 + Error string `json:"error"` // 错误 +} diff --git a/core/entity/sort.go b/core/entity/sort.go new file mode 100644 index 000000000..482dc6c85 --- /dev/null +++ b/core/entity/sort.go @@ -0,0 +1,6 @@ +package entity + +type Sort struct { + Key string `json:"key"` + Direction string `json:"d"` +} diff --git a/core/entity/spider.go b/core/entity/spider.go new file mode 100644 index 000000000..616d3bbf7 --- /dev/null +++ b/core/entity/spider.go @@ -0,0 +1,17 @@ +package entity + +type SpiderType struct { + Type string `json:"type" bson:"_id"` + Count int `json:"count" bson:"count"` +} + +type ScrapySettingParam struct { + Key string `json:"key"` + Value interface{} `json:"value"` + Type string `json:"type"` +} + +type ScrapyItem struct { + Name string `json:"name"` + Fields []string `json:"fields"` +} diff --git a/core/entity/stats.go b/core/entity/stats.go new file mode 100644 index 000000000..7cc228cb2 --- /dev/null +++ b/core/entity/stats.go @@ -0,0 +1,12 @@ +package entity + +type StatsDailyItem struct { + Date string `json:"date" bson:"_id"` + Tasks int64 `json:"tasks" bson:"tasks"` + Results int64 `json:"results" bson:"results"` +} + +type StatsTasksByStatusItem struct { + Status string `json:"status" bson:"_id"` + Tasks int64 `json:"tasks" bson:"tasks"` +} diff --git a/core/entity/system_info.go b/core/entity/system_info.go new file mode 100644 index 000000000..692a27755 --- /dev/null +++ b/core/entity/system_info.go @@ -0,0 +1,6 @@ +package entity + +type SystemInfo struct { + Edition string `json:"edition"` // edition. e.g. community / pro + Version string `json:"version"` // version. e.g. v0.6.0 +} diff --git a/core/entity/task.go b/core/entity/task.go new file mode 100644 index 000000000..aae4c199b --- /dev/null +++ b/core/entity/task.go @@ -0,0 +1,30 @@ +package entity + +import ( + "encoding/json" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type TaskMessage struct { + Id primitive.ObjectID `json:"id"` + Key string `json:"key"` + Cmd string `json:"cmd"` + Param string `json:"param"` +} + +func (m *TaskMessage) ToString() (string, error) { + data, err := json.Marshal(&m) + if err != nil { + return "", err + } + return string(data), err +} + +type TaskRunOptions struct { +} + +type StreamMessageTaskData struct { + TaskId primitive.ObjectID `json:"task_id"` + Records []Result `json:"data"` + Logs []string `json:"logs"` +} diff --git a/core/entity/translation.go b/core/entity/translation.go new file mode 100644 index 000000000..f54ff2946 --- /dev/null +++ b/core/entity/translation.go @@ -0,0 +1,11 @@ +package entity + +type Translation struct { + Lang string `json:"lang"` + Key string `json:"key"` + Value string `json:"value"` +} + +func (t Translation) GetLang() (l string) { + return t.Lang +} diff --git a/core/entity/ttl_map.go b/core/entity/ttl_map.go new file mode 100644 index 000000000..57f7fd646 --- /dev/null +++ b/core/entity/ttl_map.go @@ -0,0 +1,58 @@ +package entity + +import ( + "sync" + "time" +) + +type TTLMap struct { + TTL time.Duration + + data sync.Map +} + +type expireEntry struct { + ExpiresAt time.Time + Value interface{} +} + +func (t *TTLMap) Store(key string, val interface{}) { + t.data.Store(key, expireEntry{ + ExpiresAt: time.Now().Add(t.TTL), + Value: val, + }) +} + +func (t *TTLMap) Load(key string) (val interface{}) { + entry, ok := t.data.Load(key) + if !ok { + return nil + } + + expireEntry := entry.(expireEntry) + if expireEntry.ExpiresAt.After(time.Now()) { + return nil + } + + return expireEntry.Value +} + +func NewTTLMap(ttl time.Duration) (m *TTLMap) { + m = &TTLMap{ + TTL: ttl, + } + + go func() { + for now := range time.Tick(time.Second) { + m.data.Range(func(k, v interface{}) bool { + expiresAt := v.(expireEntry).ExpiresAt + if expiresAt.Before(now) { + m.data.Delete(k) + } + return true + }) + } + }() + + return +} diff --git a/core/entity/version.go b/core/entity/version.go new file mode 100644 index 000000000..97a0278d5 --- /dev/null +++ b/core/entity/version.go @@ -0,0 +1,23 @@ +package entity + +type Release struct { + Name string `json:"name"` + Draft bool `json:"draft"` + PreRelease bool `json:"pre_release"` + PublishedAt string `json:"published_at"` + Body string `json:"body"` +} + +type ReleaseSlices []Release + +func (r ReleaseSlices) Len() int { + return len(r) +} + +func (r ReleaseSlices) Less(i, j int) bool { + return r[i].PublishedAt < r[j].PublishedAt +} + +func (r ReleaseSlices) Swap(i, j int) { + r[i], r[j] = r[j], r[i] +} diff --git a/core/errors/base.go b/core/errors/base.go new file mode 100644 index 000000000..0acfb6815 --- /dev/null +++ b/core/errors/base.go @@ -0,0 +1,33 @@ +package errors + +import ( + "errors" + "fmt" +) + +const ( + ErrorPrefixController = "controller" + ErrorPrefixModel = "model" + ErrorPrefixFilter = "filter" + ErrorPrefixHttp = "http" + ErrorPrefixGrpc = "grpc" + ErrorPrefixNode = "node" + ErrorPrefixInject = "inject" + ErrorPrefixSpider = "spider" + ErrorPrefixFs = "fs" + ErrorPrefixTask = "task" + ErrorPrefixSchedule = "schedule" + ErrorPrefixUser = "user" + ErrorPrefixStats = "stats" + ErrorPrefixEvent = "event" + ErrorPrefixProcess = "process" + ErrorPrefixGit = "git" + ErrorPrefixResult = "result" + ErrorPrefixDataSource = "data_source" +) + +type ErrorPrefix string + +func NewError(prefix ErrorPrefix, msg string) (err error) { + return errors.New(fmt.Sprintf("%s error: %s", prefix, msg)) +} diff --git a/core/errors/controller.go b/core/errors/controller.go new file mode 100644 index 000000000..7c2f6bffb --- /dev/null +++ b/core/errors/controller.go @@ -0,0 +1,19 @@ +package errors + +func NewControllerError(msg string) (err error) { + return NewError(ErrorPrefixController, msg) +} + +var ErrorControllerInvalidControllerId = NewControllerError("invalid controller id") +var ErrorControllerInvalidType = NewControllerError("invalid type") +var ErrorControllerAddError = NewControllerError("add error") +var ErrorControllerUpdateError = NewControllerError("update error") +var ErrorControllerDeleteError = NewControllerError("delete error") +var ErrorControllerNotImplemented = NewControllerError("not implemented") +var ErrorControllerNoModelService = NewControllerError("no model service") +var ErrorControllerRequestPayloadInvalid = NewControllerError("request payload invalid") +var ErrorControllerMissingInCache = NewControllerError("missing in cache") +var ErrorControllerNotCancellable = NewControllerError("not cancellable") +var ErrorControllerMissingRequestFields = NewControllerError("missing request fields") +var ErrorControllerEmptyResponse = NewControllerError("empty response") +var ErrorControllerFilerNotFound = NewControllerError("filer not found") diff --git a/core/errors/ds.go b/core/errors/ds.go new file mode 100644 index 000000000..f9550160c --- /dev/null +++ b/core/errors/ds.go @@ -0,0 +1,15 @@ +package errors + +func NewDataSourceError(msg string) (err error) { + return NewError(ErrorPrefixDataSource, msg) +} + +var ( + ErrorDataSourceInvalidType = NewDataSourceError("invalid type") + ErrorDataSourceNotExists = NewDataSourceError("not exists") + ErrorDataSourceNotExistsInContext = NewDataSourceError("not exists in context") + ErrorDataSourceAlreadyExists = NewDataSourceError("already exists") + ErrorDataSourceMismatch = NewDataSourceError("mismatch") + ErrorDataSourceMissingRequiredFields = NewDataSourceError("missing required fields") + ErrorDataSourceUnauthorized = NewDataSourceError("unauthorized") +) diff --git a/core/errors/event.go b/core/errors/event.go new file mode 100644 index 000000000..5a5281007 --- /dev/null +++ b/core/errors/event.go @@ -0,0 +1,10 @@ +package errors + +func NewEventError(msg string) (err error) { + return NewError(ErrorPrefixEvent, msg) +} + +var ErrorEventNotFound = NewEventError("not found") +var ErrorEventInvalidType = NewEventError("invalid type") +var ErrorEventAlreadyExists = NewEventError("already exists") +var ErrorEventUnknownAction = NewEventError("unknown action") diff --git a/core/errors/filter.go b/core/errors/filter.go new file mode 100644 index 000000000..0afaec397 --- /dev/null +++ b/core/errors/filter.go @@ -0,0 +1,8 @@ +package errors + +func NewFilterError(msg string) (err error) { + return NewError(ErrorPrefixFilter, msg) +} + +var ErrorFilterInvalidOperation = NewFilterError("invalid operation") +var ErrorFilterUnableToParseQuery = NewFilterError("unable to parse query") diff --git a/core/errors/fs.go b/core/errors/fs.go new file mode 100644 index 000000000..2c91cae24 --- /dev/null +++ b/core/errors/fs.go @@ -0,0 +1,11 @@ +package errors + +func NewFsError(msg string) (err error) { + return NewError(ErrorPrefixFs, msg) +} + +var ErrorFsForbidden = NewFsError("forbidden") +var ErrorFsEmptyWorkspacePath = NewFsError("empty workspace path") +var ErrorFsInvalidType = NewFsError("invalid type") +var ErrorFsAlreadyExists = NewFsError("already exists") +var ErrorFsInvalidContent = NewFsError("invalid content") diff --git a/core/errors/git.go b/core/errors/git.go new file mode 100644 index 000000000..ea7e03a9e --- /dev/null +++ b/core/errors/git.go @@ -0,0 +1,10 @@ +package errors + +func NewGitError(msg string) (err error) { + return NewError(ErrorPrefixGit, msg) +} + +var ( + ErrorGitInvalidAuthType = NewGitError("invalid auth type") + ErrorGitNoMainBranch = NewGitError("no main branch") +) diff --git a/core/errors/grpc.go b/core/errors/grpc.go new file mode 100644 index 000000000..fbfdabb6b --- /dev/null +++ b/core/errors/grpc.go @@ -0,0 +1,20 @@ +package errors + +func NewGrpcError(msg string) (err error) { + return NewError(ErrorPrefixGrpc, msg) +} + +var ( + ErrorGrpcClientFailedToStart = NewGrpcError("client failed to start") + ErrorGrpcServerFailedToListen = NewGrpcError("server failed to listen") + ErrorGrpcServerFailedToServe = NewGrpcError("server failed to serve") + ErrorGrpcClientNotExists = NewGrpcError("client not exists") + ErrorGrpcClientAlreadyExists = NewGrpcError("client already exists") + ErrorGrpcInvalidType = NewGrpcError("invalid type") + ErrorGrpcNotAllowed = NewGrpcError("not allowed") + ErrorGrpcSubscribeNotExists = NewGrpcError("subscribe not exists") + ErrorGrpcStreamNotFound = NewGrpcError("stream not found") + ErrorGrpcInvalidCode = NewGrpcError("invalid code") + ErrorGrpcUnauthorized = NewGrpcError("unauthorized") + ErrorGrpcInvalidNodeKey = NewGrpcError("invalid node key") +) diff --git a/core/errors/http.go b/core/errors/http.go new file mode 100644 index 000000000..e10ad28fb --- /dev/null +++ b/core/errors/http.go @@ -0,0 +1,9 @@ +package errors + +func NewHttpError(msg string) (err error) { + return NewError(ErrorPrefixHttp, msg) +} + +var ErrorHttpBadRequest = NewHttpError("bad request") +var ErrorHttpUnauthorized = NewHttpError("unauthorized") +var ErrorHttpNotFound = NewHttpError("not found") diff --git a/core/errors/model.go b/core/errors/model.go new file mode 100644 index 000000000..95baa31e6 --- /dev/null +++ b/core/errors/model.go @@ -0,0 +1,19 @@ +package errors + +import "errors" + +func NewModelError(msg string) (err error) { + return NewError(ErrorPrefixModel, msg) +} + +var ErrorModelInvalidType = NewModelError("invalid type") +var ErrorModelInvalidModelId = NewModelError("invalid model id") +var ErrorModelNotImplemented = NewModelError("not implemented") +var ErrorModelNotFound = NewModelError("not found") +var ErrorModelAlreadyExists = NewModelError("already exists") +var ErrorModelNotExists = NewModelError("not exists") +var ErrorModelMissingRequiredData = NewModelError("missing required data") +var ErrorModelMissingId = errors.New("missing _id") +var ErrorModelNotAllowed = NewModelError("not allowed") +var ErrorModelDeleteListError = NewModelError("delete list error") +var ErrorModelNilPointer = NewModelError("nil pointer") diff --git a/core/errors/node.go b/core/errors/node.go new file mode 100644 index 000000000..e768ca521 --- /dev/null +++ b/core/errors/node.go @@ -0,0 +1,14 @@ +package errors + +func NewNodeError(msg string) (err error) { + return NewError(ErrorPrefixNode, msg) +} + +var ErrorNodeUnregistered = NewNodeError("unregistered") +var ErrorNodeServiceNotExists = NewNodeError("service not exists") +var ErrorNodeInvalidType = NewNodeError("invalid type") +var ErrorNodeInvalidStatus = NewNodeError("invalid status") +var ErrorNodeInvalidCode = NewNodeError("invalid code") +var ErrorNodeInvalidNodeKey = NewNodeError("invalid node key") +var ErrorNodeMonitorError = NewNodeError("monitor error") +var ErrorNodeNotExists = NewNodeError("not exists") diff --git a/core/errors/process.go b/core/errors/process.go new file mode 100644 index 000000000..aa5580b26 --- /dev/null +++ b/core/errors/process.go @@ -0,0 +1,10 @@ +package errors + +func NewProcessError(msg string) (err error) { + return NewError(ErrorPrefixProcess, msg) +} + +var ( + ErrorProcessReachedMaxErrors = NewProcessError("reached max errors") + ErrorProcessDaemonProcessExited = NewProcessError("daemon process exited") +) diff --git a/core/errors/result.go b/core/errors/result.go new file mode 100644 index 000000000..39b1f4594 --- /dev/null +++ b/core/errors/result.go @@ -0,0 +1,5 @@ +package errors + +func NewResultError(msg string) (err error) { + return NewError(ErrorPrefixResult, msg) +} diff --git a/core/errors/schedule.go b/core/errors/schedule.go new file mode 100644 index 000000000..8246b1be1 --- /dev/null +++ b/core/errors/schedule.go @@ -0,0 +1,7 @@ +package errors + +func NewScheduleError(msg string) (err error) { + return NewError(ErrorPrefixSchedule, msg) +} + +//var ErrorSchedule = NewScheduleError("unregistered") diff --git a/core/errors/spider.go b/core/errors/spider.go new file mode 100644 index 000000000..d7fa5c6ae --- /dev/null +++ b/core/errors/spider.go @@ -0,0 +1,10 @@ +package errors + +func NewSpiderError(msg string) (err error) { + return NewError(ErrorPrefixSpider, msg) +} + +var ( + ErrorSpiderMissingRequiredOption = NewSpiderError("missing required option") + ErrorSpiderForbidden = NewSpiderError("forbidden") +) diff --git a/core/errors/stats.go b/core/errors/stats.go new file mode 100644 index 000000000..fab5f4643 --- /dev/null +++ b/core/errors/stats.go @@ -0,0 +1,7 @@ +package errors + +func NewStatsError(msg string) (err error) { + return NewError(ErrorPrefixStats, msg) +} + +var ErrorStatsInvalidType = NewStatsError("invalid type") diff --git a/core/errors/store.go b/core/errors/store.go new file mode 100644 index 000000000..ead4fb67b --- /dev/null +++ b/core/errors/store.go @@ -0,0 +1,9 @@ +package errors + +func NewInjectError(msg string) (err error) { + return NewError(ErrorPrefixInject, msg) +} + +var ErrorInjectEmptyValue = NewInjectError("empty value") +var ErrorInjectNotExists = NewInjectError("not exists") +var ErrorInjectInvalidType = NewInjectError("invalid type") diff --git a/core/errors/task.go b/core/errors/task.go new file mode 100644 index 000000000..d4ec41ca5 --- /dev/null +++ b/core/errors/task.go @@ -0,0 +1,19 @@ +package errors + +func NewTaskError(msg string) (err error) { + return NewError(ErrorPrefixTask, msg) +} + +var ( + ErrorTaskNotExists = NewTaskError("not exists") + ErrorTaskAlreadyExists = NewTaskError("already exists") + ErrorTaskInvalidType = NewTaskError("invalid type") + ErrorTaskProcessStillExists = NewTaskError("process still exists") + ErrorTaskUnableToCancel = NewTaskError("unable to cancel") + ErrorTaskForbidden = NewTaskError("forbidden") + ErrorTaskNoAvailableRunners = NewTaskError("no available runner") + ErrorTaskEmptySpiderId = NewTaskError("empty spider id") + ErrorTaskNoNodeId = NewTaskError("no node id") + ErrorTaskNodeNotFound = NewTaskError("node not found") + ErrorTaskMissingRequiredOption = NewSpiderError("missing required option") +) diff --git a/core/errors/user.go b/core/errors/user.go new file mode 100644 index 000000000..b0a1e940d --- /dev/null +++ b/core/errors/user.go @@ -0,0 +1,17 @@ +package errors + +func NewUserError(msg string) (err error) { + return NewError(ErrorPrefixUser, msg) +} + +var ( + ErrorUserInvalidType = NewUserError("invalid type") + ErrorUserInvalidToken = NewUserError("invalid token") + ErrorUserNotExists = NewUserError("not exists") + ErrorUserNotExistsInContext = NewUserError("not exists in context") + ErrorUserAlreadyExists = NewUserError("already exists") + ErrorUserMismatch = NewUserError("mismatch") + ErrorUserMissingRequiredFields = NewUserError("missing required fields") + ErrorUserUnauthorized = NewUserError("unauthorized") + ErrorUserInvalidPassword = NewUserError("invalid password (length must be no less than 5)") +) diff --git a/core/event/func.go b/core/event/func.go new file mode 100644 index 000000000..21c7693ed --- /dev/null +++ b/core/event/func.go @@ -0,0 +1,6 @@ +package event + +func SendEvent(eventName string, data ...interface{}) { + svc := NewEventService() + svc.SendEvent(eventName, data...) +} diff --git a/core/event/options.go b/core/event/options.go new file mode 100644 index 000000000..0e4b82e85 --- /dev/null +++ b/core/event/options.go @@ -0,0 +1 @@ +package event diff --git a/core/event/service.go b/core/event/service.go new file mode 100644 index 000000000..8fdf93397 --- /dev/null +++ b/core/event/service.go @@ -0,0 +1,92 @@ +package event + +import ( + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/thoas/go-funk" + "regexp" +) + +var S interfaces.EventService + +type Service struct { + keys []string + includes []string + excludes []string + chs []*chan interfaces.EventData +} + +func (svc *Service) Register(key, include, exclude string, ch *chan interfaces.EventData) { + svc.keys = append(svc.keys, key) + svc.includes = append(svc.includes, include) + svc.excludes = append(svc.excludes, exclude) + svc.chs = append(svc.chs, ch) +} + +func (svc *Service) Unregister(key string) { + idx := funk.IndexOfString(svc.keys, key) + if idx != -1 { + svc.keys = append(svc.keys[:idx], svc.keys[(idx+1):]...) + svc.includes = append(svc.includes[:idx], svc.includes[(idx+1):]...) + svc.excludes = append(svc.excludes[:idx], svc.excludes[(idx+1):]...) + svc.chs = append(svc.chs[:idx], svc.chs[(idx+1):]...) + log.Infof("[EventService] unregistered %s", key) + } +} + +func (svc *Service) SendEvent(eventName string, data ...interface{}) { + for i, key := range svc.keys { + // include + include := svc.includes[i] + matchedInclude, err := regexp.MatchString(include, eventName) + if err != nil { + trace.PrintError(err) + continue + } + if !matchedInclude { + continue + } + + // exclude + exclude := svc.excludes[i] + matchedExclude, err := regexp.MatchString(exclude, eventName) + if err != nil { + trace.PrintError(err) + continue + } + if matchedExclude { + continue + } + + // send event + utils.LogDebug(fmt.Sprintf("key %s matches event %s", key, eventName)) + ch := svc.chs[i] + go func(ch *chan interfaces.EventData) { + for _, d := range data { + *ch <- &entity.EventData{ + Event: eventName, + Data: d, + } + } + }(ch) + } +} + +func NewEventService() (svc interfaces.EventService) { + if S != nil { + return S + } + + svc = &Service{ + chs: []*chan interfaces.EventData{}, + keys: []string{}, + } + + S = svc + + return svc +} diff --git a/core/export/csv_service.go b/core/export/csv_service.go new file mode 100644 index 000000000..3c1342c6e --- /dev/null +++ b/core/export/csv_service.go @@ -0,0 +1,344 @@ +package export + +import ( + "context" + "encoding/csv" + "errors" + "fmt" + "github.com/ReneKroon/ttlcache" + "github.com/apex/log" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/hashicorp/go-uuid" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "os" + "path" + "sort" + "strconv" + "time" +) + +type CsvService struct { + cache *ttlcache.Cache +} + +func (svc *CsvService) GenerateId() (exportId string, err error) { + exportId, err = uuid.GenerateUUID() + if err != nil { + return "", trace.TraceError(err) + } + return exportId, nil +} + +func (svc *CsvService) Export(exportType, target string, filter interfaces.Filter) (exportId string, err error) { + // generate export id + exportId, err = svc.GenerateId() + if err != nil { + return "", err + } + + // export + export := &entity.Export{ + Id: exportId, + Type: exportType, + Target: target, + Filter: filter, + Status: constants.TaskStatusRunning, + StartTs: time.Now(), + FileName: svc.getFileName(exportId), + DownloadPath: svc.getDownloadPath(exportId), + Limit: 100, + } + + // save to cache + svc.cache.Set(exportId, export) + + // execute export + go svc.export(export) + + return exportId, nil +} + +func (svc *CsvService) GetExport(exportId string) (export interfaces.Export, err error) { + // get export from cache + res, ok := svc.cache.Get(exportId) + if !ok { + return nil, trace.TraceError(errors.New("export not found")) + } + export = res.(interfaces.Export) + return export, nil +} + +func (svc *CsvService) export(export *entity.Export) { + // check empty + if export.Target == "" { + err := errors.New("empty target") + export.Status = constants.TaskStatusError + export.EndTs = time.Now() + log.Errorf("export error (id: %s): %v", export.Id, err) + trace.PrintError(err) + svc.cache.Set(export.Id, export) + return + } + + // mongo collection + col := mongo.GetMongoCol(export.Target) + + // mongo query + query, err := utils.FilterToQuery(export.Filter) + if err != nil { + export.Status = constants.TaskStatusError + export.EndTs = time.Now() + log.Errorf("export error (id: %s): %v", export.Id, err) + trace.PrintError(err) + svc.cache.Set(export.Id, export) + return + } + + // mongo cursor + cur := col.Find(query, nil).GetCursor() + + // csv writer + csvWriter, csvFile, err := svc.getCsvWriter(export) + defer func() { + csvWriter.Flush() + _ = csvFile.Close() + }() + if err != nil { + export.Status = constants.TaskStatusError + export.EndTs = time.Now() + log.Errorf("export error (id: %s): %v", export.Id, err) + trace.PrintError(err) + svc.cache.Set(export.Id, export) + return + } + + // write bom + bom := []byte{0xEF, 0xBB, 0xBF} + _, err = csvFile.Write(bom) + if err != nil { + trace.PrintError(err) + return + } + + // write csv header row + columns, err := svc.getColumns(query, export) + err = csvWriter.Write(columns) + if err != nil { + export.Status = constants.TaskStatusError + export.EndTs = time.Now() + log.Errorf("export error (id: %s): %v", export.Id, err) + trace.PrintError(err) + svc.cache.Set(export.Id, export) + return + } + csvWriter.Flush() + + // iterate cursor + i := 0 + for { + // increment counter + i++ + + // check error + err := cur.Err() + if err != nil { + if err != mongo2.ErrNoDocuments { + // error + export.Status = constants.TaskStatusError + export.EndTs = time.Now() + log.Errorf("export error (id: %s): %v", export.Id, err) + trace.PrintError(err) + } else { + // no more data + export.Status = constants.TaskStatusFinished + export.EndTs = time.Now() + log.Infof("export finished (id: %s)", export.Id) + } + svc.cache.Set(export.Id, export) + return + } + + // has data + if !cur.Next(context.Background()) { + // no more data + export.Status = constants.TaskStatusFinished + export.EndTs = time.Now() + log.Infof("export finished (id: %s)", export.Id) + svc.cache.Set(export.Id, export) + return + } + + // convert raw data to entity + var data bson.M + err = cur.Decode(&data) + if err != nil { + // error + export.Status = constants.TaskStatusError + export.EndTs = time.Now() + log.Errorf("export error (id: %s): %v", export.Id, err) + trace.PrintError(err) + svc.cache.Set(export.Id, export) + return + } + + // write csv row cells + cells := svc.getRowCells(columns, data) + err = csvWriter.Write(cells) + if err != nil { + // error + export.Status = constants.TaskStatusError + export.EndTs = time.Now() + log.Errorf("export error (id: %s): %v", export.Id, err) + trace.PrintError(err) + svc.cache.Set(export.Id, export) + return + } + + // flush if limit reached + if i >= export.Limit { + csvWriter.Flush() + i = 0 + } + } +} + +func (svc *CsvService) getExportDir() (dir string, err error) { + tempDir := os.TempDir() + exportDir := path.Join(tempDir, "export", "csv") + if !utils.Exists(exportDir) { + err := os.MkdirAll(exportDir, 0755) + if err != nil { + return "", err + } + } + return exportDir, nil +} + +func (svc *CsvService) getFileName(exportId string) (fileName string) { + return exportId + "_" + time.Now().Format("20060102150405") + ".csv" +} + +// getDownloadPath returns the download path for the export +// format: /export//_.csv +func (svc *CsvService) getDownloadPath(exportId string) (downloadPath string) { + exportDir, err := svc.getExportDir() + if err != nil { + return "" + } + downloadPath = path.Join(exportDir, svc.getFileName(exportId)) + return downloadPath +} + +func (svc *CsvService) getCsvWriter(export *entity.Export) (csvWriter *csv.Writer, csvFile *os.File, err error) { + // open file + csvFile, err = os.Create(export.DownloadPath) + if err != nil { + return nil, nil, trace.TraceError(err) + } + + // create csv writer + csvWriter = csv.NewWriter(csvFile) + + return csvWriter, csvFile, nil +} + +func (svc *CsvService) getColumns(query bson.M, export interfaces.Export) (columns []string, err error) { + // get mongo collection + col := mongo.GetMongoCol(export.GetTarget()) + + // get 10 records + var data []bson.M + if err := col.Find(query, &mongo.FindOptions{Limit: 10}).All(&data); err != nil { + return nil, trace.TraceError(err) + } + + // columns set + columnsSet := make(map[string]bool) + for _, d := range data { + for k := range d { + columnsSet[k] = true + } + } + + // columns + columns = make([]string, 0, len(columnsSet)) + for k := range columnsSet { + // skip task key + if k == constants.TaskKey { + continue + } + + // skip _id + if k == "_id" { + continue + } + + // append to columns + columns = append(columns, k) + } + + // order columns + sort.Strings(columns) + + return columns, nil +} + +func (svc *CsvService) getRowCells(columns []string, data bson.M) (cells []string) { + for _, c := range columns { + v, ok := data[c] + if !ok { + cells = append(cells, "") + continue + } + switch v.(type) { + case string: + cells = append(cells, v.(string)) + case time.Time: + cells = append(cells, v.(time.Time).Format("2006-01-02 15:04:05")) + case int: + cells = append(cells, strconv.Itoa(v.(int))) + case int32: + cells = append(cells, strconv.Itoa(int(v.(int32)))) + case int64: + cells = append(cells, strconv.FormatInt(v.(int64), 10)) + case float32: + cells = append(cells, strconv.FormatFloat(float64(v.(float32)), 'f', -1, 32)) + case float64: + cells = append(cells, strconv.FormatFloat(v.(float64), 'f', -1, 64)) + case bool: + cells = append(cells, strconv.FormatBool(v.(bool))) + case primitive.ObjectID: + cells = append(cells, v.(primitive.ObjectID).Hex()) + case primitive.DateTime: + cells = append(cells, v.(primitive.DateTime).Time().Format("2006-01-02 15:04:05")) + default: + cells = append(cells, fmt.Sprintf("%v", v)) + } + } + return cells +} + +func NewCsvService() (svc2 interfaces.ExportService) { + cache := ttlcache.NewCache() + cache.SetTTL(time.Minute * 5) + svc := &CsvService{ + cache: cache, + } + return svc +} + +var _csvService interfaces.ExportService + +func GetCsvService() (svc interfaces.ExportService) { + if _csvService == nil { + _csvService = NewCsvService() + } + return _csvService +} diff --git a/core/export/csv_service_test.go b/core/export/csv_service_test.go new file mode 100644 index 000000000..44e072aa4 --- /dev/null +++ b/core/export/csv_service_test.go @@ -0,0 +1,139 @@ +package export + +import ( + "encoding/csv" + "fmt" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "os" + "strconv" + "testing" + "time" +) + +func TestCsvService_Export(t *testing.T) { + // test data rows + var rows []interface{} + for i := 0; i < 10; i++ { + data := bson.M{ + "no": i, + "string_field": "test", + "int_field": 1, + "float_field": 1.1, + "bool_field": true, + "time_field": time.Now(), + "object_id_field": primitive.NewObjectID(), + } + rows = append(rows, data) + } + + // test mongo collection name + collectionName := "test_collection" + + // test mongo collection + collection := mongo.GetMongoCol(collectionName) + + // delete records of test mongo collection after test + t.Cleanup(func() { + _ = collection.Delete(bson.M{}) + }) + + // save test data rows to mongo collection + _, err := collection.InsertMany(rows) + require.Nil(t, err) + + // export service + csvSvc := NewCsvService() + + // export + exportId, err := csvSvc.Export(collectionName, collectionName, nil) + require.Nil(t, err) + + // get export + export, err := csvSvc.GetExport(exportId) + require.Nil(t, err) + require.NotNil(t, export) + require.Equal(t, exportId, export.GetId()) + require.NotNil(t, export.GetDownloadPath()) + + // wait for export to finish with timeout of 5 seconds + timeout := time.After(5 * time.Second) + finished := false + for { + if finished { + break + } + select { + case <-timeout: + t.Fatal("export timeout") + default: + if export.GetStatus() == constants.TaskStatusFinished { + finished = true + continue + } + time.Sleep(100 * time.Millisecond) + } + } + + // export file path + exportFilePath := export.GetDownloadPath() + require.FileExists(t, exportFilePath) + + // csv file + csvFile, err := os.Open(exportFilePath) + require.Nil(t, err) + defer csvFile.Close() + + // csv file reader + csvFileReader := csv.NewReader(csvFile) + + // csv file rows + csvFileRows, err := csvFileReader.ReadAll() + require.Nil(t, err) + require.Equal(t, len(rows), len(csvFileRows)-1) + + // csv file columns + csvFileColumns := csvFileRows[0] + + // iterate csv file records and compare with test data rows + for i, row := range rows { + // csv file record + csvFileRecord := csvFileRows[i+1] + + // iterate csv file columns and compare with test data row + for j, column := range csvFileColumns { + // csv file column value + csvFileColumnValue := csvFileRecord[j] + + // compare csv file column value with test data row + switch column { + case "no": + // convert int to string + stringValue := fmt.Sprintf("%d", row.(bson.M)["no"].(int)) + require.Equal(t, stringValue, csvFileColumnValue) + case "string_field": + require.Equal(t, row.(bson.M)["string_field"], csvFileColumnValue) + case "int_field": + // convert int to string + stringValue := fmt.Sprintf("%d", row.(bson.M)["int_field"]) + require.Equal(t, stringValue, csvFileColumnValue) + case "float_field": + // convert string to float + floatValue, err := strconv.ParseFloat(csvFileColumnValue, 64) + require.Nil(t, err) + require.Equal(t, row.(bson.M)["float_field"].(float64), floatValue) + case "bool_field": + // convert bool to string + stringValue := fmt.Sprintf("%t", row.(bson.M)["bool_field"]) + require.Equal(t, stringValue, csvFileColumnValue) + case "time_field": + // convert time to string + stringValue := row.(bson.M)["time_field"].(time.Time).Format("2006-01-02 15:04:05") + require.Equal(t, stringValue, csvFileColumnValue) + } + } + } +} diff --git a/core/export/json_service.go b/core/export/json_service.go new file mode 100644 index 000000000..d1eef9484 --- /dev/null +++ b/core/export/json_service.go @@ -0,0 +1,223 @@ +package export + +import ( + "context" + "encoding/json" + "errors" + "github.com/ReneKroon/ttlcache" + "github.com/apex/log" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/hashicorp/go-uuid" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "os" + "path" + "time" +) + +type JsonService struct { + cache *ttlcache.Cache +} + +func (svc *JsonService) GenerateId() (exportId string, err error) { + exportId, err = uuid.GenerateUUID() + if err != nil { + return "", trace.TraceError(err) + } + return exportId, nil +} + +func (svc *JsonService) Export(exportType, target string, filter interfaces.Filter) (exportId string, err error) { + // generate export id + exportId, err = svc.GenerateId() + if err != nil { + return "", err + } + + // export + export := &entity.Export{ + Id: exportId, + Type: exportType, + Target: target, + Filter: filter, + Status: constants.TaskStatusRunning, + StartTs: time.Now(), + FileName: svc.getFileName(exportId), + DownloadPath: svc.getDownloadPath(exportId), + Limit: 100, + } + + // save to cache + svc.cache.Set(exportId, export) + + // execute export + go svc.export(export) + + return exportId, nil +} + +func (svc *JsonService) GetExport(exportId string) (export interfaces.Export, err error) { + // get export from cache + res, ok := svc.cache.Get(exportId) + if !ok { + return nil, trace.TraceError(errors.New("export not found")) + } + export = res.(interfaces.Export) + return export, nil +} + +func (svc *JsonService) export(export *entity.Export) { + // check empty + if export.Target == "" { + err := errors.New("empty target") + export.Status = constants.TaskStatusError + export.EndTs = time.Now() + log.Errorf("export error (id: %s): %v", export.Id, err) + trace.PrintError(err) + svc.cache.Set(export.Id, export) + return + } + + // mongo collection + col := mongo.GetMongoCol(export.Target) + + // mongo query + query, err := utils.FilterToQuery(export.Filter) + if err != nil { + export.Status = constants.TaskStatusError + export.EndTs = time.Now() + log.Errorf("export error (id: %s): %v", export.Id, err) + trace.PrintError(err) + svc.cache.Set(export.Id, export) + return + } + + // mongo cursor + cur := col.Find(query, nil).GetCursor() + + // data + var jsonData []interface{} + + // iterate cursor + i := 0 + for { + // increment counter + i++ + + // check error + err := cur.Err() + if err != nil { + if err != mongo2.ErrNoDocuments { + // error + export.Status = constants.TaskStatusError + export.EndTs = time.Now() + log.Errorf("export error (id: %s): %v", export.Id, err) + trace.PrintError(err) + } else { + // no more data + export.Status = constants.TaskStatusFinished + export.EndTs = time.Now() + log.Infof("export finished (id: %s)", export.Id) + } + svc.cache.Set(export.Id, export) + return + } + + // has data + if !cur.Next(context.Background()) { + // no more data + export.Status = constants.TaskStatusFinished + export.EndTs = time.Now() + log.Infof("export finished (id: %s)", export.Id) + svc.cache.Set(export.Id, export) + break + } + + // convert raw data to entity + var data map[string]interface{} + err = cur.Decode(&data) + if err != nil { + // error + export.Status = constants.TaskStatusError + export.EndTs = time.Now() + log.Errorf("export error (id: %s): %v", export.Id, err) + trace.PrintError(err) + svc.cache.Set(export.Id, export) + return + } + + jsonData = append(jsonData, data) + } + + jsonBytes, err := json.Marshal(jsonData) + if err != nil { + // error + export.Status = constants.TaskStatusError + export.EndTs = time.Now() + log.Errorf("export error (id: %s): %v", export.Id, err) + trace.PrintError(err) + svc.cache.Set(export.Id, export) + return + } + jsonString := string(jsonBytes) + f := utils.OpenFile(export.DownloadPath) + _, err = f.WriteString(jsonString) + if err != nil { + // error + export.Status = constants.TaskStatusError + export.EndTs = time.Now() + log.Errorf("export error (id: %s): %v", export.Id, err) + trace.PrintError(err) + svc.cache.Set(export.Id, export) + return + } +} + +func (svc *JsonService) getExportDir() (dir string, err error) { + tempDir := os.TempDir() + exportDir := path.Join(tempDir, "export", "json") + if !utils.Exists(exportDir) { + err := os.MkdirAll(exportDir, 0755) + if err != nil { + return "", err + } + } + return exportDir, nil +} + +func (svc *JsonService) getFileName(exportId string) (fileName string) { + return exportId + "_" + time.Now().Format("20060102150405") + ".json" +} + +// getDownloadPath returns the download path for the export +// format: /export//_.csv +func (svc *JsonService) getDownloadPath(exportId string) (downloadPath string) { + exportDir, err := svc.getExportDir() + if err != nil { + return "" + } + downloadPath = path.Join(exportDir, svc.getFileName(exportId)) + return downloadPath +} + +func NewJsonService() (svc2 interfaces.ExportService) { + cache := ttlcache.NewCache() + cache.SetTTL(time.Minute * 5) + svc := &JsonService{ + cache: cache, + } + return svc +} + +var _jsonService interfaces.ExportService + +func GetJsonService() (svc interfaces.ExportService) { + if _jsonService == nil { + _jsonService = NewJsonService() + } + return _jsonService +} diff --git a/core/fs/default.go b/core/fs/default.go new file mode 100644 index 000000000..91fbb9591 --- /dev/null +++ b/core/fs/default.go @@ -0,0 +1,24 @@ +package fs + +import ( + "github.com/apex/log" + "github.com/mitchellh/go-homedir" + "github.com/spf13/viper" + "path/filepath" +) + +func init() { + rootDir, err := homedir.Dir() + if err != nil { + log.Warnf("cannot find home directory: %v", err) + return + } + DefaultWorkspacePath = filepath.Join(rootDir, "crawlab_workspace") + + workspacePath := viper.GetString("workspace") + if workspacePath == "" { + viper.Set("workspace", DefaultWorkspacePath) + } +} + +var DefaultWorkspacePath string diff --git a/core/fs/service_v2.go b/core/fs/service_v2.go new file mode 100644 index 000000000..6ca1f057c --- /dev/null +++ b/core/fs/service_v2.go @@ -0,0 +1,167 @@ +package fs + +import ( + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/utils" + "io" + "os" + "path/filepath" +) + +type ServiceV2 struct { + // settings + rootPath string + skipNames []string +} + +func (svc *ServiceV2) List(path string) (files []interfaces.FsFileInfo, err error) { + // Normalize the provided path + normPath := filepath.Clean(path) + if normPath == "." { + normPath = "" + } + fullPath := filepath.Join(svc.rootPath, normPath) + + // Temporary map to hold directory information and their children + dirMap := make(map[string]*entity.FsFileInfo) + + // Use filepath.Walk to recursively traverse directories + err = filepath.Walk(fullPath, func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(svc.rootPath, p) + if err != nil { + return err + } + + fi := &entity.FsFileInfo{ + Name: info.Name(), + Path: filepath.ToSlash(relPath), + FullPath: p, + Extension: filepath.Ext(p), + IsDir: info.IsDir(), + FileSize: info.Size(), + ModTime: info.ModTime(), + Mode: info.Mode(), + Children: nil, + } + + // Skip files/folders matching the pattern + for _, name := range svc.skipNames { + if fi.Name == name { + return nil + } + } + + if info.IsDir() { + dirMap[p] = fi + } + + if parentDir := filepath.Dir(p); parentDir != p && dirMap[parentDir] != nil { + dirMap[parentDir].Children = append(dirMap[parentDir].Children, fi) + } + + return nil + }) + + if rootInfo, ok := dirMap[fullPath]; ok { + for _, info := range rootInfo.GetChildren() { + files = append(files, info) + } + } + + return files, err +} + +func (svc *ServiceV2) GetFile(path string) (data []byte, err error) { + return os.ReadFile(filepath.Join(svc.rootPath, path)) +} + +func (svc *ServiceV2) GetFileInfo(path string) (file interfaces.FsFileInfo, err error) { + f, err := os.Stat(filepath.Join(svc.rootPath, path)) + if err != nil { + return nil, err + } + return &entity.FsFileInfo{ + Name: f.Name(), + Path: path, + FullPath: filepath.Join(svc.rootPath, path), + Extension: filepath.Ext(path), + IsDir: f.IsDir(), + FileSize: f.Size(), + ModTime: f.ModTime(), + Mode: f.Mode(), + Children: nil, + }, nil +} + +func (svc *ServiceV2) Save(path string, data []byte) (err error) { + // Create directories if not exist + dir := filepath.Dir(filepath.Join(svc.rootPath, path)) + if _, err := os.Stat(dir); os.IsNotExist(err) { + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + } + + // Write file + return os.WriteFile(filepath.Join(svc.rootPath, path), data, 0644) +} + +func (svc *ServiceV2) CreateDir(path string) (err error) { + return os.MkdirAll(filepath.Join(svc.rootPath, path), 0755) +} + +func (svc *ServiceV2) Rename(path, newPath string) (err error) { + oldPath := filepath.Join(svc.rootPath, path) + newFullPath := filepath.Join(svc.rootPath, newPath) + return os.Rename(oldPath, newFullPath) +} + +func (svc *ServiceV2) Delete(path string) (err error) { + fullPath := filepath.Join(svc.rootPath, path) + return os.RemoveAll(fullPath) +} + +func (svc *ServiceV2) Copy(path, newPath string) (err error) { + srcPath := filepath.Join(svc.rootPath, path) + destPath := filepath.Join(svc.rootPath, newPath) + + // Get source info + srcInfo, err := os.Stat(srcPath) + if err != nil { + return err + } + + // If source is file, copy it + if !srcInfo.IsDir() { + srcFile, err := os.Open(srcPath) + if err != nil { + return err + } + defer srcFile.Close() + + destFile, err := os.Create(destPath) + if err != nil { + return err + } + defer destFile.Close() + + _, err = io.Copy(destFile, srcFile) + + return err + } else { + // If source is directory, copy it recursively + return utils.CopyDir(srcPath, destPath) + } +} + +func NewFsServiceV2(path string) (svc interfaces.FsServiceV2) { + return &ServiceV2{ + rootPath: path, + skipNames: []string{".git"}, + } +} diff --git a/core/fs/service_v2_test.go b/core/fs/service_v2_test.go new file mode 100644 index 000000000..09191305d --- /dev/null +++ b/core/fs/service_v2_test.go @@ -0,0 +1,238 @@ +package fs + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestServiceV2_List(t *testing.T) { + rootDir, err := ioutil.TempDir("", "fsTest") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(rootDir) // clean up + + testDir := filepath.Join(rootDir, "dir") + os.Mkdir(testDir, 0755) + ioutil.WriteFile(filepath.Join(testDir, "file1.txt"), []byte("hello world"), 0644) + ioutil.WriteFile(filepath.Join(testDir, "file2.txt"), []byte("hello again"), 0644) + subDir := filepath.Join(testDir, "subdir") + os.Mkdir(subDir, 0755) + ioutil.WriteFile(filepath.Join(subDir, "file3.txt"), []byte("subdir file"), 0644) + os.Mkdir(filepath.Join(testDir, "empty"), 0755) // explicitly testing empty dir inclusion + + svc := NewFsServiceV2(rootDir) + + files, err := svc.List("dir") + if err != nil { + t.Errorf("Failed to list files: %v", err) + } + + // Assert correct number of items + assert.Len(t, files, 4) + // Use a map to verify presence and characteristics of files/directories to avoid order issues + items := make(map[string]bool) + for _, item := range files { + items[item.GetName()] = item.GetIsDir() + } + + _, file1Exists := items["file1.txt"] + _, file2Exists := items["file2.txt"] + _, subdirExists := items["subdir"] + _, emptyExists := items["empty"] + + assert.True(t, file1Exists) + assert.True(t, file2Exists) + assert.True(t, subdirExists) + assert.True(t, emptyExists) // Verify that the empty directory is included + + if subdirExists && len(files[2].GetChildren()) > 0 { + assert.Equal(t, "file3.txt", files[2].GetChildren()[0].GetName()) + } +} + +func TestServiceV2_GetFile(t *testing.T) { + rootDir, err := ioutil.TempDir("", "fsTest") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(rootDir) // clean up + + expectedContent := []byte("hello world") + ioutil.WriteFile(filepath.Join(rootDir, "file.txt"), expectedContent, 0644) + + svc := NewFsServiceV2(rootDir) + + content, err := svc.GetFile("file.txt") + if err != nil { + t.Errorf("Failed to get file: %v", err) + } + assert.Equal(t, expectedContent, content) +} + +func TestServiceV2_Delete(t *testing.T) { + rootDir, err := ioutil.TempDir("", "fsTest") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(rootDir) // clean up + + filePath := filepath.Join(rootDir, "file.txt") + ioutil.WriteFile(filePath, []byte("hello world"), 0644) + + svc := NewFsServiceV2(rootDir) + + // Delete the file + err = svc.Delete("file.txt") + if err != nil { + t.Errorf("Failed to delete file: %v", err) + } + + // Verify deletion + _, err = os.Stat(filePath) + assert.True(t, os.IsNotExist(err)) +} + +func TestServiceV2_CreateDir(t *testing.T) { + rootDir, err := ioutil.TempDir("", "fsTest") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(rootDir) // clean up + + svc := NewFsServiceV2(rootDir) + + // Create a new directory + err = svc.CreateDir("newDir") + if err != nil { + t.Errorf("Failed to create directory: %v", err) + } + + // Verify the directory was created + _, err = os.Stat(filepath.Join(rootDir, "newDir")) + assert.NoError(t, err) +} + +func TestServiceV2_Save(t *testing.T) { + rootDir, err := ioutil.TempDir("", "fsTest") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(rootDir) // clean up + + svc := NewFsServiceV2(rootDir) + + // Save a new file + err = svc.Save("newFile.txt", []byte("Hello, world!")) + if err != nil { + t.Errorf("Failed to save file: %v", err) + } + + // Verify the file was saved + data, err := ioutil.ReadFile(filepath.Join(rootDir, "newFile.txt")) + assert.NoError(t, err) + assert.Equal(t, "Hello, world!", string(data)) +} + +func TestServiceV2_Rename(t *testing.T) { + rootDir, err := ioutil.TempDir("", "fsTest") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(rootDir) // clean up + + svc := NewFsServiceV2(rootDir) + + // Create a file to rename + ioutil.WriteFile(filepath.Join(rootDir, "oldName.txt"), []byte("Hello, world!"), 0644) + + // Rename the file + err = svc.Rename("oldName.txt", "newName.txt") + if err != nil { + t.Errorf("Failed to rename file: %v", err) + } + + // Verify the file was renamed + _, err = os.Stat(filepath.Join(rootDir, "newName.txt")) + assert.NoError(t, err) +} + +func TestServiceV2_RenameDir(t *testing.T) { + rootDir, err := ioutil.TempDir("", "fsTest") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(rootDir) // clean up + + svc := NewFsServiceV2(rootDir) + + // Create a directory to rename + os.Mkdir(filepath.Join(rootDir, "oldName"), 0755) + + // Rename the directory + err = svc.Rename("oldName", "newName") + if err != nil { + t.Errorf("Failed to rename directory: %v", err) + } + + // Verify the directory was renamed + _, err = os.Stat(filepath.Join(rootDir, "newName")) + assert.NoError(t, err) +} + +func TestServiceV2_Copy(t *testing.T) { + rootDir, err := ioutil.TempDir("", "fsTest") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(rootDir) // clean up + + svc := NewFsServiceV2(rootDir) + + // Create a file to copy + ioutil.WriteFile(filepath.Join(rootDir, "source.txt"), []byte("Hello, world!"), 0644) + + // Copy the file + err = svc.Copy("source.txt", "copy.txt") + if err != nil { + t.Errorf("Failed to copy file: %v", err) + } + + // Verify the file was copied + data, err := ioutil.ReadFile(filepath.Join(rootDir, "copy.txt")) + assert.NoError(t, err) + assert.Equal(t, "Hello, world!", string(data)) +} + +func TestServiceV2_CopyDir(t *testing.T) { + rootDir, err := ioutil.TempDir("", "fsTest") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(rootDir) // clean up + + svc := NewFsServiceV2(rootDir) + + // Create a directory to copy + os.Mkdir(filepath.Join(rootDir, "sourceDir"), 0755) + ioutil.WriteFile(filepath.Join(rootDir, "sourceDir", "file.txt"), []byte("Hello, world!"), 0644) + + // Copy the directory + err = svc.Copy("sourceDir", "copyDir") + if err != nil { + t.Errorf("Failed to copy directory: %v", err) + } + + // Verify the directory was copied + _, err = os.Stat(filepath.Join(rootDir, "copyDir")) + assert.NoError(t, err) + + // Verify the file inside the directory was copied + data, err := ioutil.ReadFile(filepath.Join(rootDir, "copyDir", "file.txt")) + assert.NoError(t, err) + assert.Equal(t, "Hello, world!", string(data)) +} diff --git a/core/go.mod b/core/go.mod new file mode 100644 index 000000000..a7b1bed53 --- /dev/null +++ b/core/go.mod @@ -0,0 +1,184 @@ +module github.com/crawlab-team/crawlab/core + +go 1.22 + +replace ( + github.com/crawlab-team/crawlab/grpc latest => ../grpc latest +) + +require ( + github.com/ReneKroon/ttlcache v1.7.0 + github.com/apex/log v1.9.0 + github.com/cenkalti/backoff/v4 v4.1.0 + github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 + github.com/crawlab-team/crawlab-fs v0.6.3-2 + github.com/crawlab-team/crawlab/grpc latest + github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185 + github.com/crawlab-team/go-trace v0.1.1 + github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e + github.com/elastic/go-elasticsearch/v8 v8.7.0 + github.com/emirpasic/gods v1.18.1 + github.com/fsnotify/fsnotify v1.5.1 + github.com/gavv/httpexpect/v2 v2.16.0 + github.com/gin-gonic/gin v1.9.1 + github.com/go-git/go-git/v5 v5.12.0 + github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/google/uuid v1.6.0 + github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 + github.com/hashicorp/go-uuid v1.0.1 + github.com/imroc/req v0.3.0 + github.com/matcornic/hermes/v2 v2.1.0 + github.com/mitchellh/go-homedir v1.1.0 + github.com/pkg/errors v0.9.1 + github.com/robfig/cron/v3 v3.0.0 + github.com/segmentio/kafka-go v0.4.39 + github.com/shirou/gopsutil v3.21.11+incompatible + github.com/sirupsen/logrus v1.9.0 + github.com/smartystreets/goconvey v1.6.4 + github.com/spf13/cobra v1.3.0 + github.com/spf13/viper v1.10.0 + github.com/stretchr/testify v1.9.0 + github.com/thoas/go-funk v0.9.1 + github.com/upper/db/v4 v4.6.0 + go.mongodb.org/mongo-driver v1.15.0 + go.uber.org/dig v1.10.0 + google.golang.org/grpc v1.64.0 + gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df +) + +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect +) + +require ( + cloud.google.com/go v0.99.0 // indirect + github.com/Masterminds/semver v1.4.2 // indirect + github.com/Masterminds/sprig v2.16.0+incompatible // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/ProtonMail/go-crypto v1.0.0 // indirect + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect + github.com/ajg/form v1.5.1 // indirect + github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/aokoli/goutils v1.0.1 // indirect + github.com/bytedance/sonic v1.9.1 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/cloudflare/circl v1.3.7 // indirect + github.com/crawlab-team/goseaweedfs v0.6.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/denisenkom/go-mssqldb v0.11.0 // indirect + github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect + github.com/fatih/color v1.15.0 // indirect + github.com/fatih/structs v1.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect + github.com/gorilla/css v1.0.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/huandu/xstrings v1.2.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect + github.com/imkira/go-interpol v1.1.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.11.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.2.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.10.0 // indirect + github.com/jackc/pgx/v4 v4.15.0 // indirect + github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/compress v1.15.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/leodido/go-urn v1.2.4 // indirect + github.com/lib/pq v1.10.4 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.3 // indirect + github.com/mattn/go-sqlite3 v1.14.9 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/olekukonko/tablewriter v0.0.1 // indirect + github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sanity-io/litter v1.5.5 // indirect + github.com/segmentio/fasthash v1.0.3 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/skeema/knownhosts v1.2.2 // indirect + github.com/smartystreets/assertions v1.0.0 // indirect + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + github.com/tklauser/go-sysconf v0.3.9 // indirect + github.com/tklauser/numcpus v0.3.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.34.0 // indirect + github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 // indirect + github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + github.com/yudai/gojsondiff v1.0.0 // indirect + github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect + github.com/ztrue/tracerr v0.4.0 // indirect + golang.org/x/arch v0.3.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.13.0 // indirect + google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect + gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/sourcemap.v1 v1.0.5 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + moul.io/http2curl/v2 v2.3.0 // indirect +) diff --git a/core/go.sum b/core/go.sum new file mode 100644 index 000000000..900342218 --- /dev/null +++ b/core/go.sum @@ -0,0 +1,1420 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= +cloud.google.com/go v0.99.0 h1:y/cM2iqGgGi5D5DQZl6D9STN/3dR/Vx5Mp8s752oJTY= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY= +github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/ReneKroon/ttlcache v1.7.0 h1:8BkjFfrzVFXyrqnMtezAaJ6AHPSsVV10m6w28N/Fgkk= +github.com/ReneKroon/ttlcache v1.7.0/go.mod h1:8BGGzdumrIjWxdRx8zpK6L3oGMWvIXdvB2GD1cfvd+I= +github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4= +github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w= +github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= +github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= +github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= +github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= +github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= +github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= +github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= +github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= +github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crawlab-team/crawlab-db v0.6.0-1/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= +github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 h1:1CXWC3lYcVWcgPRc3PNKzZ3fcfX5WZ/V8xwzHEMUFHQ= +github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= +github.com/crawlab-team/crawlab-fs v0.6.3-2 h1:GAovTF1R1PLoQgj+0F1GpePrlp1k7RtW/jK8p0L9HhA= +github.com/crawlab-team/crawlab-fs v0.6.3-2/go.mod h1:Nob0uFr82IPbkk6LEYG0BAB2NgJ3PKoNVhtcbf5fLf0= +github.com/crawlab-team/crawlab/grpc v0.6.4-0.20240614065204-c753fc33f7ca h1:7LOFAWCIK+153f5J437xXhlI1miUQ3mrm3OXGq2/m3U= +github.com/crawlab-team/crawlab/grpc v0.6.4-0.20240614065204-c753fc33f7ca/go.mod h1:j5FaFuWfIxHbmFXedFaUZHO2DpLqS3QZ0x7W3APN0fQ= +github.com/crawlab-team/crawlab/grpc v0.6.4-0.20240614072247-7558f2150cee h1:jlT64UPWbf5Xny3jOzKE35od1F+OuGK+y8PTNomyxiM= +github.com/crawlab-team/crawlab/grpc v0.6.4-0.20240614072247-7558f2150cee/go.mod h1:j5FaFuWfIxHbmFXedFaUZHO2DpLqS3QZ0x7W3APN0fQ= +github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185 h1:A/XSUuGgGMn+z+lFd2ye2ClgIKhDZYUerhOL5jePQhU= +github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185/go.mod h1:YHMYUEoSqfXUZHsWW/M/DaLh/zOpRtiElaRWcrGyv/I= +github.com/crawlab-team/go-trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= +github.com/crawlab-team/go-trace v0.1.1 h1:AecgAOld+ZrSVvujyhK3zoaOmViGKHSCT8/weJ7adB8= +github.com/crawlab-team/go-trace v0.1.1/go.mod h1:4U+pWgLhRuD3pbXHonwcaHcW+y8AUqyOfKoZnvKwCug= +github.com/crawlab-team/goseaweedfs v0.6.3 h1:f96H2QCLrZpof9na1mhIKouKrv8p32XRUyouSVm4YHU= +github.com/crawlab-team/goseaweedfs v0.6.3/go.mod h1:Anqw9QErRJpTeVAVdcSfzprGzUz7OW4MVCHLJjKeO1U= +github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e h1:Gwg9kKNZUAI4bSssomlzXCN01Q3MapgwQOCeOxGX/NU= +github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e/go.mod h1:FImmp7V0VcIdTRM68F3PQUqewzuShvUjYBhAHRjD1Aw= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI= +github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/elastic-transport-go/v8 v8.2.0 h1:hkK5IIs/15mpSXzd5THWVlWTKJyMw6cbCWM3T/B2S5E= +github.com/elastic/elastic-transport-go/v8 v8.2.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= +github.com/elastic/go-elasticsearch/v8 v8.7.0 h1:ZvbT1YHppBC0QxGnMmaDUxoDa26clwhRaB3Gp5E3UcY= +github.com/elastic/go-elasticsearch/v8 v8.7.0/go.mod h1:lVb8SvJV8McVkdswpL8YR5QKIkhlWaoSq60YpHilOLI= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gavv/httpexpect/v2 v2.16.0 h1:Ty2favARiTYTOkCRZGX7ojXXjGyNAIohM1lZ3vqaEwI= +github.com/gavv/httpexpect/v2 v2.16.0/go.mod h1:uJLaO+hQ25ukBJtQi750PsztObHybNllN+t+MbbW8PY= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= +github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df/go.mod h1:GJr+FCSXshIwgHBtLglIg9M2l2kQSi6QjVAngtzI08Y= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= +github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f h1:7LYC+Yfkj3CTRcShK0KOL/w6iTiKyqqBA9a41Wnggw8= +github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= +github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= +github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U= +github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= +github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= +github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ= +github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/matcornic/hermes/v2 v2.1.0 h1:9TDYFBPFv6mcXanaDmRDEp/RTWj0dTTi+LpFnnnfNWc= +github.com/matcornic/hermes/v2 v2.1.0/go.mod h1:2+ziJeoyRfaLiATIL8VZ7f9hpzH4oDHqTmn0bhrsgVI= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= +github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= +github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= +github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olivere/elastic/v7 v7.0.15/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 h1:ewTtJ72GFy2e0e8uyiDwMG3pKCS5mBh+hdSTYsPKEP8= +github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= +github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= +github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= +github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= +github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= +github.com/segmentio/kafka-go v0.4.39 h1:75smaomhvkYRwtuOwqLsdhgCG30B82NsbdkdDfFbvrw= +github.com/segmentio/kafka-go v0.4.39/go.mod h1:T0MLgygYvmqmBvC+s8aCcbVNfJN4znVne5j0Pzowp/Q= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= +github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= +github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= +github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= +github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= +github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= +github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= +github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= +github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= +github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= +github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= +github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= +github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= +github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/upper/db/v4 v4.6.0 h1:0VmASnqrl/XN8Ehoq++HBgZ4zRD5j3GXygW8FhP0C5I= +github.com/upper/db/v4 v4.6.0/go.mod h1:2mnRcPf+RcCXmVcD+o04LYlyu3UuF7ubamJia7CkN6s= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= +github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 h1:L0rPdfzq43+NV8rfIx2kA4iSSLRj2jN5ijYHoeXRwvQ= +github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04/go.mod h1:tcnB1voG49QhCrwq1W0w5hhGasvOg+VQp9i9H1rCM1w= +github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe h1:9YnI5plmy+ad6BM+JCLJb2ZV7/TNiE5l7SNKfumYKgc= +github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe/go.mod h1:JTFJA/t820uFDoyPpErFQ3rb3amdZoPtxcKervG0OE4= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= +github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= +github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= +github.com/ztrue/tracerr v0.4.0 h1:vT5PFxwIGs7rCg9ZgJ/y0NmOpJkPCPFK8x0vVIYzd04= +github.com/ztrue/tracerr v0.4.0/go.mod h1:PaFfYlas0DfmXNpo7Eay4MFhZUONqvXM+T2HyGPpngk= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= +go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= +go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY= +go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= +go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= +go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029175232-7e6ffbd03851/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190225065934-cc5685c2db12/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= +gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +modernc.org/b v1.0.2/go.mod h1:fVGfCIzkZw5RsuF2A2WHbJmY7FiMIq30nP4s52uWsoY= +modernc.org/db v1.0.3/go.mod h1:L4ltUg8tu2pkSJk+fKaRrXs/3EdW79ZKYQ5PfVDT53U= +modernc.org/file v1.0.3/go.mod h1:CNj/pwOfCtCbqiHcXDUlHBB2vWrzdaDCWdcnjtS1+XY= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= +modernc.org/golex v1.0.1/go.mod h1:QCA53QtsT1NdGkaZZkF5ezFwk4IXh4BGNafAARTC254= +modernc.org/internal v1.0.0/go.mod h1:VUD/+JAkhCpvkUitlEOnhpVxCgsBI90oTzSCRcqQVSM= +modernc.org/internal v1.0.2/go.mod h1:bycJAcev709ZU/47nil584PeBD+kbu8nv61ozeMso9E= +modernc.org/lex v1.0.0/go.mod h1:G6rxMTy3cH2iA0iXL/HRRv4Znu8MK4higxph/lE7ypk= +modernc.org/lexer v1.0.0/go.mod h1:F/Dld0YKYdZCLQ7bD0USbWL4YKCyTDRDHiDTOs0q0vk= +modernc.org/lldb v1.0.2/go.mod h1:ovbKqyzA9H/iPwHkAOH0qJbIQVT9rlijecenxDwVUi0= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/ql v1.4.0/go.mod h1:q4c29Bgdx+iAtxx47ODW5Xo2X0PDkjSCK9NdQl6KFxc= +modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/zappy v1.0.3/go.mod h1:w/Akq8ipfols/xZJdR5IYiQNOqC80qz2mVvsEwEbkiI= +moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs= +moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/core/grpc/client/client.go b/core/grpc/client/client.go new file mode 100644 index 000000000..15a6c5e34 --- /dev/null +++ b/core/grpc/client/client.go @@ -0,0 +1,420 @@ +package client + +import ( + "context" + "encoding/json" + "github.com/apex/log" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/grpc/middlewares" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/utils" + grpc2 "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "github.com/spf13/viper" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "io" + "time" +) + +type Client struct { + // dependencies + nodeCfgSvc interfaces.NodeConfigService + + // settings + cfgPath string + address interfaces.Address + timeout time.Duration + subscribeType string + handleMessage bool + + // internals + conn *grpc.ClientConn + stream grpc2.NodeService_SubscribeClient + msgCh chan *grpc2.StreamMessage + err error + + // grpc clients + ModelDelegateClient grpc2.ModelDelegateClient + ModelBaseServiceClient grpc2.ModelBaseServiceClient + NodeClient grpc2.NodeServiceClient + TaskClient grpc2.TaskServiceClient + MessageClient grpc2.MessageServiceClient +} + +func (c *Client) Init() (err error) { + // do nothing + return nil +} + +func (c *Client) Start() (err error) { + // connect + if err := c.connect(); err != nil { + return err + } + + // register rpc services + if err := c.Register(); err != nil { + return err + } + + // subscribe + if err := c.subscribe(); err != nil { + return err + } + + // handle stream message + if c.handleMessage { + go c.handleStreamMessage() + } + + return nil +} + +func (c *Client) Stop() (err error) { + // skip if connection is nil + if c.conn == nil { + return nil + } + + // grpc server address + address := c.address.String() + + // unsubscribe + if err := c.unsubscribe(); err != nil { + return err + } + log.Infof("grpc client unsubscribed from %s", address) + + // close connection + if err := c.conn.Close(); err != nil { + return err + } + log.Infof("grpc client disconnected from %s", address) + + return nil +} + +func (c *Client) Register() (err error) { + // model delegate + c.ModelDelegateClient = grpc2.NewModelDelegateClient(c.conn) + + // model base service + c.ModelBaseServiceClient = grpc2.NewModelBaseServiceClient(c.conn) + + // node + c.NodeClient = grpc2.NewNodeServiceClient(c.conn) + + // task + c.TaskClient = grpc2.NewTaskServiceClient(c.conn) + + // message + c.MessageClient = grpc2.NewMessageServiceClient(c.conn) + + // log + log.Infof("[GrpcClient] grpc client registered client services") + log.Debugf("[GrpcClient] ModelDelegateClient: %v", c.ModelDelegateClient) + log.Debugf("[GrpcClient] ModelBaseServiceClient: %v", c.ModelBaseServiceClient) + log.Debugf("[GrpcClient] NodeClient: %v", c.NodeClient) + log.Debugf("[GrpcClient] TaskClient: %v", c.TaskClient) + log.Debugf("[GrpcClient] MessageClient: %v", c.MessageClient) + + return nil +} + +func (c *Client) GetModelDelegateClient() (res grpc2.ModelDelegateClient) { + return c.ModelDelegateClient +} + +func (c *Client) GetModelBaseServiceClient() (res grpc2.ModelBaseServiceClient) { + return c.ModelBaseServiceClient +} + +func (c *Client) GetNodeClient() grpc2.NodeServiceClient { + return c.NodeClient +} + +func (c *Client) GetTaskClient() grpc2.TaskServiceClient { + return c.TaskClient +} + +func (c *Client) GetMessageClient() grpc2.MessageServiceClient { + return c.MessageClient +} + +func (c *Client) SetAddress(address interfaces.Address) { + c.address = address +} + +func (c *Client) SetTimeout(timeout time.Duration) { + c.timeout = timeout +} + +func (c *Client) SetSubscribeType(value string) { + c.subscribeType = value +} + +func (c *Client) SetHandleMessage(handleMessage bool) { + c.handleMessage = handleMessage +} + +func (c *Client) Context() (ctx context.Context, cancel context.CancelFunc) { + return context.WithTimeout(context.Background(), c.timeout) +} + +func (c *Client) NewRequest(d interface{}) (req *grpc2.Request) { + return &grpc2.Request{ + NodeKey: c.nodeCfgSvc.GetNodeKey(), + Data: c.getRequestData(d), + } +} + +func (c *Client) GetConfigPath() (path string) { + return c.cfgPath +} + +func (c *Client) SetConfigPath(path string) { + c.cfgPath = path +} + +func (c *Client) NewModelBaseServiceRequest(id interfaces.ModelId, params interfaces.GrpcBaseServiceParams) (req *grpc2.Request, err error) { + data, err := json.Marshal(params) + if err != nil { + return nil, trace.TraceError(err) + } + msg := &entity.GrpcBaseServiceMessage{ + ModelId: id, + Data: data, + } + return c.NewRequest(msg), nil +} + +func (c *Client) GetMessageChannel() (msgCh chan *grpc2.StreamMessage) { + return c.msgCh +} + +func (c *Client) Restart() (err error) { + if c.needRestart() { + return c.Start() + } + return nil +} + +func (c *Client) IsStarted() (res bool) { + return c.conn != nil +} + +func (c *Client) IsClosed() (res bool) { + if c.conn != nil { + return c.conn.GetState() == connectivity.Shutdown + } + return false +} + +func (c *Client) Err() (err error) { + return c.err +} + +func (c *Client) GetStream() (stream grpc2.NodeService_SubscribeClient) { + return c.stream +} + +func (c *Client) connect() (err error) { + return backoff.RetryNotify(c._connect, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client connect")) +} + +func (c *Client) _connect() (err error) { + // grpc server address + address := c.address.String() + + // timeout context + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + + // connection + // TODO: configure dial options + var opts []grpc.DialOption + opts = append(opts, grpc.WithInsecure()) + opts = append(opts, grpc.WithBlock()) + opts = append(opts, grpc.WithChainUnaryInterceptor(middlewares.GetAuthTokenUnaryChainInterceptor(c.nodeCfgSvc))) + opts = append(opts, grpc.WithChainStreamInterceptor(middlewares.GetAuthTokenStreamChainInterceptor(c.nodeCfgSvc))) + c.conn, err = grpc.DialContext(ctx, address, opts...) + if err != nil { + _ = trace.TraceError(err) + return errors.ErrorGrpcClientFailedToStart + } + log.Infof("[GrpcClient] grpc client connected to %s", address) + + return nil +} + +func (c *Client) subscribe() (err error) { + var op func() error + switch c.subscribeType { + case constants.GrpcSubscribeTypeNode: + op = c._subscribeNode + default: + return errors.ErrorGrpcInvalidType + } + return backoff.RetryNotify(op, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client subscribe")) +} + +func (c *Client) _subscribeNode() (err error) { + req := c.NewRequest(&entity.NodeInfo{ + Key: c.nodeCfgSvc.GetNodeKey(), + IsMaster: false, + }) + c.stream, err = c.GetNodeClient().Subscribe(context.Background(), req) + if err != nil { + return trace.TraceError(err) + } + + // log + log.Infof("[GrpcClient] grpc client subscribed to remote server") + + return nil +} + +func (c *Client) unsubscribe() (err error) { + req := c.NewRequest(&entity.NodeInfo{ + Key: c.nodeCfgSvc.GetNodeKey(), + IsMaster: false, + }) + if _, err = c.GetNodeClient().Unsubscribe(context.Background(), req); err != nil { + return trace.TraceError(err) + } + return nil +} + +func (c *Client) handleStreamMessage() { + log.Infof("[GrpcClient] start handling stream message...") + for { + // resubscribe if stream is set to nil + if c.stream == nil { + if err := backoff.RetryNotify(c.subscribe, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client subscribe")); err != nil { + log.Errorf("subscribe") + return + } + } + + // receive stream message + msg, err := c.stream.Recv() + log.Debugf("[GrpcClient] received message: %v", msg) + if err != nil { + // set error + c.err = err + + // end + if err == io.EOF { + log.Infof("[GrpcClient] received EOF signal, disconnecting") + return + } + + // connection closed + if c.IsClosed() { + return + } + + // error + trace.PrintError(err) + c.stream = nil + time.Sleep(1 * time.Second) + continue + } + + // send stream message to channel + c.msgCh <- msg + + // reset error + c.err = nil + } +} + +func (c *Client) needRestart() bool { + switch c.conn.GetState() { + case connectivity.Shutdown, connectivity.TransientFailure: + return true + case connectivity.Idle, connectivity.Connecting, connectivity.Ready: + return false + default: + return false + } +} + +func (c *Client) getRequestData(d interface{}) (data []byte) { + if d == nil { + return data + } + switch d.(type) { + case []byte: + data = d.([]byte) + default: + var err error + data, err = json.Marshal(d) + if err != nil { + panic(err) + } + } + return data +} + +func NewClient() (res interfaces.GrpcClient, err error) { + // client + client := &Client{ + address: entity.NewAddress(&entity.AddressOptions{ + Host: constants.DefaultGrpcClientRemoteHost, + Port: constants.DefaultGrpcClientRemotePort, + }), + timeout: 10 * time.Second, + msgCh: make(chan *grpc2.StreamMessage), + subscribeType: constants.GrpcSubscribeTypeNode, + handleMessage: true, + } + + if viper.GetString("grpc.address") != "" { + client.address, err = entity.NewAddressFromString(viper.GetString("grpc.address")) + if err != nil { + return nil, trace.TraceError(err) + } + } + + // dependency injection + if err := container.GetContainer().Invoke(func(nodeCfgSvc interfaces.NodeConfigService) { + client.nodeCfgSvc = nodeCfgSvc + }); err != nil { + return nil, err + } + + // init + if err := client.Init(); err != nil { + return nil, err + } + + return client, nil +} + +var _client interfaces.GrpcClient + +func GetClient() (c interfaces.GrpcClient, err error) { + if _client != nil { + return _client, nil + } + _client, err = createClient() + if err != nil { + return nil, err + } + return _client, nil +} + +func createClient() (client2 interfaces.GrpcClient, err error) { + if err := container.GetContainer().Invoke(func(client interfaces.GrpcClient) { + client2 = client + }); err != nil { + return nil, trace.TraceError(err) + } + return client2, nil +} diff --git a/core/grpc/client/client_v2.go b/core/grpc/client/client_v2.go new file mode 100644 index 000000000..8a55a3049 --- /dev/null +++ b/core/grpc/client/client_v2.go @@ -0,0 +1,287 @@ +package client + +import ( + "context" + "encoding/json" + "github.com/apex/log" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/grpc/middlewares" + "github.com/crawlab-team/crawlab/core/interfaces" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/core/utils" + grpc2 "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "github.com/spf13/viper" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "io" + "time" +) + +type GrpcClientV2 struct { + // dependencies + nodeCfgSvc interfaces.NodeConfigService + + // settings + address interfaces.Address + timeout time.Duration + + // internals + conn *grpc.ClientConn + stream grpc2.NodeService_SubscribeClient + msgCh chan *grpc2.StreamMessage + err error + + // clients + NodeClient grpc2.NodeServiceClient + TaskClient grpc2.TaskServiceClient + ModelBaseServiceV2Client grpc2.ModelBaseServiceV2Client + DependenciesClient grpc2.DependencyServiceV2Client +} + +func (c *GrpcClientV2) Init() (err error) { + return nil +} + +func (c *GrpcClientV2) Start() (err error) { + // connect + if err := c.connect(); err != nil { + return err + } + + // register rpc services + c.Register() + + // subscribe + if err := c.subscribe(); err != nil { + return err + } + + // handle stream message + go c.handleStreamMessage() + + return nil +} + +func (c *GrpcClientV2) Stop() (err error) { + // skip if connection is nil + if c.conn == nil { + return nil + } + + // grpc server address + address := c.address.String() + + // unsubscribe + if err := c.unsubscribe(); err != nil { + return err + } + log.Infof("grpc client unsubscribed from %s", address) + + // close connection + if err := c.conn.Close(); err != nil { + return err + } + log.Infof("grpc client disconnected from %s", address) + + return nil +} + +func (c *GrpcClientV2) Register() { + c.NodeClient = grpc2.NewNodeServiceClient(c.conn) + c.ModelBaseServiceV2Client = grpc2.NewModelBaseServiceV2Client(c.conn) + c.TaskClient = grpc2.NewTaskServiceClient(c.conn) + c.DependenciesClient = grpc2.NewDependencyServiceV2Client(c.conn) + + // log + log.Infof("[GrpcClient] grpc client registered client services") + log.Debugf("[GrpcClient] NodeClient: %v", c.NodeClient) + log.Debugf("[GrpcClient] ModelBaseServiceV2Client: %v", c.ModelBaseServiceV2Client) +} + +func (c *GrpcClientV2) Context() (ctx context.Context, cancel context.CancelFunc) { + return context.WithTimeout(context.Background(), c.timeout) +} + +func (c *GrpcClientV2) NewRequest(d interface{}) (req *grpc2.Request) { + return &grpc2.Request{ + NodeKey: c.nodeCfgSvc.GetNodeKey(), + Data: c.getRequestData(d), + } +} + +func (c *GrpcClientV2) IsStarted() (res bool) { + return c.conn != nil +} + +func (c *GrpcClientV2) IsClosed() (res bool) { + if c.conn != nil { + return c.conn.GetState() == connectivity.Shutdown + } + return false +} + +func (c *GrpcClientV2) GetMessageChannel() (msgCh chan *grpc2.StreamMessage) { + return c.msgCh +} + +func (c *GrpcClientV2) getRequestData(d interface{}) (data []byte) { + if d == nil { + return data + } + switch d.(type) { + case []byte: + data = d.([]byte) + default: + var err error + data, err = json.Marshal(d) + if err != nil { + panic(err) + } + } + return data +} + +func (c *GrpcClientV2) unsubscribe() (err error) { + req := c.NewRequest(&entity.NodeInfo{ + Key: c.nodeCfgSvc.GetNodeKey(), + IsMaster: false, + }) + if _, err = c.NodeClient.Unsubscribe(context.Background(), req); err != nil { + return trace.TraceError(err) + } + return nil +} + +func (c *GrpcClientV2) connect() (err error) { + op := func() error { + // grpc server address + address := c.address.String() + + // timeout context + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + + // connection + // TODO: configure dial options + var opts []grpc.DialOption + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + opts = append(opts, grpc.WithBlock()) + opts = append(opts, grpc.WithChainUnaryInterceptor(middlewares.GetAuthTokenUnaryChainInterceptor(c.nodeCfgSvc))) + opts = append(opts, grpc.WithChainStreamInterceptor(middlewares.GetAuthTokenStreamChainInterceptor(c.nodeCfgSvc))) + c.conn, err = grpc.DialContext(ctx, address, opts...) + if err != nil { + _ = trace.TraceError(err) + return errors.ErrorGrpcClientFailedToStart + } + log.Infof("[GrpcClient] grpc client connected to %s", address) + + return nil + } + return backoff.RetryNotify(op, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client connect")) +} + +func (c *GrpcClientV2) subscribe() (err error) { + op := func() error { + req := c.NewRequest(&entity.NodeInfo{ + Key: c.nodeCfgSvc.GetNodeKey(), + IsMaster: false, + }) + c.stream, err = c.NodeClient.Subscribe(context.Background(), req) + if err != nil { + return trace.TraceError(err) + } + + // log + log.Infof("[GrpcClient] grpc client subscribed to remote server") + + return nil + } + return backoff.RetryNotify(op, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client subscribe")) +} + +func (c *GrpcClientV2) handleStreamMessage() { + log.Infof("[GrpcClient] start handling stream message...") + for { + // resubscribe if stream is set to nil + if c.stream == nil { + if err := backoff.RetryNotify(c.subscribe, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client subscribe")); err != nil { + log.Errorf("subscribe") + return + } + } + + // receive stream message + msg, err := c.stream.Recv() + log.Debugf("[GrpcClient] received message: %v", msg) + if err != nil { + // set error + c.err = err + + // end + if err == io.EOF { + log.Infof("[GrpcClient] received EOF signal, disconnecting") + return + } + + // connection closed + if c.IsClosed() { + return + } + + // error + trace.PrintError(err) + c.stream = nil + time.Sleep(1 * time.Second) + continue + } + + // send stream message to channel + c.msgCh <- msg + + // reset error + c.err = nil + } +} + +func NewGrpcClientV2() (c *GrpcClientV2, err error) { + client := &GrpcClientV2{ + address: entity.NewAddress(&entity.AddressOptions{ + Host: constants.DefaultGrpcClientRemoteHost, + Port: constants.DefaultGrpcClientRemotePort, + }), + timeout: 10 * time.Second, + msgCh: make(chan *grpc2.StreamMessage), + } + client.nodeCfgSvc = nodeconfig.GetNodeConfigService() + + if viper.GetString("grpc.address") != "" { + client.address, err = entity.NewAddressFromString(viper.GetString("grpc.address")) + if err != nil { + return nil, trace.TraceError(err) + } + } + + if err := client.Init(); err != nil { + return nil, err + } + + return client, nil +} + +var _clientV2 *GrpcClientV2 + +func GetGrpcClientV2() (client *GrpcClientV2, err error) { + if _clientV2 != nil { + return _clientV2, nil + } + _clientV2, err = NewGrpcClientV2() + if err != nil { + return nil, err + } + return _clientV2, nil +} diff --git a/core/grpc/client/options.go b/core/grpc/client/options.go new file mode 100644 index 000000000..8f69e3a98 --- /dev/null +++ b/core/grpc/client/options.go @@ -0,0 +1,51 @@ +package client + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type Option func(client interfaces.GrpcClient) + +func WithConfigPath(path string) Option { + return func(c interfaces.GrpcClient) { + c.SetConfigPath(path) + } +} + +func WithAddress(address interfaces.Address) Option { + return func(c interfaces.GrpcClient) { + c.SetAddress(address) + } +} + +func WithTimeout(timeout time.Duration) Option { + return func(c interfaces.GrpcClient) { + } +} + +func WithSubscribeType(subscribeType string) Option { + return func(c interfaces.GrpcClient) { + c.SetSubscribeType(subscribeType) + } +} + +func WithHandleMessage(handleMessage bool) Option { + return func(c interfaces.GrpcClient) { + c.SetHandleMessage(handleMessage) + } +} + +type PoolOption func(p interfaces.GrpcClientPool) + +func WithPoolConfigPath(path string) PoolOption { + return func(c interfaces.GrpcClientPool) { + c.SetConfigPath(path) + } +} + +func WithPoolSize(size int) PoolOption { + return func(c interfaces.GrpcClientPool) { + c.SetSize(size) + } +} diff --git a/core/grpc/client/pool.go b/core/grpc/client/pool.go new file mode 100644 index 000000000..5ecd8dadb --- /dev/null +++ b/core/grpc/client/pool.go @@ -0,0 +1,88 @@ +package client + +import ( + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/go-trace" + "github.com/emirpasic/gods/lists/arraylist" + "math/rand" +) + +type Pool struct { + // settings + size int + cfgPath string + + // internals + clients *arraylist.List +} + +func (p *Pool) GetConfigPath() (path string) { + return p.cfgPath +} + +func (p *Pool) SetConfigPath(path string) { + p.cfgPath = path +} + +func (p *Pool) Init() (err error) { + for i := 0; i < p.size; i++ { + if err := p.NewClient(); err != nil { + return err + } + } + return nil +} + +func (p *Pool) NewClient() (err error) { + c, err := NewClient() + if err != nil { + return trace.TraceError(err) + } + if err := c.Start(); err != nil { + return err + } + p.clients.Add(c) + return nil +} + +func (p *Pool) GetClient() (c interfaces.GrpcClient, err error) { + idx := p.getRandomIndex() + res, ok := p.clients.Get(idx) + if !ok { + return nil, trace.TraceError(errors.ErrorGrpcClientNotExists) + } + c, ok = res.(interfaces.GrpcClient) + if !ok { + return nil, trace.TraceError(errors.ErrorGrpcInvalidType) + } + return c, nil +} + +func (p *Pool) SetSize(size int) { + p.size = size +} + +func (p *Pool) getRandomIndex() (idx int) { + return rand.Intn(p.clients.Size()) +} + +func NewPool(opts ...PoolOption) (p interfaces.GrpcClientPool, err error) { + // pool + p = &Pool{ + size: 1, + clients: arraylist.New(), + } + + // apply options + for _, opt := range opts { + opt(p) + } + + // initialize + if err := p.Init(); err != nil { + return nil, err + } + + return p, nil +} diff --git a/core/grpc/client/utils_proto.go b/core/grpc/client/utils_proto.go new file mode 100644 index 000000000..e47592330 --- /dev/null +++ b/core/grpc/client/utils_proto.go @@ -0,0 +1,5 @@ +package client + +import grpc2 "github.com/crawlab-team/crawlab/grpc" + +var EmptyRequest = &grpc2.Request{} diff --git a/core/grpc/middlewares/auth_token.go b/core/grpc/middlewares/auth_token.go new file mode 100644 index 000000000..7135ec34d --- /dev/null +++ b/core/grpc/middlewares/auth_token.go @@ -0,0 +1,67 @@ +package middlewares + +import ( + "context" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/grpc-ecosystem/go-grpc-middleware/auth" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +func GetAuthTokenFunc(nodeCfgSvc interfaces.NodeConfigService) grpc_auth.AuthFunc { + return func(ctx context.Context) (ctx2 context.Context, err error) { + // authentication (token verification) + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, errors.ErrorGrpcUnauthorized + } + + // auth key from incoming context + res, ok := md[constants.GrpcHeaderAuthorization] + if !ok { + return ctx, errors.ErrorGrpcUnauthorized + } + if len(res) != 1 { + return ctx, errors.ErrorGrpcUnauthorized + } + authKey := res[0] + + // validate + svrAuthKey := nodeCfgSvc.GetAuthKey() + if authKey != svrAuthKey { + return ctx, errors.ErrorGrpcUnauthorized + } + + return ctx, nil + } +} + +func GetAuthTokenUnaryChainInterceptor(nodeCfgSvc interfaces.NodeConfigService) grpc.UnaryClientInterceptor { + // set auth key + md := metadata.Pairs(constants.GrpcHeaderAuthorization, nodeCfgSvc.GetAuthKey()) + //header := metadata.MD{} + //header[constants.GrpcHeaderAuthorization] = []string{nodeCfgSvc.GetAuthKey()} + return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + ctx = metadata.NewOutgoingContext(context.Background(), md) + //opts = append(opts, grpc.Header(&header)) + return invoker(ctx, method, req, reply, cc, opts...) + } +} + +func GetAuthTokenStreamChainInterceptor(nodeCfgSvc interfaces.NodeConfigService) grpc.StreamClientInterceptor { + // set auth key + md := metadata.Pairs(constants.GrpcHeaderAuthorization, nodeCfgSvc.GetAuthKey()) + //header := metadata.MD{} + //header[constants.GrpcHeaderAuthorization] = []string{nodeCfgSvc.GetAuthKey()} + return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + ctx = metadata.NewOutgoingContext(context.Background(), md) + //opts = append(opts, grpc.Header(&header)) + s, err := streamer(ctx, desc, cc, method, opts...) + if err != nil { + return nil, err + } + return s, nil + } +} diff --git a/core/grpc/payload/model_service_v2_payload.go b/core/grpc/payload/model_service_v2_payload.go new file mode 100644 index 000000000..454d964d6 --- /dev/null +++ b/core/grpc/payload/model_service_v2_payload.go @@ -0,0 +1,17 @@ +package payload + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type ModelServiceV2Payload struct { + Type string `json:"type,omitempty"` + Id primitive.ObjectID `json:"_id,omitempty"` + Query bson.M `json:"query,omitempty"` + FindOptions *mongo.FindOptions `json:"find_options,omitempty"` + Model any `json:"model,omitempty"` + Update bson.M `json:"update,omitempty"` + Models []any `json:"models,omitempty"` +} diff --git a/core/grpc/server/dependencies_server_v2.go b/core/grpc/server/dependencies_server_v2.go new file mode 100644 index 000000000..aca7e0086 --- /dev/null +++ b/core/grpc/server/dependencies_server_v2.go @@ -0,0 +1,32 @@ +package server + +import ( + "context" + grpc "github.com/crawlab-team/crawlab/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type DependenciesServerV2 struct { + grpc.UnimplementedDependencyServiceV2Server +} + +func (svr DependenciesServerV2) Connect(stream grpc.DependencyServiceV2_ConnectServer) (err error) { + return status.Errorf(codes.Unimplemented, "method Connect not implemented") +} + +func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.DependenciesServiceV2SyncRequest) (response *grpc.Response, err error) { + return nil, status.Errorf(codes.Unimplemented, "method Sync not implemented") +} + +func (svr DependenciesServerV2) Install(stream grpc.DependencyServiceV2_InstallServer) (err error) { + return status.Errorf(codes.Unimplemented, "method Install not implemented") +} + +func (svr DependenciesServerV2) UninstallDependencies(stream grpc.DependencyServiceV2_UninstallDependenciesServer) (err error) { + return status.Errorf(codes.Unimplemented, "method UninstallDependencies not implemented") +} + +func NewDependenciesServerV2() *DependenciesServerV2 { + return &DependenciesServerV2{} +} diff --git a/core/grpc/server/message_server.go b/core/grpc/server/message_server.go new file mode 100644 index 000000000..d39893415 --- /dev/null +++ b/core/grpc/server/message_server.go @@ -0,0 +1,93 @@ +package server + +import ( + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/service" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "io" +) + +type MessageServer struct { + grpc.UnimplementedMessageServiceServer + + // dependencies + modelSvc service.ModelService + cfgSvc interfaces.NodeConfigService + + // internals + server interfaces.GrpcServer +} + +func (svr MessageServer) Connect(stream grpc.MessageService_ConnectServer) (err error) { + finished := make(chan bool) + for { + msg, err := stream.Recv() + nodeKey := "unknown node key" + if msg != nil { + nodeKey = msg.NodeKey + } + if err == io.EOF { + log.Infof("[MessageServer] received signal EOF from node[%s], now quit", nodeKey) + return nil + } + if err != nil { + log.Errorf("[MessageServer] receiving message error from node[%s]: %v", nodeKey, err) + return err + } + switch msg.Code { + case grpc.StreamMessageCode_CONNECT: + log.Infof("[MessageServer] received connect request from node[%s], key: %s", nodeKey, msg.Key) + svr.server.SetSubscribe(msg.Key, &entity.GrpcSubscribe{ + Stream: stream, + Finished: finished, + }) + case grpc.StreamMessageCode_DISCONNECT: + log.Infof("[MessageServer] received disconnect request from node[%s], key: %s", nodeKey, msg.Key) + svr.server.DeleteSubscribe(msg.Key) + return nil + case grpc.StreamMessageCode_SEND: + log.Debugf("[MessageServer] received send request from node[%s] to %s", nodeKey, msg.To) + sub, err := svr.server.GetSubscribe(msg.To) + if err != nil { + return err + } + svr.redirectMessage(sub, msg) + } + } +} + +func (svr MessageServer) redirectMessage(sub interfaces.GrpcSubscribe, msg *grpc.StreamMessage) { + stream := sub.GetStream() + if stream == nil { + trace.PrintError(errors.ErrorGrpcStreamNotFound) + return + } + log.Debugf("[MessageServer] redirect message: %v", msg) + if err := stream.Send(msg); err != nil { + trace.PrintError(err) + return + } +} + +func NewMessageServer() (res *MessageServer, err error) { + // message server + svr := &MessageServer{} + + // dependency injection + if err := container.GetContainer().Invoke(func( + modelSvc service.ModelService, + cfgSvc interfaces.NodeConfigService, + ) { + svr.modelSvc = modelSvc + svr.cfgSvc = cfgSvc + }); err != nil { + return nil, err + } + + return svr, nil +} diff --git a/core/grpc/server/model_base_service_binder.go b/core/grpc/server/model_base_service_binder.go new file mode 100644 index 000000000..20244dbf1 --- /dev/null +++ b/core/grpc/server/model_base_service_binder.go @@ -0,0 +1,59 @@ +package server + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" +) + +func NewModelBaseServiceBinder(req *grpc.Request) (b *ModelBaseServiceBinder) { + return &ModelBaseServiceBinder{ + req: req, + msg: &entity.GrpcBaseServiceMessage{}, + } +} + +type ModelBaseServiceBinder struct { + req *grpc.Request + msg interfaces.GrpcModelBaseServiceMessage +} + +func (b *ModelBaseServiceBinder) Bind() (res *entity.GrpcBaseServiceParams, err error) { + if err := b.bindBaseServiceMessage(); err != nil { + return nil, err + } + params := &entity.GrpcBaseServiceParams{} + return b.process(params) +} + +func (b *ModelBaseServiceBinder) MustBind() (res interface{}) { + res, err := b.Bind() + if err != nil { + panic(err) + } + return res +} + +func (b *ModelBaseServiceBinder) BindWithBaseServiceMessage() (params *entity.GrpcBaseServiceParams, msg interfaces.GrpcModelBaseServiceMessage, err error) { + if err := json.Unmarshal(b.req.Data, b.msg); err != nil { + return nil, nil, err + } + params, err = b.Bind() + if err != nil { + return nil, nil, err + } + return params, b.msg, nil +} + +func (b *ModelBaseServiceBinder) process(params *entity.GrpcBaseServiceParams) (res *entity.GrpcBaseServiceParams, err error) { + if err := json.Unmarshal(b.msg.GetData(), params); err != nil { + return nil, trace.TraceError(err) + } + return params, nil +} + +func (b *ModelBaseServiceBinder) bindBaseServiceMessage() (err error) { + return json.Unmarshal(b.req.Data, b.msg) +} diff --git a/core/grpc/server/model_base_service_server.go b/core/grpc/server/model_base_service_server.go new file mode 100644 index 000000000..7f373affc --- /dev/null +++ b/core/grpc/server/model_base_service_server.go @@ -0,0 +1,139 @@ +package server + +import ( + "context" + "encoding/json" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" +) + +type ModelBaseServiceServer struct { + grpc.UnimplementedModelBaseServiceServer + + // dependencies + modelSvc interfaces.ModelService +} + +func (svr ModelBaseServiceServer) GetById(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { + return svc.GetById(params.Id) + }) +} + +func (svr ModelBaseServiceServer) Get(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { + return svc.Get(utils.NormalizeBsonMObjectId(params.Query), params.FindOptions) + }) +} + +func (svr ModelBaseServiceServer) GetList(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { + list, err := svc.GetList(utils.NormalizeBsonMObjectId(params.Query), params.FindOptions) + if err != nil { + return nil, err + } + data, err := json.Marshal(list) + if err != nil { + return nil, err + } + return data, nil + }) +} + +func (svr ModelBaseServiceServer) DeleteById(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { + err := svc.DeleteById(params.Id, params.User) + return nil, err + }) +} + +func (svr ModelBaseServiceServer) Delete(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { + err := svc.Delete(utils.NormalizeBsonMObjectId(params.Query), params.User) + return nil, err + }) +} + +func (svr ModelBaseServiceServer) DeleteList(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { + err := svc.DeleteList(utils.NormalizeBsonMObjectId(params.Query), params.User) + return nil, err + }) +} + +func (svr ModelBaseServiceServer) ForceDeleteList(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { + err := svc.ForceDeleteList(utils.NormalizeBsonMObjectId(params.Query), params.User) + return nil, err + }) +} + +func (svr ModelBaseServiceServer) UpdateById(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { + err := svc.UpdateById(params.Id, params.Update) + return nil, err + }) +} + +func (svr ModelBaseServiceServer) Update(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { + err := svc.Update(utils.NormalizeBsonMObjectId(params.Query), params.Update, params.Fields, params.User) + return nil, err + }) +} + +func (svr ModelBaseServiceServer) UpdateDoc(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { + err := svc.UpdateDoc(utils.NormalizeBsonMObjectId(params.Query), params.Doc, params.Fields, params.User) + return nil, err + }) +} + +func (svr ModelBaseServiceServer) Insert(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { + err := svc.Insert(params.User, params.Docs...) + return nil, err + }) +} + +func (svr ModelBaseServiceServer) Count(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return svr.handleRequest(req, func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) { + return svc.Count(utils.NormalizeBsonMObjectId(params.Query)) + }) +} + +func (svr ModelBaseServiceServer) handleRequest(req *grpc.Request, handle handleBaseServiceRequest) (res *grpc.Response, err error) { + params, msg, err := NewModelBaseServiceBinder(req).BindWithBaseServiceMessage() + if err != nil { + return HandleError(err) + } + svc := svr.modelSvc.GetBaseService(msg.GetModelId()) + d, err := handle(params, svc) + if err != nil { + return HandleError(err) + } + if d == nil { + return HandleSuccess() + } + return HandleSuccessWithData(d) +} + +type handleBaseServiceRequest func(params *entity.GrpcBaseServiceParams, svc interfaces.ModelBaseService) (interface{}, error) + +func NewModelBaseServiceServer() (svr2 *ModelBaseServiceServer, err error) { + svr := &ModelBaseServiceServer{} + + // dependency injection + if err := container.GetContainer().Invoke(func(modelSvc service.ModelService) { + svr.modelSvc = modelSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + return svr, nil +} diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go new file mode 100644 index 000000000..8ae328c52 --- /dev/null +++ b/core/grpc/server/model_base_service_v2_server.go @@ -0,0 +1,306 @@ +package server + +import ( + "context" + "encoding/json" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + grpc "github.com/crawlab-team/crawlab/grpc" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "reflect" +) + +var ( + typeNameColNameMap = make(map[string]string) + typeOneNameModelMap = make(map[string]any) + typeOneInstances = []any{ + *new(models.TestModel), + *new(models.DataCollectionV2), + *new(models.DataSourceV2), + *new(models.DependencySettingV2), + *new(models.EnvironmentV2), + *new(models.GitV2), + *new(models.NodeV2), + *new(models.PermissionV2), + *new(models.ProjectV2), + *new(models.RolePermissionV2), + *new(models.RoleV2), + *new(models.ScheduleV2), + *new(models.SettingV2), + *new(models.SpiderV2), + *new(models.TaskQueueItemV2), + *new(models.TaskStatV2), + *new(models.TaskV2), + *new(models.TokenV2), + *new(models.UserRoleV2), + *new(models.UserV2), + } +) + +func init() { + for _, v := range typeOneInstances { + t := reflect.TypeOf(v) + typeName := t.Name() + colName := service.GetCollectionNameByInstance(v) + typeNameColNameMap[typeName] = colName + typeOneNameModelMap[typeName] = v + } +} + +func GetOneInstanceModel(typeName string) any { + return typeOneNameModelMap[typeName] +} + +type ModelBaseServiceServerV2 struct { + grpc.UnimplementedModelBaseServiceV2Server +} + +func (svr ModelBaseServiceServerV2) GetById(ctx context.Context, req *grpc.ModelServiceV2GetByIdRequest) (res *grpc.Response, err error) { + id, err := primitive.ObjectIDFromHex(req.Id) + if err != nil { + return HandleError(err) + } + modelSvc := service.NewModelServiceV2WithColName[bson.M](typeNameColNameMap[req.ModelType]) + data, err := modelSvc.GetById(id) + if err != nil { + return HandleError(err) + } + return HandleSuccessWithData(data) +} + +func (svr ModelBaseServiceServerV2) GetOne(ctx context.Context, req *grpc.ModelServiceV2GetOneRequest) (res *grpc.Response, err error) { + var query bson.M + err = json.Unmarshal(req.Query, &query) + if err != nil { + return HandleError(err) + } + var options mongo.FindOptions + err = json.Unmarshal(req.FindOptions, &options) + if err != nil { + return HandleError(err) + } + modelSvc := service.NewModelServiceV2WithColName[bson.M](typeNameColNameMap[req.ModelType]) + data, err := modelSvc.GetOne(query, &options) + if err != nil { + return HandleError(err) + } + return HandleSuccessWithData(data) +} + +func (svr ModelBaseServiceServerV2) GetMany(ctx context.Context, req *grpc.ModelServiceV2GetManyRequest) (res *grpc.Response, err error) { + var query bson.M + err = json.Unmarshal(req.Query, &query) + if err != nil { + return HandleError(err) + } + var options mongo.FindOptions + err = json.Unmarshal(req.FindOptions, &options) + if err != nil { + return HandleError(err) + } + modelSvc := service.NewModelServiceV2WithColName[bson.M](typeNameColNameMap[req.ModelType]) + data, err := modelSvc.GetMany(query, &options) + if err != nil { + return HandleError(err) + } + return HandleSuccessWithData(data) +} + +func (svr ModelBaseServiceServerV2) DeleteById(ctx context.Context, req *grpc.ModelServiceV2DeleteByIdRequest) (res *grpc.Response, err error) { + id, err := primitive.ObjectIDFromHex(req.Id) + if err != nil { + return HandleError(err) + } + modelSvc := GetModelService[bson.M](req.ModelType) + err = modelSvc.DeleteById(id) + if err != nil { + return HandleError(err) + } + return HandleSuccess() +} + +func (svr ModelBaseServiceServerV2) DeleteOne(ctx context.Context, req *grpc.ModelServiceV2DeleteOneRequest) (res *grpc.Response, err error) { + var query bson.M + err = json.Unmarshal(req.Query, &query) + if err != nil { + return HandleError(err) + } + modelSvc := GetModelService[bson.M](req.ModelType) + err = modelSvc.DeleteOne(query) + if err != nil { + return HandleError(err) + } + return HandleSuccess() +} + +func (svr ModelBaseServiceServerV2) DeleteMany(ctx context.Context, req *grpc.ModelServiceV2DeleteManyRequest) (res *grpc.Response, err error) { + var query bson.M + err = json.Unmarshal(req.Query, &query) + if err != nil { + return HandleError(err) + } + modelSvc := GetModelService[bson.M](req.ModelType) + err = modelSvc.DeleteMany(query) + if err != nil { + return HandleError(err) + } + return HandleSuccess() +} + +func (svr ModelBaseServiceServerV2) UpdateById(ctx context.Context, req *grpc.ModelServiceV2UpdateByIdRequest) (res *grpc.Response, err error) { + id, err := primitive.ObjectIDFromHex(req.Id) + if err != nil { + return HandleError(err) + } + var update bson.M + err = json.Unmarshal(req.Update, &update) + if err != nil { + return HandleError(err) + } + modelSvc := GetModelService[bson.M](req.ModelType) + err = modelSvc.UpdateById(id, update) + if err != nil { + return HandleError(err) + } + return HandleSuccess() +} + +func (svr ModelBaseServiceServerV2) UpdateOne(ctx context.Context, req *grpc.ModelServiceV2UpdateOneRequest) (res *grpc.Response, err error) { + var query bson.M + err = json.Unmarshal(req.Query, &query) + if err != nil { + return HandleError(err) + } + var update bson.M + err = json.Unmarshal(req.Update, &update) + if err != nil { + return HandleError(err) + } + modelSvc := GetModelService[bson.M](req.ModelType) + err = modelSvc.UpdateOne(query, update) + if err != nil { + return HandleError(err) + } + return HandleSuccess() +} + +func (svr ModelBaseServiceServerV2) UpdateMany(ctx context.Context, req *grpc.ModelServiceV2UpdateManyRequest) (res *grpc.Response, err error) { + var query bson.M + err = json.Unmarshal(req.Query, &query) + if err != nil { + return HandleError(err) + } + var update bson.M + err = json.Unmarshal(req.Update, &update) + if err != nil { + return HandleError(err) + } + modelSvc := GetModelService[bson.M](req.ModelType) + err = modelSvc.UpdateMany(query, update) + if err != nil { + return HandleError(err) + } + return HandleSuccess() +} + +func (svr ModelBaseServiceServerV2) ReplaceById(ctx context.Context, req *grpc.ModelServiceV2ReplaceByIdRequest) (res *grpc.Response, err error) { + id, err := primitive.ObjectIDFromHex(req.Id) + if err != nil { + return HandleError(err) + } + model := GetOneInstanceModel(req.ModelType) + modelType := reflect.TypeOf(model) + modelValuePtr := reflect.New(modelType).Interface() + err = json.Unmarshal(req.Model, modelValuePtr) + if err != nil { + return HandleError(err) + } + modelSvc := GetModelService[bson.M](req.ModelType) + err = modelSvc.GetCol().ReplaceId(id, modelValuePtr) + if err != nil { + return HandleError(err) + } + return HandleSuccess() +} + +func (svr ModelBaseServiceServerV2) ReplaceOne(ctx context.Context, req *grpc.ModelServiceV2ReplaceOneRequest) (res *grpc.Response, err error) { + var query bson.M + err = json.Unmarshal(req.Query, &query) + if err != nil { + return HandleError(err) + } + model := GetOneInstanceModel(req.ModelType) + modelType := reflect.TypeOf(model) + modelValuePtr := reflect.New(modelType).Interface() + err = json.Unmarshal(req.Model, &modelValuePtr) + if err != nil { + return HandleError(err) + } + modelSvc := GetModelService[bson.M](req.ModelType) + err = modelSvc.GetCol().Replace(query, modelValuePtr) + if err != nil { + return HandleError(err) + } + return HandleSuccess() +} + +func (svr ModelBaseServiceServerV2) InsertOne(ctx context.Context, req *grpc.ModelServiceV2InsertOneRequest) (res *grpc.Response, err error) { + model := GetOneInstanceModel(req.ModelType) + modelType := reflect.TypeOf(model) + modelValuePtr := reflect.New(modelType).Interface() + err = json.Unmarshal(req.Model, modelValuePtr) + if err != nil { + return HandleError(err) + } + modelSvc := GetModelService[bson.M](req.ModelType) + r, err := modelSvc.GetCol().GetCollection().InsertOne(modelSvc.GetCol().GetContext(), modelValuePtr) + if err != nil { + return HandleError(err) + } + return HandleSuccessWithData(r.InsertedID) +} + +func (svr ModelBaseServiceServerV2) InsertMany(ctx context.Context, req *grpc.ModelServiceV2InsertManyRequest) (res *grpc.Response, err error) { + model := GetOneInstanceModel(req.ModelType) + modelType := reflect.TypeOf(model) + modelsSliceType := reflect.SliceOf(modelType) + modelsSlicePtr := reflect.New(modelsSliceType).Interface() + err = json.Unmarshal(req.Models, modelsSlicePtr) + if err != nil { + return HandleError(err) + } + modelsSlice := reflect.ValueOf(modelsSlicePtr).Elem() + modelsInterface := make([]any, modelsSlice.Len()) + for i := 0; i < modelsSlice.Len(); i++ { + modelsInterface[i] = modelsSlice.Index(i).Interface() + } + modelSvc := GetModelService[bson.M](req.ModelType) + r, err := modelSvc.GetCol().GetCollection().InsertMany(modelSvc.GetCol().GetContext(), modelsInterface) + if err != nil { + return HandleError(err) + } + return HandleSuccessWithData(r.InsertedIDs) +} + +func (svr ModelBaseServiceServerV2) Count(ctx context.Context, req *grpc.ModelServiceV2CountRequest) (res *grpc.Response, err error) { + var query bson.M + err = json.Unmarshal(req.Query, &query) + if err != nil { + return HandleError(err) + } + count, err := GetModelService[bson.M](req.ModelType).Count(query) + if err != nil { + return HandleError(err) + } + return HandleSuccessWithData(count) +} + +func GetModelService[T any](typeName string) *service.ModelServiceV2[T] { + return service.NewModelServiceV2WithColName[T](typeNameColNameMap[typeName]) +} + +func NewModelBaseServiceV2Server() *ModelBaseServiceServerV2 { + return &ModelBaseServiceServerV2{} +} diff --git a/core/grpc/server/model_delegate_binder.go b/core/grpc/server/model_delegate_binder.go new file mode 100644 index 000000000..172eae0ec --- /dev/null +++ b/core/grpc/server/model_delegate_binder.go @@ -0,0 +1,120 @@ +package server + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/grpc" +) + +func NewModelDelegateBinder(req *grpc.Request) (b *ModelDelegateBinder) { + return &ModelDelegateBinder{ + req: req, + msg: &entity.GrpcDelegateMessage{}, + } +} + +type ModelDelegateBinder struct { + req *grpc.Request + msg interfaces.GrpcModelDelegateMessage +} + +func (b *ModelDelegateBinder) Bind() (res interface{}, err error) { + if err := b.bindDelegateMessage(); err != nil { + return nil, err + } + + m := models.NewModelMap() + + switch b.msg.GetModelId() { + case interfaces.ModelIdArtifact: + return b.process(&m.Artifact, interfaces.ModelIdTag) + case interfaces.ModelIdTag: + return b.process(&m.Tag, interfaces.ModelIdTag) + case interfaces.ModelIdNode: + return b.process(&m.Node, interfaces.ModelIdTag) + case interfaces.ModelIdProject: + return b.process(&m.Project, interfaces.ModelIdTag) + case interfaces.ModelIdSpider: + return b.process(&m.Spider, interfaces.ModelIdTag) + case interfaces.ModelIdTask: + return b.process(&m.Task) + case interfaces.ModelIdJob: + return b.process(&m.Job) + case interfaces.ModelIdSchedule: + return b.process(&m.Schedule) + case interfaces.ModelIdUser: + return b.process(&m.User) + case interfaces.ModelIdSetting: + return b.process(&m.Setting) + case interfaces.ModelIdToken: + return b.process(&m.Token) + case interfaces.ModelIdVariable: + return b.process(&m.Variable) + case interfaces.ModelIdTaskQueue: + return b.process(&m.TaskQueueItem) + case interfaces.ModelIdTaskStat: + return b.process(&m.TaskStat) + case interfaces.ModelIdSpiderStat: + return b.process(&m.SpiderStat) + case interfaces.ModelIdDataSource: + return b.process(&m.DataSource) + case interfaces.ModelIdDataCollection: + return b.process(&m.DataCollection) + case interfaces.ModelIdResult: + return b.process(&m.Result) + case interfaces.ModelIdPassword: + return b.process(&m.Password) + case interfaces.ModelIdExtraValue: + return b.process(&m.ExtraValue) + case interfaces.ModelIdGit: + return b.process(&m.Git) + case interfaces.ModelIdRole: + return b.process(&m.Role) + case interfaces.ModelIdUserRole: + return b.process(&m.UserRole) + case interfaces.ModelIdPermission: + return b.process(&m.Permission) + case interfaces.ModelIdRolePermission: + return b.process(&m.RolePermission) + case interfaces.ModelIdEnvironment: + return b.process(&m.Environment) + case interfaces.ModelIdDependencySetting: + return b.process(&m.DependencySetting) + default: + return nil, errors.ErrorModelInvalidModelId + } +} + +func (b *ModelDelegateBinder) MustBind() (res interface{}) { + res, err := b.Bind() + if err != nil { + panic(err) + } + return res +} + +func (b *ModelDelegateBinder) BindWithDelegateMessage() (res interface{}, msg interfaces.GrpcModelDelegateMessage, err error) { + if err := json.Unmarshal(b.req.Data, b.msg); err != nil { + return nil, nil, err + } + res, err = b.Bind() + if err != nil { + return nil, nil, err + } + return res, b.msg, nil +} + +func (b *ModelDelegateBinder) process(d interface{}, fieldIds ...interfaces.ModelId) (res interface{}, err error) { + if err := json.Unmarshal(b.msg.GetData(), d); err != nil { + return nil, err + } + //return models.AssignFields(d, fieldIds...) // TODO: do we need to assign fields? + return d, nil +} + +func (b *ModelDelegateBinder) bindDelegateMessage() (err error) { + return json.Unmarshal(b.req.Data, b.msg) +} diff --git a/core/grpc/server/model_delegate_server.go b/core/grpc/server/model_delegate_server.go new file mode 100644 index 000000000..4f097d559 --- /dev/null +++ b/core/grpc/server/model_delegate_server.go @@ -0,0 +1,67 @@ +package server + +import ( + "context" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + grpc "github.com/crawlab-team/crawlab/grpc" +) + +type ModelDelegateServer struct { + grpc.UnimplementedModelDelegateServer +} + +// Do and perform an RPC action of constants.Delegate +func (svr ModelDelegateServer) Do(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + // bind message + obj, msg, err := NewModelDelegateBinder(req).BindWithDelegateMessage() + if err != nil { + return HandleError(err) + } + + // convert to model + doc, ok := obj.(interfaces.Model) + if !ok { + return HandleError(errors.ErrorModelInvalidType) + } + + // model delegate + d := delegate.NewModelDelegate(doc) + + // apply method + switch msg.GetMethod() { + case interfaces.ModelDelegateMethodAdd: + err = d.Add() + case interfaces.ModelDelegateMethodSave: + err = d.Save() + case interfaces.ModelDelegateMethodDelete: + err = d.Delete() + case interfaces.ModelDelegateMethodGetArtifact, interfaces.ModelDelegateMethodRefresh: + err = d.Refresh() + } + if err != nil { + return HandleError(err) + } + + // model + m := d.GetModel() + if msg.GetMethod() == interfaces.ModelDelegateMethodGetArtifact { + m, err = d.GetArtifact() + if err != nil { + return nil, err + } + } + + // json bytes + data, err := d.ToBytes(m) + if err != nil { + return nil, err + } + + return HandleSuccessWithData(data) +} + +func NewModelDelegateServer() (svr *ModelDelegateServer) { + return &ModelDelegateServer{} +} diff --git a/core/grpc/server/node_server.go b/core/grpc/server/node_server.go new file mode 100644 index 000000000..c548ea7f7 --- /dev/null +++ b/core/grpc/server/node_server.go @@ -0,0 +1,202 @@ +package server + +import ( + "context" + "encoding/json" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/grpc" + "go.mongodb.org/mongo-driver/mongo" +) + +type NodeServer struct { + grpc.UnimplementedNodeServiceServer + + // dependencies + modelSvc service.ModelService + cfgSvc interfaces.NodeConfigService + + // internals + server interfaces.GrpcServer +} + +// Register from handler/worker to master +func (svr NodeServer) Register(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + // unmarshall data + var nodeInfo entity.NodeInfo + if req.Data != nil { + if err := json.Unmarshal(req.Data, &nodeInfo); err != nil { + return HandleError(err) + } + + if nodeInfo.IsMaster { + // error: cannot register master node + return HandleError(errors.ErrorGrpcNotAllowed) + } + } + + // node key + var nodeKey string + if req.NodeKey != "" { + nodeKey = req.NodeKey + } else { + nodeKey = nodeInfo.Key + } + if nodeKey == "" { + return HandleError(errors.ErrorModelMissingRequiredData) + } + + // find in db + node, err := svr.modelSvc.GetNodeByKey(nodeKey, nil) + if err == nil { + if node.IsMaster { + // error: cannot register master node + return HandleError(errors.ErrorGrpcNotAllowed) + } else { + // register existing + node.Status = constants.NodeStatusRegistered + node.Active = true + nodeD := delegate.NewModelNodeDelegate(node) + if err := nodeD.Save(); err != nil { + return HandleError(err) + } + var ok bool + node, ok = nodeD.GetModel().(*models.Node) + if !ok { + return HandleError(errors.ErrorGrpcInvalidType) + } + log.Infof("[NodeServer] updated worker[%s] in db. id: %s", nodeKey, nodeD.GetModel().GetId().Hex()) + } + } else if err == mongo.ErrNoDocuments { + // register new + node = &models.Node{ + Key: nodeKey, + Name: nodeInfo.Name, + Ip: nodeInfo.Ip, + Hostname: nodeInfo.Hostname, + Description: nodeInfo.Description, + MaxRunners: nodeInfo.MaxRunners, + Status: constants.NodeStatusRegistered, + Active: true, + Enabled: true, + } + if node.Name == "" { + node.Name = nodeKey + } + nodeD := delegate.NewModelDelegate(node) + if err := nodeD.Add(); err != nil { + return HandleError(err) + } + var ok bool + node, ok = nodeD.GetModel().(*models.Node) + if !ok { + return HandleError(errors.ErrorGrpcInvalidType) + } + log.Infof("[NodeServer] added worker[%s] in db. id: %s", nodeKey, nodeD.GetModel().GetId().Hex()) + } else { + // error + return HandleError(err) + } + + log.Infof("[NodeServer] master registered worker[%s]", req.GetNodeKey()) + + return HandleSuccessWithData(node) +} + +// SendHeartbeat from worker to master +func (svr NodeServer) SendHeartbeat(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + // find in db + node, err := svr.modelSvc.GetNodeByKey(req.NodeKey, nil) + if err != nil { + if err == mongo.ErrNoDocuments { + return HandleError(errors.ErrorNodeNotExists) + } + return HandleError(err) + } + + // validate status + if node.Status == constants.NodeStatusUnregistered { + return HandleError(errors.ErrorNodeUnregistered) + } + + // update status + nodeD := delegate.NewModelNodeDelegate(node) + if err := nodeD.UpdateStatusOnline(); err != nil { + return HandleError(err) + } + + return HandleSuccessWithData(node) +} + +// Ping from worker to master +func (svr NodeServer) Ping(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return HandleSuccess() +} + +func (svr NodeServer) Subscribe(request *grpc.Request, stream grpc.NodeService_SubscribeServer) (err error) { + log.Infof("[NodeServer] master received subscribe request from node[%s]", request.NodeKey) + + // finished channel + finished := make(chan bool) + + // set subscribe + svr.server.SetSubscribe("node:"+request.NodeKey, &entity.GrpcSubscribe{ + Stream: stream, + Finished: finished, + }) + ctx := stream.Context() + + log.Infof("[NodeServer] master subscribed node[%s]", request.NodeKey) + + // Keep this scope alive because once this scope exits - the stream is closed + for { + select { + case <-finished: + log.Infof("[NodeServer] closing stream for node[%s]", request.NodeKey) + return nil + case <-ctx.Done(): + log.Infof("[NodeServer] node[%s] has disconnected", request.NodeKey) + return nil + } + } +} + +func (svr NodeServer) Unsubscribe(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + sub, err := svr.server.GetSubscribe("node:" + req.NodeKey) + if err != nil { + return nil, errors.ErrorGrpcSubscribeNotExists + } + select { + case sub.GetFinished() <- true: + log.Infof("unsubscribed node[%s]", req.NodeKey) + default: + // Default case is to avoid blocking in case client has already unsubscribed + } + svr.server.DeleteSubscribe(req.NodeKey) + return &grpc.Response{ + Code: grpc.ResponseCode_OK, + Message: "unsubscribed successfully", + }, nil +} + +func NewNodeServer() (res *NodeServer, err error) { + // node server + svr := &NodeServer{} + svr.modelSvc, err = service.GetService() + if err != nil { + return nil, err + } + svr.cfgSvc, err = nodeconfig.NewNodeConfigService() + if err != nil { + return nil, err + } + + return svr, nil +} diff --git a/core/grpc/server/node_server_v2.go b/core/grpc/server/node_server_v2.go new file mode 100644 index 000000000..ac9499ca8 --- /dev/null +++ b/core/grpc/server/node_server_v2.go @@ -0,0 +1,189 @@ +package server + +import ( + "context" + "encoding/json" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/grpc" + errors2 "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "time" +) + +type NodeServerV2 struct { + grpc.UnimplementedNodeServiceServer + + // dependencies + cfgSvc interfaces.NodeConfigService + + // internals + server *GrpcServerV2 +} + +// Register from handler/worker to master +func (svr NodeServerV2) Register(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + // unmarshall data + var node models.NodeV2 + if req.Data != nil { + if err := json.Unmarshal(req.Data, &node); err != nil { + return HandleError(err) + } + + if node.IsMaster { + // error: cannot register master node + return HandleError(errors.ErrorGrpcNotAllowed) + } + } + + // node key + var nodeKey string + if req.NodeKey != "" { + nodeKey = req.NodeKey + } else { + nodeKey = node.Key + } + if nodeKey == "" { + return HandleError(errors.ErrorModelMissingRequiredData) + } + + // find in db + nodeDb, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + if err == nil { + if node.IsMaster { + // error: cannot register master node + return HandleError(errors.ErrorGrpcNotAllowed) + } else { + // register existing + nodeDb.Status = constants.NodeStatusRegistered + nodeDb.Active = true + err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(nodeDb.Id, *nodeDb) + if err != nil { + return HandleError(err) + } + log.Infof("[NodeServerV2] updated worker[%s] in db. id: %s", nodeKey, node.Id.Hex()) + } + } else if errors2.Is(err, mongo.ErrNoDocuments) { + // register new + node.Key = nodeKey + node.Status = constants.NodeStatusRegistered + node.Active = true + node.ActiveAt = time.Now() + node.Enabled = true + if node.Name == "" { + node.Name = nodeKey + } + node.SetCreated(primitive.NilObjectID) + node.SetUpdated(primitive.NilObjectID) + _, err = service.NewModelServiceV2[models.NodeV2]().InsertOne(*nodeDb) + if err != nil { + return HandleError(err) + } + log.Infof("[NodeServerV2] added worker[%s] in db. id: %s", nodeKey, node.Id.Hex()) + } else { + // error + return HandleError(err) + } + + log.Infof("[NodeServerV2] master registered worker[%s]", req.GetNodeKey()) + + return HandleSuccessWithData(node) +} + +// SendHeartbeat from worker to master +func (svr NodeServerV2) SendHeartbeat(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + // find in db + node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil) + if err != nil { + if errors2.Is(err, mongo.ErrNoDocuments) { + return HandleError(errors.ErrorNodeNotExists) + } + return HandleError(err) + } + + // validate status + if node.Status == constants.NodeStatusUnregistered { + return HandleError(errors.ErrorNodeUnregistered) + } + + // update status + node.Status = constants.NodeStatusOnline + node.Active = true + node.ActiveAt = time.Now() + err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + if err != nil { + return HandleError(err) + } + + return HandleSuccessWithData(node) +} + +// Ping from worker to master +func (svr NodeServerV2) Ping(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + return HandleSuccess() +} + +func (svr NodeServerV2) Subscribe(request *grpc.Request, stream grpc.NodeService_SubscribeServer) (err error) { + log.Infof("[NodeServerV2] master received subscribe request from node[%s]", request.NodeKey) + + // finished channel + finished := make(chan bool) + + // set subscribe + svr.server.SetSubscribe("node:"+request.NodeKey, &entity.GrpcSubscribe{ + Stream: stream, + Finished: finished, + }) + ctx := stream.Context() + + log.Infof("[NodeServerV2] master subscribed node[%s]", request.NodeKey) + + // Keep this scope alive because once this scope exits - the stream is closed + for { + select { + case <-finished: + log.Infof("[NodeServerV2] closing stream for node[%s]", request.NodeKey) + return nil + case <-ctx.Done(): + log.Infof("[NodeServerV2] node[%s] has disconnected", request.NodeKey) + return nil + } + } +} + +func (svr NodeServerV2) Unsubscribe(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { + sub, err := svr.server.GetSubscribe("node:" + req.NodeKey) + if err != nil { + return nil, errors.ErrorGrpcSubscribeNotExists + } + select { + case sub.GetFinished() <- true: + log.Infof("unsubscribed node[%s]", req.NodeKey) + default: + // Default case is to avoid blocking in case client has already unsubscribed + } + svr.server.DeleteSubscribe(req.NodeKey) + return &grpc.Response{ + Code: grpc.ResponseCode_OK, + Message: "unsubscribed successfully", + }, nil +} + +func NewNodeServerV2() (res *NodeServerV2, err error) { + // node server + svr := &NodeServerV2{} + svr.cfgSvc, err = nodeconfig.NewNodeConfigService() + if err != nil { + return nil, err + } + + return svr, nil +} diff --git a/core/grpc/server/options.go b/core/grpc/server/options.go new file mode 100644 index 000000000..afae5dbae --- /dev/null +++ b/core/grpc/server/options.go @@ -0,0 +1,43 @@ +package server + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" +) + +type Option func(svr interfaces.GrpcServer) + +func WithConfigPath(path string) Option { + return func(svr interfaces.GrpcServer) { + svr.SetConfigPath(path) + } +} + +func WithAddress(address interfaces.Address) Option { + return func(svr interfaces.GrpcServer) { + svr.SetAddress(address) + } +} + +type NodeServerOption func(svr *NodeServer) + +func WithServerNodeServerService(server interfaces.GrpcServer) NodeServerOption { + return func(svr *NodeServer) { + svr.server = server + } +} + +type TaskServerOption func(svr *TaskServer) + +func WithServerTaskServerService(server interfaces.GrpcServer) TaskServerOption { + return func(svr *TaskServer) { + svr.server = server + } +} + +type MessageServerOption func(svr *MessageServer) + +func WithServerMessageServerService(server interfaces.GrpcServer) MessageServerOption { + return func(svr *MessageServer) { + svr.server = server + } +} diff --git a/core/grpc/server/server.go b/core/grpc/server/server.go new file mode 100644 index 000000000..6e7e83994 --- /dev/null +++ b/core/grpc/server/server.go @@ -0,0 +1,265 @@ +package server + +import ( + "encoding/json" + "fmt" + "github.com/apex/log" + config2 "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/grpc/middlewares" + "github.com/crawlab-team/crawlab/core/interfaces" + grpc2 "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" + "github.com/grpc-ecosystem/go-grpc-middleware/recovery" + "github.com/spf13/viper" + "go/types" + "google.golang.org/grpc" + "net" + "sync" +) + +var subs = sync.Map{} + +type Server struct { + // dependencies + nodeCfgSvc interfaces.NodeConfigService + nodeSvr *NodeServer + taskSvr *TaskServer + messageSvr *MessageServer + modelDelegateSvr *ModelDelegateServer + modelBaseServiceSvr *ModelBaseServiceServer + + // settings + cfgPath string + address interfaces.Address + + // internals + svr *grpc.Server + l net.Listener + stopped bool +} + +func (svr *Server) Init() (err error) { + // register + if err := svr.Register(); err != nil { + return err + } + + return nil +} + +func (svr *Server) Start() (err error) { + // grpc server binding address + address := svr.address.String() + + // listener + svr.l, err = net.Listen("tcp", address) + if err != nil { + _ = trace.TraceError(err) + return errors.ErrorGrpcServerFailedToListen + } + log.Infof("grpc server listens to %s", address) + + // start grpc server + go func() { + if err := svr.svr.Serve(svr.l); err != nil { + if err == grpc.ErrServerStopped { + return + } + trace.PrintError(err) + log.Error(errors.ErrorGrpcServerFailedToServe.Error()) + } + }() + + return nil +} + +func (svr *Server) Stop() (err error) { + // skip if listener is nil + if svr.l == nil { + return nil + } + + // graceful stop + log.Infof("grpc server stopping...") + svr.svr.Stop() + + // close listener + log.Infof("grpc server closing listener...") + _ = svr.l.Close() + + // mark as stopped + svr.stopped = true + + // log + log.Infof("grpc server stopped") + + return nil +} + +func (svr *Server) Register() (err error) { + grpc2.RegisterModelDelegateServer(svr.svr, *svr.modelDelegateSvr) // model delegate + grpc2.RegisterModelBaseServiceServer(svr.svr, *svr.modelBaseServiceSvr) // model base service + grpc2.RegisterNodeServiceServer(svr.svr, *svr.nodeSvr) // node service + grpc2.RegisterTaskServiceServer(svr.svr, *svr.taskSvr) // task service + grpc2.RegisterMessageServiceServer(svr.svr, *svr.messageSvr) // message service + + return nil +} + +func (svr *Server) SetAddress(address interfaces.Address) { + svr.address = address +} + +func (svr *Server) GetConfigPath() (path string) { + return svr.cfgPath +} + +func (svr *Server) SetConfigPath(path string) { + svr.cfgPath = path +} + +func (svr *Server) GetSubscribe(key string) (sub interfaces.GrpcSubscribe, err error) { + res, ok := subs.Load(key) + if !ok { + return nil, trace.TraceError(errors.ErrorGrpcStreamNotFound) + } + sub, ok = res.(interfaces.GrpcSubscribe) + if !ok { + return nil, trace.TraceError(errors.ErrorGrpcInvalidType) + } + return sub, nil +} + +func (svr *Server) SetSubscribe(key string, sub interfaces.GrpcSubscribe) { + subs.Store(key, sub) +} + +func (svr *Server) DeleteSubscribe(key string) { + subs.Delete(key) +} + +func (svr *Server) SendStreamMessage(key string, code grpc2.StreamMessageCode) (err error) { + return svr.SendStreamMessageWithData(key, code, nil) +} + +func (svr *Server) SendStreamMessageWithData(key string, code grpc2.StreamMessageCode, d interface{}) (err error) { + var data []byte + switch d.(type) { + case types.Nil: + // do nothing + case []byte: + data = d.([]byte) + default: + var err error + data, err = json.Marshal(d) + if err != nil { + panic(err) + } + } + sub, err := svr.GetSubscribe(key) + if err != nil { + return err + } + msg := &grpc2.StreamMessage{ + Code: code, + Key: svr.nodeCfgSvc.GetNodeKey(), + Data: data, + } + return sub.GetStream().Send(msg) +} + +func (svr *Server) IsStopped() (res bool) { + return svr.stopped +} + +func (svr *Server) recoveryHandlerFunc(p interface{}) (err error) { + err = errors.NewError(errors.ErrorPrefixGrpc, fmt.Sprintf("%v", p)) + trace.PrintError(err) + return err +} + +func NewServer() (svr2 interfaces.GrpcServer, err error) { + // server + svr := &Server{ + cfgPath: config2.GetConfigPath(), + address: entity.NewAddress(&entity.AddressOptions{ + Host: constants.DefaultGrpcServerHost, + Port: constants.DefaultGrpcServerPort, + }), + } + + if viper.GetString("grpc.server.address") != "" { + svr.address, err = entity.NewAddressFromString(viper.GetString("grpc.server.address")) + if err != nil { + return nil, err + } + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + nodeCfgSvc interfaces.NodeConfigService, + modelDelegateSvr *ModelDelegateServer, + modelBaseServiceSvr *ModelBaseServiceServer, + nodeSvr *NodeServer, + taskSvr *TaskServer, + messageSvr *MessageServer, + ) { + // dependencies + svr.nodeCfgSvc = nodeCfgSvc + svr.modelDelegateSvr = modelDelegateSvr + svr.modelBaseServiceSvr = modelBaseServiceSvr + svr.nodeSvr = nodeSvr + svr.taskSvr = taskSvr + svr.messageSvr = messageSvr + + // server + svr.nodeSvr.server = svr + svr.taskSvr.server = svr + svr.messageSvr.server = svr + }); err != nil { + return nil, err + } + + // recovery options + recoveryOpts := []grpc_recovery.Option{ + grpc_recovery.WithRecoveryHandler(svr.recoveryHandlerFunc), + } + + // grpc server + svr.svr = grpc.NewServer( + grpc_middleware.WithUnaryServerChain( + grpc_recovery.UnaryServerInterceptor(recoveryOpts...), + grpc_auth.UnaryServerInterceptor(middlewares.GetAuthTokenFunc(svr.nodeCfgSvc)), + ), + grpc_middleware.WithStreamServerChain( + grpc_recovery.StreamServerInterceptor(recoveryOpts...), + grpc_auth.StreamServerInterceptor(middlewares.GetAuthTokenFunc(svr.nodeCfgSvc)), + ), + ) + + // initialize + if err := svr.Init(); err != nil { + return nil, err + } + + return svr, nil +} + +var _server interfaces.GrpcServer + +func GetServer() (svr interfaces.GrpcServer, err error) { + if _server != nil { + return _server, nil + } + _server, err = NewServer() + if err != nil { + return nil, err + } + return _server, nil +} diff --git a/core/grpc/server/server_v2.go b/core/grpc/server/server_v2.go new file mode 100644 index 000000000..3174bdafe --- /dev/null +++ b/core/grpc/server/server_v2.go @@ -0,0 +1,256 @@ +package server + +import ( + "encoding/json" + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/grpc/middlewares" + "github.com/crawlab-team/crawlab/core/interfaces" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + grpc2 "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" + grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" + errors2 "github.com/pkg/errors" + "github.com/spf13/viper" + "go/types" + "google.golang.org/grpc" + "net" + "sync" +) + +var ( + subsV2 = map[string]interfaces.GrpcSubscribe{} + mutexSubsV2 = &sync.Mutex{} +) + +type GrpcServerV2 struct { + // settings + cfgPath string + address interfaces.Address + + // internals + svr *grpc.Server + l net.Listener + stopped bool + + // dependencies + nodeCfgSvc interfaces.NodeConfigService + + // servers + nodeSvr *NodeServerV2 + taskSvr *TaskServerV2 + modelBaseServiceSvr *ModelBaseServiceServerV2 + dependenciesSvr *DependenciesServerV2 +} + +func (svr *GrpcServerV2) GetConfigPath() (path string) { + return svr.cfgPath +} + +func (svr *GrpcServerV2) SetConfigPath(path string) { + svr.cfgPath = path +} + +func (svr *GrpcServerV2) Init() (err error) { + // register + if err := svr.Register(); err != nil { + return err + } + + return nil +} + +func (svr *GrpcServerV2) Start() (err error) { + // grpc server binding address + address := svr.address.String() + + // listener + svr.l, err = net.Listen("tcp", address) + if err != nil { + _ = trace.TraceError(err) + return errors.ErrorGrpcServerFailedToListen + } + log.Infof("grpc server listens to %s", address) + + // start grpc server + go func() { + if err := svr.svr.Serve(svr.l); err != nil { + if errors2.Is(err, grpc.ErrServerStopped) { + return + } + trace.PrintError(err) + log.Error(errors.ErrorGrpcServerFailedToServe.Error()) + } + }() + + return nil +} + +func (svr *GrpcServerV2) Stop() (err error) { + // skip if listener is nil + if svr.l == nil { + return nil + } + + // graceful stop + log.Infof("grpc server stopping...") + svr.svr.Stop() + + // close listener + log.Infof("grpc server closing listener...") + _ = svr.l.Close() + + // mark as stopped + svr.stopped = true + + // log + log.Infof("grpc server stopped") + + return nil +} + +func (svr *GrpcServerV2) Register() (err error) { + grpc2.RegisterNodeServiceServer(svr.svr, *svr.nodeSvr) // node service + grpc2.RegisterModelBaseServiceV2Server(svr.svr, *svr.modelBaseServiceSvr) + grpc2.RegisterTaskServiceServer(svr.svr, *svr.taskSvr) + + return nil +} + +func (svr *GrpcServerV2) recoveryHandlerFunc(p interface{}) (err error) { + err = errors.NewError(errors.ErrorPrefixGrpc, fmt.Sprintf("%v", p)) + trace.PrintError(err) + return err +} + +func (svr *GrpcServerV2) SetAddress(address interfaces.Address) { + +} + +func (svr *GrpcServerV2) GetSubscribe(key string) (sub interfaces.GrpcSubscribe, err error) { + mutexSubsV2.Lock() + defer mutexSubsV2.Unlock() + sub, ok := subsV2[key] + if !ok { + return nil, errors.ErrorGrpcSubscribeNotExists + } + return sub, nil +} + +func (svr *GrpcServerV2) SetSubscribe(key string, sub interfaces.GrpcSubscribe) { + mutexSubsV2.Lock() + defer mutexSubsV2.Unlock() + subsV2[key] = sub +} + +func (svr *GrpcServerV2) DeleteSubscribe(key string) { + mutexSubsV2.Lock() + defer mutexSubsV2.Unlock() + delete(subsV2, key) +} + +func (svr *GrpcServerV2) SendStreamMessage(key string, code grpc2.StreamMessageCode) (err error) { + return svr.SendStreamMessageWithData(key, code, nil) +} + +func (svr *GrpcServerV2) SendStreamMessageWithData(key string, code grpc2.StreamMessageCode, d interface{}) (err error) { + var data []byte + switch d.(type) { + case types.Nil: + // do nothing + case []byte: + data = d.([]byte) + default: + var err error + data, err = json.Marshal(d) + if err != nil { + return err + } + } + sub, err := svr.GetSubscribe(key) + if err != nil { + return err + } + msg := &grpc2.StreamMessage{ + Code: code, + Key: svr.nodeCfgSvc.GetNodeKey(), + Data: data, + } + return sub.GetStream().Send(msg) +} + +func (svr *GrpcServerV2) IsStopped() (res bool) { + return svr.stopped +} + +func NewGrpcServerV2() (svr *GrpcServerV2, err error) { + // server + svr = &GrpcServerV2{ + address: entity.NewAddress(&entity.AddressOptions{ + Host: constants.DefaultGrpcServerHost, + Port: constants.DefaultGrpcServerPort, + }), + } + + if viper.GetString("grpc.server.address") != "" { + svr.address, err = entity.NewAddressFromString(viper.GetString("grpc.server.address")) + if err != nil { + return nil, err + } + } + + svr.nodeCfgSvc = nodeconfig.GetNodeConfigService() + + svr.nodeSvr, err = NewNodeServerV2() + if err != nil { + return nil, err + } + svr.modelBaseServiceSvr = NewModelBaseServiceV2Server() + svr.taskSvr, err = NewTaskServerV2() + if err != nil { + return nil, err + } + svr.dependenciesSvr = NewDependenciesServerV2() + + // recovery options + recoveryOpts := []grpc_recovery.Option{ + grpc_recovery.WithRecoveryHandler(svr.recoveryHandlerFunc), + } + + // grpc server + svr.svr = grpc.NewServer( + grpc_middleware.WithUnaryServerChain( + grpc_recovery.UnaryServerInterceptor(recoveryOpts...), + grpc_auth.UnaryServerInterceptor(middlewares.GetAuthTokenFunc(svr.nodeCfgSvc)), + ), + grpc_middleware.WithStreamServerChain( + grpc_recovery.StreamServerInterceptor(recoveryOpts...), + grpc_auth.StreamServerInterceptor(middlewares.GetAuthTokenFunc(svr.nodeCfgSvc)), + ), + ) + + // initialize + if err := svr.Init(); err != nil { + return nil, err + } + + return svr, nil +} + +var _serverV2 *GrpcServerV2 + +func GetGrpcServerV2() (svr *GrpcServerV2, err error) { + if _serverV2 != nil { + return _serverV2, nil + } + _serverV2, err = NewGrpcServerV2() + if err != nil { + return nil, err + } + return _serverV2, nil +} diff --git a/core/grpc/server/task_server.go b/core/grpc/server/task_server.go new file mode 100644 index 000000000..62b68a5b6 --- /dev/null +++ b/core/grpc/server/task_server.go @@ -0,0 +1,238 @@ +package server + +import ( + "context" + "encoding/json" + "github.com/apex/log" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/notification" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "io" + "strings" +) + +type TaskServer struct { + grpc.UnimplementedTaskServiceServer + + // dependencies + modelSvc service.ModelService + cfgSvc interfaces.NodeConfigService + statsSvc interfaces.TaskStatsService + + // internals + server interfaces.GrpcServer +} + +// Subscribe to task stream when a task runner in a node starts +func (svr TaskServer) Subscribe(stream grpc.TaskService_SubscribeServer) (err error) { + for { + msg, err := stream.Recv() + utils.LogDebug(msg.String()) + if err == io.EOF { + return nil + } + if err != nil { + if strings.HasSuffix(err.Error(), "context canceled") { + return nil + } + trace.PrintError(err) + continue + } + switch msg.Code { + case grpc.StreamMessageCode_INSERT_DATA: + err = svr.handleInsertData(msg) + case grpc.StreamMessageCode_INSERT_LOGS: + err = svr.handleInsertLogs(msg) + default: + err = errors.ErrorGrpcInvalidCode + log.Errorf("invalid stream message code: %d", msg.Code) + continue + } + if err != nil { + log.Errorf("grpc error[%d]: %v", msg.Code, err) + } + } +} + +// Fetch tasks to be executed by a task handler +func (svr TaskServer) Fetch(ctx context.Context, request *grpc.Request) (response *grpc.Response, err error) { + nodeKey := request.GetNodeKey() + if nodeKey == "" { + return nil, trace.TraceError(errors.ErrorGrpcInvalidNodeKey) + } + n, err := svr.modelSvc.GetNodeByKey(nodeKey, nil) + if err != nil { + return nil, trace.TraceError(err) + } + var tid primitive.ObjectID + opts := &mongo.FindOptions{ + Sort: bson.D{ + {"p", 1}, + {"_id", 1}, + }, + Limit: 1, + } + if err := mongo.RunTransactionWithContext(ctx, func(sc mongo2.SessionContext) (err error) { + // get task queue item assigned to this node + tid, err = svr.getTaskQueueItemIdAndDequeue(bson.M{"nid": n.Id}, opts, n.Id) + if err != nil { + return err + } + if !tid.IsZero() { + return nil + } + + // get task queue item assigned to any node (random mode) + tid, err = svr.getTaskQueueItemIdAndDequeue(bson.M{"nid": nil}, opts, n.Id) + if !tid.IsZero() { + return nil + } + if err != nil { + return err + } + return nil + }); err != nil { + return nil, err + } + return HandleSuccessWithData(tid) +} + +func (svr TaskServer) SendNotification(ctx context.Context, request *grpc.Request) (response *grpc.Response, err error) { + svc := notification.GetService() + var t = new(models.Task) + if err := json.Unmarshal(request.Data, t); err != nil { + return nil, trace.TraceError(err) + } + t, err = svr.modelSvc.GetTaskById(t.Id) + if err != nil { + return nil, trace.TraceError(err) + } + td, err := json.Marshal(t) + if err != nil { + return nil, trace.TraceError(err) + } + var e bson.M + if err := json.Unmarshal(td, &e); err != nil { + return nil, trace.TraceError(err) + } + ts, err := svr.modelSvc.GetTaskStatById(t.Id) + if err != nil { + return nil, trace.TraceError(err) + } + settings, _, err := svc.GetSettingList(bson.M{ + "enabled": true, + }, nil, nil) + if err != nil { + return nil, trace.TraceError(err) + } + for _, s := range settings { + switch s.TaskTrigger { + case constants.NotificationTriggerTaskFinish: + if t.Status != constants.TaskStatusPending && t.Status != constants.TaskStatusRunning { + _ = svc.Send(s, e) + } + case constants.NotificationTriggerTaskError: + if t.Status == constants.TaskStatusError || t.Status == constants.TaskStatusAbnormal { + _ = svc.Send(s, e) + } + case constants.NotificationTriggerTaskEmptyResults: + if t.Status != constants.TaskStatusPending && t.Status != constants.TaskStatusRunning { + if ts.ResultCount == 0 { + _ = svc.Send(s, e) + } + } + case constants.NotificationTriggerTaskNever: + } + } + return nil, nil +} + +func (svr TaskServer) handleInsertData(msg *grpc.StreamMessage) (err error) { + data, err := svr.deserialize(msg) + if err != nil { + return err + } + var records []interface{} + for _, d := range data.Records { + res, ok := d[constants.TaskKey] + if ok { + switch res.(type) { + case string: + id, err := primitive.ObjectIDFromHex(res.(string)) + if err == nil { + d[constants.TaskKey] = id + } + } + } + records = append(records, d) + } + return svr.statsSvc.InsertData(data.TaskId, records...) +} + +func (svr TaskServer) handleInsertLogs(msg *grpc.StreamMessage) (err error) { + data, err := svr.deserialize(msg) + if err != nil { + return err + } + return svr.statsSvc.InsertLogs(data.TaskId, data.Logs...) +} + +func (svr TaskServer) getTaskQueueItemIdAndDequeue(query bson.M, opts *mongo.FindOptions, nid primitive.ObjectID) (tid primitive.ObjectID, err error) { + var tq models.TaskQueueItem + if err := mongo.GetMongoCol(interfaces.ModelColNameTaskQueue).Find(query, opts).One(&tq); err != nil { + if err == mongo2.ErrNoDocuments { + return tid, nil + } + return tid, trace.TraceError(err) + } + t, err := svr.modelSvc.GetTaskById(tq.Id) + if err == nil { + t.NodeId = nid + _ = delegate.NewModelDelegate(t).Save() + } + _ = delegate.NewModelDelegate(&tq).Delete() + return tq.Id, nil +} + +func (svr TaskServer) deserialize(msg *grpc.StreamMessage) (data entity.StreamMessageTaskData, err error) { + if err := json.Unmarshal(msg.Data, &data); err != nil { + return data, trace.TraceError(err) + } + if data.TaskId.IsZero() { + return data, trace.TraceError(errors.ErrorGrpcInvalidType) + } + return data, nil +} + +func NewTaskServer() (res *TaskServer, err error) { + // task server + svr := &TaskServer{} + + // dependency injection + if err := container.GetContainer().Invoke(func( + modelSvc service.ModelService, + statsSvc interfaces.TaskStatsService, + cfgSvc interfaces.NodeConfigService, + ) { + svr.modelSvc = modelSvc + svr.statsSvc = statsSvc + svr.cfgSvc = cfgSvc + }); err != nil { + return nil, err + } + + return svr, nil +} diff --git a/core/grpc/server/task_server_v2.go b/core/grpc/server/task_server_v2.go new file mode 100644 index 000000000..62121cb68 --- /dev/null +++ b/core/grpc/server/task_server_v2.go @@ -0,0 +1,237 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "github.com/apex/log" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/core/notification" + "github.com/crawlab-team/crawlab/core/task/stats" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "io" + "strings" +) + +type TaskServerV2 struct { + grpc.UnimplementedTaskServiceServer + + // dependencies + cfgSvc interfaces.NodeConfigService + statsSvc *stats.ServiceV2 + + // internals + server interfaces.GrpcServer +} + +// Subscribe to task stream when a task runner in a node starts +func (svr TaskServerV2) Subscribe(stream grpc.TaskService_SubscribeServer) (err error) { + for { + msg, err := stream.Recv() + utils.LogDebug(msg.String()) + if err == io.EOF { + return nil + } + if err != nil { + if strings.HasSuffix(err.Error(), "context canceled") { + return nil + } + trace.PrintError(err) + continue + } + switch msg.Code { + case grpc.StreamMessageCode_INSERT_DATA: + err = svr.handleInsertData(msg) + case grpc.StreamMessageCode_INSERT_LOGS: + err = svr.handleInsertLogs(msg) + default: + err = errors.New("invalid stream message code") + log.Errorf("invalid stream message code: %d", msg.Code) + continue + } + if err != nil { + log.Errorf("grpc error[%d]: %v", msg.Code, err) + } + } +} + +// Fetch tasks to be executed by a task handler +func (svr TaskServerV2) Fetch(ctx context.Context, request *grpc.Request) (response *grpc.Response, err error) { + nodeKey := request.GetNodeKey() + if nodeKey == "" { + return nil, errors.New("invalid node key") + } + n, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + if err != nil { + return nil, trace.TraceError(err) + } + var tid primitive.ObjectID + opts := &mongo.FindOptions{ + Sort: bson.D{ + {"p", 1}, + {"_id", 1}, + }, + Limit: 1, + } + if err := mongo.RunTransactionWithContext(ctx, func(sc mongo2.SessionContext) (err error) { + // get task queue item assigned to this node + tid, err = svr.getTaskQueueItemIdAndDequeue(bson.M{"nid": n.Id}, opts, n.Id) + if err != nil { + return err + } + if !tid.IsZero() { + return nil + } + + // get task queue item assigned to any node (random mode) + tid, err = svr.getTaskQueueItemIdAndDequeue(bson.M{"nid": nil}, opts, n.Id) + if !tid.IsZero() { + return nil + } + if err != nil { + return err + } + return nil + }); err != nil { + return nil, err + } + return HandleSuccessWithData(tid) +} + +func (svr TaskServerV2) SendNotification(ctx context.Context, request *grpc.Request) (response *grpc.Response, err error) { + svc := notification.GetServiceV2() + var t = new(models.TaskV2) + if err := json.Unmarshal(request.Data, t); err != nil { + return nil, trace.TraceError(err) + } + t, err = service.NewModelServiceV2[models.TaskV2]().GetById(t.Id) + if err != nil { + return nil, trace.TraceError(err) + } + td, err := json.Marshal(t) + if err != nil { + return nil, trace.TraceError(err) + } + var e bson.M + if err := json.Unmarshal(td, &e); err != nil { + return nil, trace.TraceError(err) + } + ts, err := service.NewModelServiceV2[models.TaskStatV2]().GetById(t.Id) + if err != nil { + return nil, trace.TraceError(err) + } + settings, _, err := svc.GetSettingList(bson.M{ + "enabled": true, + }, nil, nil) + if err != nil { + return nil, trace.TraceError(err) + } + for _, s := range settings { + switch s.TaskTrigger { + case constants.NotificationTriggerTaskFinish: + if t.Status != constants.TaskStatusPending && t.Status != constants.TaskStatusRunning { + _ = svc.Send(&s, e) + } + case constants.NotificationTriggerTaskError: + if t.Status == constants.TaskStatusError || t.Status == constants.TaskStatusAbnormal { + _ = svc.Send(&s, e) + } + case constants.NotificationTriggerTaskEmptyResults: + if t.Status != constants.TaskStatusPending && t.Status != constants.TaskStatusRunning { + if ts.ResultCount == 0 { + _ = svc.Send(&s, e) + } + } + case constants.NotificationTriggerTaskNever: + } + } + return nil, nil +} + +func (svr TaskServerV2) handleInsertData(msg *grpc.StreamMessage) (err error) { + data, err := svr.deserialize(msg) + if err != nil { + return err + } + var records []interface{} + for _, d := range data.Records { + res, ok := d[constants.TaskKey] + if ok { + switch res.(type) { + case string: + id, err := primitive.ObjectIDFromHex(res.(string)) + if err == nil { + d[constants.TaskKey] = id + } + } + } + records = append(records, d) + } + return svr.statsSvc.InsertData(data.TaskId, records...) +} + +func (svr TaskServerV2) handleInsertLogs(msg *grpc.StreamMessage) (err error) { + data, err := svr.deserialize(msg) + if err != nil { + return err + } + return svr.statsSvc.InsertLogs(data.TaskId, data.Logs...) +} + +func (svr TaskServerV2) getTaskQueueItemIdAndDequeue(query bson.M, opts *mongo.FindOptions, nid primitive.ObjectID) (tid primitive.ObjectID, err error) { + tq, err := service.NewModelServiceV2[models.TaskQueueItemV2]().GetOne(query, opts) + if err != nil { + if errors.Is(err, mongo2.ErrNoDocuments) { + return tid, nil + } + return tid, trace.TraceError(err) + } + t, err := service.NewModelServiceV2[models.TaskV2]().GetById(tq.Id) + if err == nil { + t.NodeId = nid + err = service.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t) + if err != nil { + return tid, trace.TraceError(err) + } + } + err = service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteById(tq.Id) + if err != nil { + return tid, trace.TraceError(err) + } + return tq.Id, nil +} + +func (svr TaskServerV2) deserialize(msg *grpc.StreamMessage) (data entity.StreamMessageTaskData, err error) { + if err := json.Unmarshal(msg.Data, &data); err != nil { + return data, trace.TraceError(err) + } + if data.TaskId.IsZero() { + return data, errors.New("invalid task id") + } + return data, nil +} + +func NewTaskServerV2() (res *TaskServerV2, err error) { + // task server + svr := &TaskServerV2{} + + svr.cfgSvc = nodeconfig.GetNodeConfigService() + + svr.statsSvc, err = stats.GetTaskStatsServiceV2() + if err != nil { + return nil, err + } + + return svr, nil +} diff --git a/core/grpc/server/utils_handle.go b/core/grpc/server/utils_handle.go new file mode 100644 index 000000000..c44ac20f0 --- /dev/null +++ b/core/grpc/server/utils_handle.go @@ -0,0 +1,53 @@ +package server + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" +) + +func HandleError(err error) (res *grpc.Response, err2 error) { + trace.PrintError(err) + return &grpc.Response{ + Code: grpc.ResponseCode_ERROR, + Error: err.Error(), + }, err +} + +func HandleSuccess() (res *grpc.Response, err error) { + return &grpc.Response{ + Code: grpc.ResponseCode_OK, + Message: "success", + }, nil +} + +func HandleSuccessWithData(data interface{}) (res *grpc.Response, err error) { + var bytes []byte + switch data.(type) { + case []byte: + bytes = data.([]byte) + default: + bytes, err = json.Marshal(data) + if err != nil { + return HandleError(err) + } + } + return &grpc.Response{ + Code: grpc.ResponseCode_OK, + Message: "success", + Data: bytes, + }, nil +} + +func HandleSuccessWithListData(data interface{}, total int) (res *grpc.Response, err error) { + bytes, err := json.Marshal(data) + if err != nil { + return HandleError(err) + } + return &grpc.Response{ + Code: grpc.ResponseCode_OK, + Message: "success", + Data: bytes, + Total: int64(total), + }, nil +} diff --git a/core/grpc/test/auth_token_test.go b/core/grpc/test/auth_token_test.go new file mode 100644 index 000000000..fa09e976a --- /dev/null +++ b/core/grpc/test/auth_token_test.go @@ -0,0 +1,96 @@ +package test + +import ( + "context" + "encoding/json" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/grpc/client" + "github.com/crawlab-team/crawlab/core/grpc/server" + "github.com/crawlab-team/crawlab/core/node/config" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/stretchr/testify/require" + "io/ioutil" + "os" + "path" + "testing" +) + +func TestAuthToken(t *testing.T) { + var err error + + // auth key + authKey := "test-auth-key" + + // auth key (invalid) + authKeyInvalid := "test-auth-key-invalid" + + // tmp dir + tmpDir := os.TempDir() + + // master config + masterConfigPath := path.Join(tmpDir, "config-master.json") + masterConfig := config.Config{ + Key: "master", + IsMaster: true, + AuthKey: authKey, + } + masterConfigData, err := json.Marshal(&masterConfig) + require.Nil(t, err) + err = ioutil.WriteFile(masterConfigPath, masterConfigData, os.FileMode(0777)) + + // worker config + workerConfigPath := path.Join(tmpDir, "config-worker.json") + workerConfig := config.Config{ + Key: "worker", + IsMaster: false, + AuthKey: authKey, + } + workerConfigData, err := json.Marshal(&workerConfig) + require.Nil(t, err) + err = ioutil.WriteFile(workerConfigPath, workerConfigData, os.FileMode(0777)) + + // worker config (invalid) + workerInvalidConfigPath := path.Join(tmpDir, "worker-invalid") + workerInvalidConfig := config.Config{ + Key: "worker", + IsMaster: false, + AuthKey: authKeyInvalid, + } + workerInvalidConfigData, err := json.Marshal(&workerInvalidConfig) + require.Nil(t, err) + err = ioutil.WriteFile(workerInvalidConfigPath, workerInvalidConfigData, os.FileMode(0777)) + + // server + svr, err := server.NewServer( + server.WithConfigPath(masterConfigPath), + server.WithAddress(entity.NewAddress(&entity.AddressOptions{ + Host: "0.0.0.0", + Port: "9999", + })), + ) + require.Nil(t, err) + err = svr.Start() + require.Nil(t, err) + + // client + c, err := client.GetClient(workerConfigPath, client.WithAddress(entity.NewAddress(&entity.AddressOptions{ + Host: "localhost", + Port: "9999", + }))) + require.Nil(t, err) + err = c.Start() + require.Nil(t, err) + _, err = c.GetNodeClient().Ping(context.Background(), &grpc.Request{NodeKey: workerConfig.Key}) + require.Nil(t, err) + + // client (invalid) + ci, err := client.GetClient(workerInvalidConfigPath, client.WithAddress(entity.NewAddress(&entity.AddressOptions{ + Host: "localhost", + Port: "9999", + }))) + require.Nil(t, err) + err = ci.Start() + require.Nil(t, err) + _, err = ci.GetNodeClient().Ping(context.Background(), &grpc.Request{NodeKey: workerInvalidConfig.Key}) + require.NotNil(t, err) +} diff --git a/core/grpc/test/base.go b/core/grpc/test/base.go new file mode 100644 index 000000000..e8e3923ea --- /dev/null +++ b/core/grpc/test/base.go @@ -0,0 +1,77 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/grpc/client" + "github.com/crawlab-team/crawlab/core/grpc/server" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/node/test" + "testing" + "time" +) + +type Test struct { + Server interfaces.GrpcServer + Client interfaces.GrpcClient + + MasterNodeInfo *entity.NodeInfo + WorkerNodeInfo *entity.NodeInfo +} + +func (t *Test) Setup(t2 *testing.T) { + test.T.Cleanup() + t2.Cleanup(t.Cleanup) + if !T.Client.IsStarted() { + _ = T.Client.Start() + } else if T.Client.IsClosed() { + _ = T.Client.Restart() + } +} + +func (t *Test) Cleanup() { + _ = t.Client.Stop() + _ = t.Server.Stop() + test.T.Cleanup() + + // wait to avoid caching + time.Sleep(200 * time.Millisecond) +} + +var T *Test + +func NewTest() (res *Test, err error) { + // test + t := &Test{} + + // server + t.Server, err = server.NewServer( + server.WithConfigPath(test.T.MasterSvc.GetConfigPath()), + server.WithAddress(test.T.MasterSvc.GetAddress()), + ) + if err != nil { + return nil, err + } + if err := t.Server.Start(); err != nil { + return nil, err + } + + // client + t.Client, err = client.GetClient(test.T.WorkerSvc.GetConfigPath()) + if err != nil { + return nil, err + } + + // master node info + t.MasterNodeInfo = &entity.NodeInfo{ + Key: "master", + IsMaster: true, + } + + // worker node info + t.WorkerNodeInfo = &entity.NodeInfo{ + Key: "worker", + IsMaster: false, + } + + return t, nil +} diff --git a/core/grpc/test/main_test.go b/core/grpc/test/main_test.go new file mode 100644 index 000000000..232a8c6e8 --- /dev/null +++ b/core/grpc/test/main_test.go @@ -0,0 +1,7 @@ +package test + +import "testing" + +func TestMain(m *testing.M) { + m.Run() +} diff --git a/core/grpc/test/model_base_service_server_test.go b/core/grpc/test/model_base_service_server_test.go new file mode 100644 index 000000000..ad9b53724 --- /dev/null +++ b/core/grpc/test/model_base_service_server_test.go @@ -0,0 +1,312 @@ +package test + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/node/test" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "testing" +) + +func TestModelBaseService_GetById(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // add + modelDelegateAdd(t) + p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) + require.Nil(t, err) + + // get by id + ctx, cancel := T.Client.Context() + defer cancel() + req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Id: p.Id}) + require.Nil(t, err) + res, err := T.Client.GetModelBaseServiceClient().GetById(ctx, req) + require.Nil(t, err) + var p2 models.Project + err = json.Unmarshal(res.Data, &p2) + require.Nil(t, err) + + // validate + require.Equal(t, p.Id, p2.Id) + require.Equal(t, p.Name, p2.Name) + require.Equal(t, p.Description, p2.Description) +} + +func TestModelBaseService_Get(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // add + modelDelegateAdd(t) + p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) + require.Nil(t, err) + + // get + ctx, cancel := T.Client.Context() + defer cancel() + req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) + require.Nil(t, err) + res, err := T.Client.GetModelBaseServiceClient().Get(ctx, req) + require.Nil(t, err) + var p2 models.Project + err = json.Unmarshal(res.Data, &p2) + require.Nil(t, err) + + // validate + require.Equal(t, p.Id, p2.Id) + require.Equal(t, p.Name, p2.Name) + require.Equal(t, p.Description, p2.Description) +} + +func TestModelBaseService_GetList(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // add + n := 10 + for i := 0; i < n; i++ { + modelDelegateAdd(t) + } + + // get list + ctx, cancel := T.Client.Context() + defer cancel() + req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) + require.Nil(t, err) + res, err := T.Client.GetModelBaseServiceClient().GetList(ctx, req) + require.Nil(t, err) + var list []models.Project + err = json.Unmarshal(res.Data, &list) + require.Nil(t, err) + + // validate + require.Equal(t, n, len(list)) +} + +func TestModelBaseService_DeleteById(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // add + modelDelegateAdd(t) + p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) + require.Nil(t, err) + + // delete by id + ctx, cancel := T.Client.Context() + defer cancel() + req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Id: p.Id}) + require.Nil(t, err) + _, err = T.Client.GetModelBaseServiceClient().DeleteById(ctx, req) + require.Nil(t, err) + + // validate + p, err = test.T.ModelSvc.GetProjectById(p.Id) + require.Equal(t, mongo2.ErrNoDocuments, err) +} + +func TestModelBaseService_Delete(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // add + modelDelegateAdd(t) + p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) + require.Nil(t, err) + + // delete by id + ctx, cancel := T.Client.Context() + defer cancel() + req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) + require.Nil(t, err) + _, err = T.Client.GetModelBaseServiceClient().Delete(ctx, req) + require.Nil(t, err) + + // validate + p, err = test.T.ModelSvc.GetProjectById(p.Id) + require.Equal(t, mongo2.ErrNoDocuments, err) +} + +func TestModelBaseService_DeleteList(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // add + n := 10 + for i := 0; i < n; i++ { + modelDelegateAdd(t) + } + + // delete by id + ctx, cancel := T.Client.Context() + defer cancel() + req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) + require.Nil(t, err) + _, err = T.Client.GetModelBaseServiceClient().DeleteList(ctx, req) + require.Nil(t, err) + + // validate + list, err := test.T.ModelSvc.GetProjectList(bson.M{"name": "test-project"}, nil) + require.Nil(t, err) + require.Equal(t, 0, len(list)) +} + +func TestModelBaseService_ForceDeleteList(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // add + n := 10 + for i := 0; i < n; i++ { + modelDelegateAdd(t) + } + + // delete by id + ctx, cancel := T.Client.Context() + defer cancel() + req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) + require.Nil(t, err) + _, err = T.Client.GetModelBaseServiceClient().ForceDeleteList(ctx, req) + require.Nil(t, err) + + // validate + list, err := test.T.ModelSvc.GetProjectList(bson.M{"name": "test-project"}, nil) + require.Nil(t, err) + require.Equal(t, 0, len(list)) +} + +func TestModelBaseService_UpdateById(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // add + modelDelegateAdd(t) + p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) + require.Nil(t, err) + + // update by id + ctx, cancel := T.Client.Context() + defer cancel() + update := bson.M{ + "name": "test-new-project", + } + req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Id: p.Id, Update: update}) + require.Nil(t, err) + _, err = T.Client.GetModelBaseServiceClient().UpdateById(ctx, req) + require.Nil(t, err) + + // validate + p2, err := test.T.ModelSvc.GetProjectById(p.Id) + require.Nil(t, err) + require.Equal(t, "test-new-project", p2.Name) +} + +func TestModelBaseService_Update(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // add + n := 10 + for i := 0; i < n; i++ { + modelDelegateAdd(t) + } + + // update + ctx, cancel := T.Client.Context() + defer cancel() + update := bson.M{ + "name": "test-new-project", + } + req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}, Update: update}) + require.Nil(t, err) + _, err = T.Client.GetModelBaseServiceClient().Update(ctx, req) + require.Nil(t, err) + + // validate + list, err := test.T.ModelSvc.GetProjectList(bson.M{"name": "test-project"}, nil) + require.Nil(t, err) + require.Equal(t, 0, len(list)) + list, err = test.T.ModelSvc.GetProjectList(bson.M{"name": "test-new-project"}, nil) + require.Nil(t, err) + require.Equal(t, n, len(list)) +} + +func TestModelBaseService_Insert(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // insert + var docs []interface{} + n := 10 + for i := 0; i < n; i++ { + docs = append(docs, models.Project{ + Id: primitive.NewObjectID(), + Name: "test-project", + }) + } + ctx, cancel := T.Client.Context() + defer cancel() + req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Docs: docs}) + require.Nil(t, err) + _, err = T.Client.GetModelBaseServiceClient().Insert(ctx, req) + require.Nil(t, err) + + // validate + list, err := test.T.ModelSvc.GetProjectList(bson.M{"name": "test-project"}, nil) + require.Nil(t, err) + require.Equal(t, n, len(list)) +} + +func TestModelBaseService_Count(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // add + n := 10 + for i := 0; i < n; i++ { + modelDelegateAdd(t) + } + + // count + ctx, cancel := T.Client.Context() + defer cancel() + req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) + require.Nil(t, err) + res, err := T.Client.GetModelBaseServiceClient().Count(ctx, req) + require.Nil(t, err) + + // validate + var total int + err = json.Unmarshal(res.Data, &total) + require.Nil(t, err) + require.Equal(t, n, total) +} diff --git a/core/grpc/test/model_delegate_server_test.go b/core/grpc/test/model_delegate_server_test.go new file mode 100644 index 000000000..c761b9b68 --- /dev/null +++ b/core/grpc/test/model_delegate_server_test.go @@ -0,0 +1,147 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/client" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/node/test" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "testing" +) + +func TestModelDelegate_Do(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // add + modelDelegateAdd(t) + project, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) + require.Nil(t, err) + + // get artifact + a := modelDelegateGetArtifact(t) + require.Equal(t, project.GetId(), a.GetId()) + + // save + modelDelegateSave(t) + project, err = test.T.ModelSvc.GetProject(bson.M{"name": "test-new-project"}, nil) + require.Nil(t, err) + require.Equal(t, "test-new-project", project.Name) + require.Equal(t, "test-new-description", project.Description) + + // delete + modelDelegateDelete(t) + _, err = test.T.ModelSvc.GetProject(bson.M{"name": "test-new-project"}, nil) + require.Equal(t, mongo2.ErrNoDocuments, err) +} + +func TestModelDelegate_Do_All(t *testing.T) { + T, _ = NewTest() + T.Setup(t) + + // add + modelDelegateAddAll(t) + modelDelegateValidateAddAll(t) +} + +func modelDelegateAdd(t *testing.T) { + // modelDelegateAdd + project := &models.Project{ + Name: "test-project", + Description: "test-description", + } + projectD := client.NewModelDelegate(project, client.WithDelegateConfigPath(T.Client.GetConfigPath())) + err := projectD.Add() + require.Nil(t, err) +} + +func modelDelegateGetArtifact(t *testing.T) interfaces.ModelArtifact { + project, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) + require.Nil(t, err) + + projectD := client.NewModelDelegate(project, client.WithDelegateConfigPath(T.Client.GetConfigPath())) + a, err := projectD.GetArtifact() + require.Nil(t, err) + return a +} + +func modelDelegateSave(t *testing.T) { + project, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) + require.Nil(t, err) + + project.Name = "test-new-project" + project.Description = "test-new-description" + + projectD := client.NewModelDelegate(project, client.WithDelegateConfigPath(T.Client.GetConfigPath())) + err = projectD.Save() + require.Nil(t, err) +} + +func modelDelegateDelete(t *testing.T) { + project, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-new-project"}, nil) + require.Nil(t, err) + + projectD := client.NewModelDelegate(project, client.WithDelegateConfigPath(T.Client.GetConfigPath())) + err = projectD.Delete() + require.Nil(t, err) +} + +func modelDelegateAddAll(t *testing.T) { + var err error + cfgOpt := client.WithDelegateConfigPath(T.Client.GetConfigPath()) + m := models.NewModelMap() + err = client.NewModelDelegate(&m.Tag, cfgOpt).Add() + require.Nil(t, err) + err = client.NewModelDelegate(&m.Node, cfgOpt).Add() + require.Nil(t, err) + err = client.NewModelDelegate(&m.Project, cfgOpt).Add() + require.Nil(t, err) + err = client.NewModelDelegate(&m.Spider, cfgOpt).Add() + require.Nil(t, err) + err = client.NewModelDelegate(&m.Task, cfgOpt).Add() + require.Nil(t, err) + err = client.NewModelDelegate(&m.Job, cfgOpt).Add() + require.Nil(t, err) + err = client.NewModelDelegate(&m.Schedule, cfgOpt).Add() + require.Nil(t, err) + err = client.NewModelDelegate(&m.User, cfgOpt).Add() + require.Nil(t, err) + err = client.NewModelDelegate(&m.Setting, cfgOpt).Add() + require.Nil(t, err) + err = client.NewModelDelegate(&m.Token, cfgOpt).Add() + require.Nil(t, err) + err = client.NewModelDelegate(&m.Variable, cfgOpt).Add() + require.Nil(t, err) + err = client.NewModelDelegate(&m.User, cfgOpt).Add() + require.Nil(t, err) +} + +func modelDelegateValidateAddAll(t *testing.T) { + var err error + _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) + require.Nil(t, err) + _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) + require.Nil(t, err) + _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) + require.Nil(t, err) + _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) + require.Nil(t, err) + _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) + require.Nil(t, err) + _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) + require.Nil(t, err) + _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) + require.Nil(t, err) + _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) + require.Nil(t, err) + _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) + require.Nil(t, err) + _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) + require.Nil(t, err) + _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) + require.Nil(t, err) +} diff --git a/core/grpc/test/node_server_test.go b/core/grpc/test/node_server_test.go new file mode 100644 index 000000000..de858bed9 --- /dev/null +++ b/core/grpc/test/node_server_test.go @@ -0,0 +1,140 @@ +package test + +import ( + "context" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/node/test" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestGrpcServer_Register(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // register + register(t) + + // validate + workerNodeKey := T.WorkerNodeInfo.Key + workerNode, err := test.T.ModelSvc.GetNodeByKey(workerNodeKey, nil) + require.Nil(t, err) + require.Equal(t, workerNodeKey, workerNode.Key) + require.Equal(t, constants.NodeStatusRegistered, workerNode.Status) +} + +func TestGrpcServer_Register_Existing(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // add to db + node := &models.Node{ + Key: T.WorkerNodeInfo.Key, + IsMaster: false, + Status: constants.NodeStatusUnregistered, + } + nodeD := delegate.NewModelDelegate(node) + err = nodeD.Add() + require.Nil(t, err) + + // register + register(t) + + // validate + workerNodeKey := T.WorkerNodeInfo.Key + workerNode, err := test.T.ModelSvc.GetNodeByKey(workerNodeKey, nil) + require.Nil(t, err) + require.Equal(t, workerNodeKey, workerNode.Key) + require.Equal(t, constants.NodeStatusRegistered, workerNode.Status) +} + +func TestGrpcServer_SendHeartbeat(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // register + register(t) + + // send heartbeat + sendHeartbeat(t) + + // validate + workerNodeKey := T.WorkerNodeInfo.Key + workerNode, err := test.T.ModelSvc.GetNodeByKey(workerNodeKey, nil) + require.Nil(t, err) + require.Equal(t, workerNodeKey, workerNode.Key) + require.Equal(t, constants.NodeStatusOnline, workerNode.Status) +} + +func TestGrpcServer_Subscribe(t *testing.T) { + var err error + + T, _ = NewTest() + T.Setup(t) + + // register + register(t) + + // handle client message + go handleClientMessage(t) + + time.Sleep(1 * time.Second) + + // server PING client + sub, err := T.Server.GetSubscribe("node:" + T.WorkerNodeInfo.Key) + require.Nil(t, err) + require.NotNil(t, sub) + err = sub.GetStream().Send(&grpc.StreamMessage{ + Code: grpc.StreamMessageCode_PING, + NodeKey: T.MasterNodeInfo.Key, + }) + require.Nil(t, err) + + // wait + time.Sleep(1 * time.Second) + + // validate + workerNode, err := test.T.ModelSvc.GetNodeByKey(T.WorkerNodeInfo.Key, nil) + require.Nil(t, err) + require.Equal(t, constants.NodeStatusOnline, workerNode.Status) +} + +func register(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + res, err := T.Client.GetNodeClient().Register(ctx, T.Client.NewRequest(T.WorkerNodeInfo)) + require.Nil(t, err) + require.Equal(t, grpc.ResponseCode_OK, res.Code) +} + +func sendHeartbeat(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + res, err := T.Client.GetNodeClient().SendHeartbeat(ctx, T.Client.NewRequest(T.WorkerNodeInfo)) + require.Nil(t, err) + require.Equal(t, grpc.ResponseCode_OK, res.Code) +} + +func handleClientMessage(t *testing.T) { + msgCh := T.Client.GetMessageChannel() + for { + msg := <-msgCh + switch msg.Code { + case grpc.StreamMessageCode_PING: + require.Equal(t, T.MasterNodeInfo.Key, msg.NodeKey) + res, err := T.Client.GetNodeClient().SendHeartbeat(context.Background(), T.Client.NewRequest(T.WorkerNodeInfo)) + require.Nil(t, err) + require.NotNil(t, res) + } + } +} diff --git a/core/i18n/service.go b/core/i18n/service.go new file mode 100644 index 000000000..ac7174756 --- /dev/null +++ b/core/i18n/service.go @@ -0,0 +1,43 @@ +package i18n + +import "github.com/crawlab-team/crawlab/core/interfaces" + +var translations []interfaces.Translation + +var _svc interfaces.I18nService + +type Service struct { +} + +func (svc *Service) AddTranslations(t []interfaces.Translation) { + translations = append(translations, t...) +} + +func (svc *Service) GetTranslations() (t []interfaces.Translation) { + return translations +} + +func GetI18nService(cfgPath string) (svc2 interfaces.I18nService, err error) { + if _svc != nil { + return _svc, nil + } + + _svc, err = NewI18nService() + if err != nil { + return nil, err + } + + return _svc, nil +} + +func ProvideGetI18nService(cfgPath string) func() (svc interfaces.I18nService, err error) { + return func() (svc interfaces.I18nService, err error) { + return GetI18nService(cfgPath) + } +} + +func NewI18nService() (svc2 interfaces.I18nService, err error) { + svc := &Service{} + + return svc, nil +} diff --git a/core/interfaces/address.go b/core/interfaces/address.go new file mode 100644 index 000000000..c80dd3457 --- /dev/null +++ b/core/interfaces/address.go @@ -0,0 +1,7 @@ +package interfaces + +type Address interface { + Entity + String() string + IsEmpty() bool +} diff --git a/core/interfaces/color.go b/core/interfaces/color.go new file mode 100644 index 000000000..7f10fbc51 --- /dev/null +++ b/core/interfaces/color.go @@ -0,0 +1,7 @@ +package interfaces + +type Color interface { + Entity + GetHex() string + GetName() string +} diff --git a/core/interfaces/color_service.go b/core/interfaces/color_service.go new file mode 100644 index 000000000..68bba285c --- /dev/null +++ b/core/interfaces/color_service.go @@ -0,0 +1,7 @@ +package interfaces + +type ColorService interface { + Injectable + GetByName(name string) (res Color, err error) + GetRandom() (res Color, err error) +} diff --git a/core/interfaces/controller_params.go b/core/interfaces/controller_params.go new file mode 100644 index 000000000..986af8922 --- /dev/null +++ b/core/interfaces/controller_params.go @@ -0,0 +1,6 @@ +package interfaces + +type ControllerParams interface { + IsZero() (ok bool) + IsDefault() (ok bool) +} diff --git a/core/interfaces/data_source_service.go b/core/interfaces/data_source_service.go new file mode 100644 index 000000000..f5309dfa8 --- /dev/null +++ b/core/interfaces/data_source_service.go @@ -0,0 +1,14 @@ +package interfaces + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type DataSourceService interface { + ChangePassword(id primitive.ObjectID, password string) (err error) + Monitor() + CheckStatus(id primitive.ObjectID) (err error) + SetTimeout(duration time.Duration) + SetMonitorInterval(duration time.Duration) +} diff --git a/core/interfaces/entity.go b/core/interfaces/entity.go new file mode 100644 index 000000000..75bab4714 --- /dev/null +++ b/core/interfaces/entity.go @@ -0,0 +1,5 @@ +package interfaces + +type Entity interface { + Value() interface{} +} diff --git a/core/interfaces/event_data.go b/core/interfaces/event_data.go new file mode 100644 index 000000000..12f790ea6 --- /dev/null +++ b/core/interfaces/event_data.go @@ -0,0 +1,6 @@ +package interfaces + +type EventData interface { + GetEvent() string + GetData() interface{} +} diff --git a/core/interfaces/event_service.go b/core/interfaces/event_service.go new file mode 100644 index 000000000..35bf05053 --- /dev/null +++ b/core/interfaces/event_service.go @@ -0,0 +1,9 @@ +package interfaces + +type EventFn func(data ...interface{}) (err error) + +type EventService interface { + Register(key, include, exclude string, ch *chan EventData) + Unregister(key string) + SendEvent(eventName string, data ...interface{}) +} diff --git a/core/interfaces/export.go b/core/interfaces/export.go new file mode 100644 index 000000000..2205b8428 --- /dev/null +++ b/core/interfaces/export.go @@ -0,0 +1,14 @@ +package interfaces + +import "time" + +type Export interface { + GetId() string + GetType() string + GetTarget() string + GetFilter() Filter + GetStatus() string + GetStartTs() time.Time + GetEndTs() time.Time + GetDownloadPath() string +} diff --git a/core/interfaces/export_service.go b/core/interfaces/export_service.go new file mode 100644 index 000000000..d48ae2b8c --- /dev/null +++ b/core/interfaces/export_service.go @@ -0,0 +1,7 @@ +package interfaces + +type ExportService interface { + GenerateId() (exportId string, err error) + Export(exportType, target string, filter Filter) (exportId string, err error) + GetExport(exportId string) (export Export, err error) +} diff --git a/core/interfaces/filter.go b/core/interfaces/filter.go new file mode 100644 index 000000000..ebf49eb99 --- /dev/null +++ b/core/interfaces/filter.go @@ -0,0 +1,9 @@ +package interfaces + +type Filter interface { + GetIsOr() (isOr bool) + SetIsOr(isOr bool) + GetConditions() (conditions []FilterCondition) + SetConditions(conditions []FilterCondition) + IsNil() (ok bool) +} diff --git a/core/interfaces/filter_condition.go b/core/interfaces/filter_condition.go new file mode 100644 index 000000000..8f91d3717 --- /dev/null +++ b/core/interfaces/filter_condition.go @@ -0,0 +1,10 @@ +package interfaces + +type FilterCondition interface { + GetKey() (key string) + SetKey(key string) + GetOp() (op string) + SetOp(op string) + GetValue() (value interface{}) + SetValue(value interface{}) +} diff --git a/core/interfaces/fs_file_info.go b/core/interfaces/fs_file_info.go new file mode 100644 index 000000000..812dfd714 --- /dev/null +++ b/core/interfaces/fs_file_info.go @@ -0,0 +1,19 @@ +package interfaces + +import ( + "os" + "time" +) + +type FsFileInfo interface { + GetName() string + GetPath() string + GetFullPath() string + GetExtension() string + GetIsDir() bool + GetFileSize() int64 + GetModTime() time.Time + GetMode() os.FileMode + GetHash() string + GetChildren() []FsFileInfo +} diff --git a/core/interfaces/fs_service.go b/core/interfaces/fs_service.go new file mode 100644 index 000000000..f856c978a --- /dev/null +++ b/core/interfaces/fs_service.go @@ -0,0 +1,28 @@ +package interfaces + +import ( + cfs "github.com/crawlab-team/crawlab-fs" + vcs "github.com/crawlab-team/crawlab-vcs" +) + +type FsService interface { + WithConfigPath + List(path string, opts ...ServiceCrudOption) (files []FsFileInfo, err error) + GetFile(path string, opts ...ServiceCrudOption) (data []byte, err error) + GetFileInfo(path string, opts ...ServiceCrudOption) (file FsFileInfo, err error) + Save(path string, data []byte, opts ...ServiceCrudOption) (err error) + Rename(path, newPath string, opts ...ServiceCrudOption) (err error) + Delete(path string, opts ...ServiceCrudOption) (err error) + Copy(path, newPath string, opts ...ServiceCrudOption) (err error) + Commit(msg string) (err error) + SyncToFs(opts ...ServiceCrudOption) (err error) + SyncToWorkspace() (err error) + GetFsPath() (path string) + SetFsPath(path string) + GetWorkspacePath() (path string) + SetWorkspacePath(path string) + GetRepoPath() (path string) + SetRepoPath(path string) + GetFs() (fs cfs.Manager) + GetGitClient() (c *vcs.GitClient) +} diff --git a/core/interfaces/fs_service_options.go b/core/interfaces/fs_service_options.go new file mode 100644 index 000000000..fa50f8b3b --- /dev/null +++ b/core/interfaces/fs_service_options.go @@ -0,0 +1,21 @@ +package interfaces + +type ServiceCrudOptions struct { + IsAbsolute bool // whether the path is absolute + OnlyFromWorkspace bool // whether only sync from workspace + NotSyncToWorkspace bool // whether not sync to workspace +} + +type ServiceCrudOption func(o *ServiceCrudOptions) + +func WithOnlyFromWorkspace() ServiceCrudOption { + return func(o *ServiceCrudOptions) { + o.OnlyFromWorkspace = true + } +} + +func WithNotSyncToWorkspace() ServiceCrudOption { + return func(o *ServiceCrudOptions) { + o.NotSyncToWorkspace = true + } +} diff --git a/core/interfaces/fs_service_v2.go b/core/interfaces/fs_service_v2.go new file mode 100644 index 000000000..4bc3d7df6 --- /dev/null +++ b/core/interfaces/fs_service_v2.go @@ -0,0 +1,12 @@ +package interfaces + +type FsServiceV2 interface { + List(path string) (files []FsFileInfo, err error) + GetFile(path string) (data []byte, err error) + GetFileInfo(path string) (file FsFileInfo, err error) + Save(path string, data []byte) (err error) + CreateDir(path string) (err error) + Rename(path, newPath string) (err error) + Delete(path string) (err error) + Copy(path, newPath string) (err error) +} diff --git a/core/interfaces/grpc_base.go b/core/interfaces/grpc_base.go new file mode 100644 index 000000000..4ab5f5464 --- /dev/null +++ b/core/interfaces/grpc_base.go @@ -0,0 +1,9 @@ +package interfaces + +type GrpcBase interface { + WithConfigPath + Init() (err error) + Start() (err error) + Stop() (err error) + Register() (err error) +} diff --git a/core/interfaces/grpc_base_service_params.go b/core/interfaces/grpc_base_service_params.go new file mode 100644 index 000000000..239c1b6ce --- /dev/null +++ b/core/interfaces/grpc_base_service_params.go @@ -0,0 +1,5 @@ +package interfaces + +type GrpcBaseServiceParams interface { + Entity +} diff --git a/core/interfaces/grpc_client.go b/core/interfaces/grpc_client.go new file mode 100644 index 000000000..12fa2e901 --- /dev/null +++ b/core/interfaces/grpc_client.go @@ -0,0 +1,30 @@ +package interfaces + +import ( + "context" + grpc "github.com/crawlab-team/crawlab/grpc" + "time" +) + +type GrpcClient interface { + GrpcBase + WithConfigPath + GetModelDelegateClient() grpc.ModelDelegateClient + GetModelBaseServiceClient() grpc.ModelBaseServiceClient + GetNodeClient() grpc.NodeServiceClient + GetTaskClient() grpc.TaskServiceClient + GetMessageClient() grpc.MessageServiceClient + SetAddress(Address) + SetTimeout(time.Duration) + SetSubscribeType(string) + SetHandleMessage(bool) + Context() (context.Context, context.CancelFunc) + NewRequest(interface{}) *grpc.Request + GetMessageChannel() chan *grpc.StreamMessage + Restart() error + NewModelBaseServiceRequest(ModelId, GrpcBaseServiceParams) (*grpc.Request, error) + IsStarted() bool + IsClosed() bool + Err() error + GetStream() grpc.NodeService_SubscribeClient +} diff --git a/core/interfaces/grpc_client_model_base_service.go b/core/interfaces/grpc_client_model_base_service.go new file mode 100644 index 000000000..87b702e41 --- /dev/null +++ b/core/interfaces/grpc_client_model_base_service.go @@ -0,0 +1,7 @@ +package interfaces + +type GrpcClientModelBaseService interface { + WithModelId + WithConfigPath + ModelBaseService +} diff --git a/core/interfaces/grpc_client_model_delegate.go b/core/interfaces/grpc_client_model_delegate.go new file mode 100644 index 000000000..b45e9e2c5 --- /dev/null +++ b/core/interfaces/grpc_client_model_delegate.go @@ -0,0 +1,7 @@ +package interfaces + +type GrpcClientModelDelegate interface { + ModelDelegate + WithConfigPath + Close() error +} diff --git a/core/interfaces/grpc_client_model_environment_service.go b/core/interfaces/grpc_client_model_environment_service.go new file mode 100644 index 000000000..9b7c07af7 --- /dev/null +++ b/core/interfaces/grpc_client_model_environment_service.go @@ -0,0 +1,14 @@ +package interfaces + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type GrpcClientModelEnvironmentService interface { + ModelBaseService + GetEnvironmentById(id primitive.ObjectID) (s Environment, err error) + GetEnvironment(query bson.M, opts *mongo.FindOptions) (s Environment, err error) + GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []Environment, err error) +} diff --git a/core/interfaces/grpc_client_model_node_service.go b/core/interfaces/grpc_client_model_node_service.go new file mode 100644 index 000000000..899868a14 --- /dev/null +++ b/core/interfaces/grpc_client_model_node_service.go @@ -0,0 +1,15 @@ +package interfaces + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type GrpcClientModelNodeService interface { + ModelBaseService + GetNodeById(id primitive.ObjectID) (n Node, err error) + GetNode(query bson.M, opts *mongo.FindOptions) (n Node, err error) + GetNodeByKey(key string) (n Node, err error) + GetNodeList(query bson.M, opts *mongo.FindOptions) (res []Node, err error) +} diff --git a/core/interfaces/grpc_client_model_service.go b/core/interfaces/grpc_client_model_service.go new file mode 100644 index 000000000..9e79c96f0 --- /dev/null +++ b/core/interfaces/grpc_client_model_service.go @@ -0,0 +1,6 @@ +package interfaces + +type GrpcClientModelService interface { + WithConfigPath + NewBaseServiceDelegate(id ModelId) (GrpcClientModelBaseService, error) +} diff --git a/core/interfaces/grpc_client_model_spider_service.go b/core/interfaces/grpc_client_model_spider_service.go new file mode 100644 index 000000000..a2d8c4d49 --- /dev/null +++ b/core/interfaces/grpc_client_model_spider_service.go @@ -0,0 +1,14 @@ +package interfaces + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type GrpcClientModelSpiderService interface { + ModelBaseService + GetSpiderById(id primitive.ObjectID) (s Spider, err error) + GetSpider(query bson.M, opts *mongo.FindOptions) (s Spider, err error) + GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []Spider, err error) +} diff --git a/core/interfaces/grpc_client_model_task_service.go b/core/interfaces/grpc_client_model_task_service.go new file mode 100644 index 000000000..82f830e77 --- /dev/null +++ b/core/interfaces/grpc_client_model_task_service.go @@ -0,0 +1,14 @@ +package interfaces + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type GrpcClientModelTaskService interface { + ModelBaseService + GetTaskById(id primitive.ObjectID) (s Task, err error) + GetTask(query bson.M, opts *mongo.FindOptions) (s Task, err error) + GetTaskList(query bson.M, opts *mongo.FindOptions) (res []Task, err error) +} diff --git a/core/interfaces/grpc_client_model_task_stat_service.go b/core/interfaces/grpc_client_model_task_stat_service.go new file mode 100644 index 000000000..956678175 --- /dev/null +++ b/core/interfaces/grpc_client_model_task_stat_service.go @@ -0,0 +1,14 @@ +package interfaces + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type GrpcClientModelTaskStatService interface { + ModelBaseService + GetTaskStatById(id primitive.ObjectID) (s TaskStat, err error) + GetTaskStat(query bson.M, opts *mongo.FindOptions) (s TaskStat, err error) + GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []TaskStat, err error) +} diff --git a/core/interfaces/grpc_client_pool.go b/core/interfaces/grpc_client_pool.go new file mode 100644 index 000000000..7294daf04 --- /dev/null +++ b/core/interfaces/grpc_client_pool.go @@ -0,0 +1,9 @@ +package interfaces + +type GrpcClientPool interface { + WithConfigPath + Init() error + NewClient() error + GetClient() (GrpcClient, error) + SetSize(int) +} diff --git a/core/interfaces/grpc_model_base_service_message.go b/core/interfaces/grpc_model_base_service_message.go new file mode 100644 index 000000000..3df2d296c --- /dev/null +++ b/core/interfaces/grpc_model_base_service_message.go @@ -0,0 +1,7 @@ +package interfaces + +type GrpcModelBaseServiceMessage interface { + GetModelId() ModelId + GetData() []byte + ToBytes() (data []byte) +} diff --git a/core/interfaces/grpc_model_binder.go b/core/interfaces/grpc_model_binder.go new file mode 100644 index 000000000..695b370b3 --- /dev/null +++ b/core/interfaces/grpc_model_binder.go @@ -0,0 +1,5 @@ +package interfaces + +type GrpcModelBinder interface { + ModelBinder +} diff --git a/core/interfaces/grpc_model_delegate_message.go b/core/interfaces/grpc_model_delegate_message.go new file mode 100644 index 000000000..dd24e129a --- /dev/null +++ b/core/interfaces/grpc_model_delegate_message.go @@ -0,0 +1,8 @@ +package interfaces + +type GrpcModelDelegateMessage interface { + GetModelId() ModelId + GetMethod() ModelDelegateMethod + GetData() []byte + ToBytes() (data []byte) +} diff --git a/core/interfaces/grpc_model_list_binder.go b/core/interfaces/grpc_model_list_binder.go new file mode 100644 index 000000000..ed96892d1 --- /dev/null +++ b/core/interfaces/grpc_model_list_binder.go @@ -0,0 +1,5 @@ +package interfaces + +type GrpcModelListBinder interface { + ModelListBinder +} diff --git a/core/interfaces/grpc_server.go b/core/interfaces/grpc_server.go new file mode 100644 index 000000000..440a7b3c9 --- /dev/null +++ b/core/interfaces/grpc_server.go @@ -0,0 +1,16 @@ +package interfaces + +import ( + grpc "github.com/crawlab-team/crawlab/grpc" +) + +type GrpcServer interface { + GrpcBase + SetAddress(Address) + GetSubscribe(key string) (sub GrpcSubscribe, err error) + SetSubscribe(key string, sub GrpcSubscribe) + DeleteSubscribe(key string) + SendStreamMessage(key string, code grpc.StreamMessageCode) (err error) + SendStreamMessageWithData(nodeKey string, code grpc.StreamMessageCode, d interface{}) (err error) + IsStopped() (res bool) +} diff --git a/core/interfaces/grpc_stream.go b/core/interfaces/grpc_stream.go new file mode 100644 index 000000000..3b87e7262 --- /dev/null +++ b/core/interfaces/grpc_stream.go @@ -0,0 +1,12 @@ +package interfaces + +import grpc "github.com/crawlab-team/crawlab/grpc" + +type GrpcStream interface { + Send(msg *grpc.StreamMessage) (err error) +} + +type GrpcStreamBidirectional interface { + GrpcStream + Recv() (msg *grpc.StreamMessage, err error) +} diff --git a/core/interfaces/grpc_subscribe.go b/core/interfaces/grpc_subscribe.go new file mode 100644 index 000000000..fcbb92773 --- /dev/null +++ b/core/interfaces/grpc_subscribe.go @@ -0,0 +1,7 @@ +package interfaces + +type GrpcSubscribe interface { + GetStream() GrpcStream + GetStreamBidirectional() GrpcStreamBidirectional + GetFinished() chan bool +} diff --git a/core/interfaces/i18n_service.go b/core/interfaces/i18n_service.go new file mode 100644 index 000000000..9436092ae --- /dev/null +++ b/core/interfaces/i18n_service.go @@ -0,0 +1,6 @@ +package interfaces + +type I18nService interface { + AddTranslations(t []Translation) + GetTranslations() (t []Translation) +} diff --git a/core/interfaces/injectable.go b/core/interfaces/injectable.go new file mode 100644 index 000000000..4ec79ad5d --- /dev/null +++ b/core/interfaces/injectable.go @@ -0,0 +1,5 @@ +package interfaces + +type Injectable interface { + Inject() error +} diff --git a/core/interfaces/list.go b/core/interfaces/list.go new file mode 100644 index 000000000..7c5c08aab --- /dev/null +++ b/core/interfaces/list.go @@ -0,0 +1,5 @@ +package interfaces + +type List interface { + GetModels() (res []Model) +} diff --git a/core/interfaces/model.go b/core/interfaces/model.go new file mode 100644 index 000000000..bcbc6f7b6 --- /dev/null +++ b/core/interfaces/model.go @@ -0,0 +1,98 @@ +package interfaces + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Model interface { + GetId() (id primitive.ObjectID) + SetId(id primitive.ObjectID) +} + +type ModelV2 interface { + GetId() (id primitive.ObjectID) + SetId(id primitive.ObjectID) + SetCreated(by primitive.ObjectID) + SetUpdated(by primitive.ObjectID) +} + +type ModelId int + +const ( + ModelIdArtifact = iota + ModelIdTag + ModelIdNode + ModelIdProject + ModelIdSpider + ModelIdTask + ModelIdJob + ModelIdSchedule + ModelIdUser + ModelIdSetting + ModelIdToken + ModelIdVariable + ModelIdTaskQueue + ModelIdTaskStat + ModelIdSpiderStat + ModelIdDataSource + ModelIdDataCollection + ModelIdResult + ModelIdPassword + ModelIdExtraValue + ModelIdGit + ModelIdRole + ModelIdUserRole + ModelIdPermission + ModelIdRolePermission + ModelIdEnvironment + ModelIdDependencySetting +) + +const ( + ModelColNameArtifact = "artifacts" + ModelColNameTag = "tags" + ModelColNameNode = "nodes" + ModelColNameProject = "projects" + ModelColNameSpider = "spiders" + ModelColNameTask = "tasks" + ModelColNameJob = "jobs" + ModelColNameSchedule = "schedules" + ModelColNameUser = "users" + ModelColNameSetting = "settings" + ModelColNameToken = "tokens" + ModelColNameVariable = "variables" + ModelColNameTaskQueue = "task_queue" + ModelColNameTaskStat = "task_stats" + ModelColNameSpiderStat = "spider_stats" + ModelColNameDataSource = "data_sources" + ModelColNameDataCollection = "data_collections" + ModelColNamePasswords = "passwords" + ModelColNameExtraValues = "extra_values" + ModelColNameGit = "gits" + ModelColNameRole = "roles" + ModelColNameUserRole = "user_roles" + ModelColNamePermission = "permissions" + ModelColNameRolePermission = "role_permissions" + ModelColNameEnvironment = "environments" + ModelColNameDependencySetting = "dependency_settings" +) + +type ModelWithTags interface { + Model + SetTags(tags []Tag) + GetTags() (tags []Tag) +} + +type ModelWithNameDescription interface { + Model + GetName() (name string) + SetName(name string) + GetDescription() (description string) + SetDescription(description string) +} + +type ModelWithKey interface { + Model + GetKey() (key string) + SetKey(key string) +} diff --git a/core/interfaces/model_artifact.go b/core/interfaces/model_artifact.go new file mode 100644 index 000000000..297c08536 --- /dev/null +++ b/core/interfaces/model_artifact.go @@ -0,0 +1,12 @@ +package interfaces + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type ModelArtifact interface { + Model + GetSys() (sys ModelArtifactSys) + GetTagIds() (ids []primitive.ObjectID) + SetTagIds(ids []primitive.ObjectID) + SetObj(obj Model) + SetDel(del bool) +} diff --git a/core/interfaces/model_artifact_sys.go b/core/interfaces/model_artifact_sys.go new file mode 100644 index 000000000..3c2e10115 --- /dev/null +++ b/core/interfaces/model_artifact_sys.go @@ -0,0 +1,21 @@ +package interfaces + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type ModelArtifactSys interface { + GetCreateTs() time.Time + SetCreateTs(ts time.Time) + GetUpdateTs() time.Time + SetUpdateTs(ts time.Time) + GetDeleteTs() time.Time + SetDeleteTs(ts time.Time) + GetCreateUid() primitive.ObjectID + SetCreateUid(id primitive.ObjectID) + GetUpdateUid() primitive.ObjectID + SetUpdateUid(id primitive.ObjectID) + GetDeleteUid() primitive.ObjectID + SetDeleteUid(id primitive.ObjectID) +} diff --git a/core/interfaces/model_base_service.go b/core/interfaces/model_base_service.go new file mode 100644 index 000000000..1ff935488 --- /dev/null +++ b/core/interfaces/model_base_service.go @@ -0,0 +1,28 @@ +package interfaces + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type ModelBaseService interface { + GetModelId() (id ModelId) + SetModelId(id ModelId) + GetById(id primitive.ObjectID) (res Model, err error) + Get(query bson.M, opts *mongo.FindOptions) (res Model, err error) + GetList(query bson.M, opts *mongo.FindOptions) (res List, err error) + DeleteById(id primitive.ObjectID, args ...interface{}) (err error) + Delete(query bson.M, args ...interface{}) (err error) + DeleteList(query bson.M, args ...interface{}) (err error) + ForceDeleteList(query bson.M, args ...interface{}) (err error) + UpdateById(id primitive.ObjectID, update bson.M, args ...interface{}) (err error) + Update(query bson.M, update bson.M, fields []string, args ...interface{}) (err error) + UpdateDoc(query bson.M, doc Model, fields []string, args ...interface{}) (err error) + Insert(u User, docs ...interface{}) (err error) + Count(query bson.M) (total int, err error) +} + +type ModelService interface { + GetBaseService(id ModelId) (svc ModelBaseService) +} diff --git a/core/interfaces/model_binder.go b/core/interfaces/model_binder.go new file mode 100644 index 000000000..1fda66721 --- /dev/null +++ b/core/interfaces/model_binder.go @@ -0,0 +1,6 @@ +package interfaces + +type ModelBinder interface { + Bind() (res Model, err error) + Process(d Model) (res Model, err error) +} diff --git a/core/interfaces/model_delegate.go b/core/interfaces/model_delegate.go new file mode 100644 index 000000000..5224d5790 --- /dev/null +++ b/core/interfaces/model_delegate.go @@ -0,0 +1,22 @@ +package interfaces + +type ModelDelegateMethod string + +type ModelDelegate interface { + Add() error + Save() error + Delete() error + GetArtifact() (ModelArtifact, error) + GetModel() Model + Refresh() error + ToBytes(interface{}) ([]byte, error) +} + +const ( + ModelDelegateMethodAdd = "add" + ModelDelegateMethodSave = "save" + ModelDelegateMethodDelete = "delete" + ModelDelegateMethodGetArtifact = "get-artifact" + ModelDelegateMethodRefresh = "refresh" + ModelDelegateMethodChange = "change" +) diff --git a/core/interfaces/model_environment.go b/core/interfaces/model_environment.go new file mode 100644 index 000000000..121d7d0db --- /dev/null +++ b/core/interfaces/model_environment.go @@ -0,0 +1,9 @@ +package interfaces + +type Environment interface { + Model + GetKey() (key string) + SetKey(key string) + GetValue() (value string) + SetValue(value string) +} diff --git a/core/interfaces/model_extra_value.go b/core/interfaces/model_extra_value.go new file mode 100644 index 000000000..a3889976e --- /dev/null +++ b/core/interfaces/model_extra_value.go @@ -0,0 +1,17 @@ +package interfaces + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type ExtraValue interface { + Model + GetValue() (v interface{}) + SetValue(v interface{}) + GetObjectId() (oid primitive.ObjectID) + SetObjectId(oid primitive.ObjectID) + GetModel() (m string) + SetModel(m string) + GetType() (t string) + SetType(t string) +} diff --git a/core/interfaces/model_git.go b/core/interfaces/model_git.go new file mode 100644 index 000000000..e24eb06c1 --- /dev/null +++ b/core/interfaces/model_git.go @@ -0,0 +1,18 @@ +package interfaces + +// Git interface +type Git interface { + Model + GetUrl() (url string) + SetUrl(url string) + GetAuthType() (authType string) + SetAuthType(authType string) + GetUsername() (username string) + SetUsername(username string) + GetPassword() (password string) + SetPassword(password string) + GetCurrentBranch() (currentBranch string) + SetCurrentBranch(currentBranch string) + GetAutoPull() (autoPull bool) + SetAutoPull(autoPull bool) +} diff --git a/core/interfaces/model_list_binder.go b/core/interfaces/model_list_binder.go new file mode 100644 index 000000000..d05ba94c8 --- /dev/null +++ b/core/interfaces/model_list_binder.go @@ -0,0 +1,6 @@ +package interfaces + +type ModelListBinder interface { + Bind() (l List, err error) + Process(d interface{}) (l List, err error) +} diff --git a/core/interfaces/model_node.go b/core/interfaces/model_node.go new file mode 100644 index 000000000..bf9046605 --- /dev/null +++ b/core/interfaces/model_node.go @@ -0,0 +1,22 @@ +package interfaces + +import "time" + +type Node interface { + ModelWithNameDescription + GetKey() (key string) + GetIsMaster() (ok bool) + GetActive() (active bool) + SetActive(active bool) + SetActiveTs(activeTs time.Time) + GetStatus() (status string) + SetStatus(status string) + GetEnabled() (enabled bool) + SetEnabled(enabled bool) + GetAvailableRunners() (runners int) + SetAvailableRunners(runners int) + GetMaxRunners() (runners int) + SetMaxRunners(runners int) + IncrementAvailableRunners() + DecrementAvailableRunners() +} diff --git a/core/interfaces/model_node_delegate.go b/core/interfaces/model_node_delegate.go new file mode 100644 index 000000000..11272fbb5 --- /dev/null +++ b/core/interfaces/model_node_delegate.go @@ -0,0 +1,10 @@ +package interfaces + +import "time" + +type ModelNodeDelegate interface { + ModelDelegate + UpdateStatus(active bool, activeTs *time.Time, status string) (err error) + UpdateStatusOnline() (err error) + UpdateStatusOffline() (err error) +} diff --git a/core/interfaces/model_permission.go b/core/interfaces/model_permission.go new file mode 100644 index 000000000..99b4e7462 --- /dev/null +++ b/core/interfaces/model_permission.go @@ -0,0 +1,14 @@ +package interfaces + +type Permission interface { + ModelWithKey + ModelWithNameDescription + GetType() (t string) + SetType(t string) + GetTarget() (target []string) + SetTarget(target []string) + GetAllow() (allow []string) + SetAllow(allow []string) + GetDeny() (deny []string) + SetDeny(deny []string) +} diff --git a/core/interfaces/model_result.go b/core/interfaces/model_result.go new file mode 100644 index 000000000..91e349e36 --- /dev/null +++ b/core/interfaces/model_result.go @@ -0,0 +1,11 @@ +package interfaces + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type Result interface { + Value() map[string]interface{} + SetValue(key string, value interface{}) + GetValue(key string) (value interface{}) + GetTaskId() (id primitive.ObjectID) + SetTaskId(id primitive.ObjectID) +} diff --git a/core/interfaces/model_role.go b/core/interfaces/model_role.go new file mode 100644 index 000000000..efcc085ec --- /dev/null +++ b/core/interfaces/model_role.go @@ -0,0 +1,6 @@ +package interfaces + +type Role interface { + ModelWithKey + ModelWithNameDescription +} diff --git a/core/interfaces/model_schedule.go b/core/interfaces/model_schedule.go new file mode 100644 index 000000000..ec805472e --- /dev/null +++ b/core/interfaces/model_schedule.go @@ -0,0 +1,28 @@ +package interfaces + +import ( + "github.com/robfig/cron/v3" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Schedule interface { + Model + GetEnabled() (enabled bool) + SetEnabled(enabled bool) + GetEntryId() (id cron.EntryID) + SetEntryId(id cron.EntryID) + GetCron() (c string) + SetCron(c string) + GetSpiderId() (id primitive.ObjectID) + SetSpiderId(id primitive.ObjectID) + GetMode() (mode string) + SetMode(mode string) + GetNodeIds() (ids []primitive.ObjectID) + SetNodeIds(ids []primitive.ObjectID) + GetCmd() (cmd string) + SetCmd(cmd string) + GetParam() (param string) + SetParam(param string) + GetPriority() (p int) + SetPriority(p int) +} diff --git a/core/interfaces/model_service_v2.go b/core/interfaces/model_service_v2.go new file mode 100644 index 000000000..c4d1cace3 --- /dev/null +++ b/core/interfaces/model_service_v2.go @@ -0,0 +1,25 @@ +package interfaces + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type ModelServiceV2[T any] interface { + GetById(id primitive.ObjectID) (model *T, err error) + Get(query bson.M, options *mongo.FindOptions) (model *T, err error) + GetList(query bson.M, options *mongo.FindOptions) (models []T, err error) + DeleteById(id primitive.ObjectID) (err error) + Delete(query bson.M) (err error) + DeleteList(query bson.M) (err error) + UpdateById(id primitive.ObjectID, update bson.M) (err error) + UpdateOne(query bson.M, update bson.M) (err error) + UpdateMany(query bson.M, update bson.M) (err error) + ReplaceById(id primitive.ObjectID, model T) (err error) + Replace(query bson.M, model T) (err error) + InsertOne(model T) (id primitive.ObjectID, err error) + InsertMany(models []T) (ids []primitive.ObjectID, err error) + Count(query bson.M) (total int, err error) + GetCol() (col *mongo.Col) +} diff --git a/core/interfaces/model_spider.go b/core/interfaces/model_spider.go new file mode 100644 index 000000000..c6cfe416e --- /dev/null +++ b/core/interfaces/model_spider.go @@ -0,0 +1,24 @@ +package interfaces + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type Spider interface { + ModelWithNameDescription + GetType() (ty string) + GetMode() (mode string) + SetMode(mode string) + GetNodeIds() (ids []primitive.ObjectID) + SetNodeIds(ids []primitive.ObjectID) + GetCmd() (cmd string) + SetCmd(cmd string) + GetParam() (param string) + SetParam(param string) + GetPriority() (p int) + SetPriority(p int) + GetColId() (id primitive.ObjectID) + SetColId(id primitive.ObjectID) + GetIncrementalSync() (incrementalSync bool) + SetIncrementalSync(incrementalSync bool) + GetAutoInstall() (autoInstall bool) + SetAutoInstall(autoInstall bool) +} diff --git a/core/interfaces/model_tag.go b/core/interfaces/model_tag.go new file mode 100644 index 000000000..5afd89bab --- /dev/null +++ b/core/interfaces/model_tag.go @@ -0,0 +1,8 @@ +package interfaces + +type Tag interface { + Model + GetName() string + GetColor() string + SetCol(string) +} diff --git a/core/interfaces/model_task.go b/core/interfaces/model_task.go new file mode 100644 index 000000000..7eb07a79d --- /dev/null +++ b/core/interfaces/model_task.go @@ -0,0 +1,23 @@ +package interfaces + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type Task interface { + Model + GetNodeId() (id primitive.ObjectID) + SetNodeId(id primitive.ObjectID) + GetNodeIds() (ids []primitive.ObjectID) + GetStatus() (status string) + SetStatus(status string) + GetError() (error string) + SetError(error string) + GetPid() (pid int) + SetPid(pid int) + GetSpiderId() (id primitive.ObjectID) + GetType() (ty string) + GetCmd() (cmd string) + GetParam() (param string) + GetPriority() (p int) + GetUserId() (id primitive.ObjectID) + SetUserId(id primitive.ObjectID) +} diff --git a/core/interfaces/model_task_stat.go b/core/interfaces/model_task_stat.go new file mode 100644 index 000000000..7ddce156f --- /dev/null +++ b/core/interfaces/model_task_stat.go @@ -0,0 +1,23 @@ +package interfaces + +import "time" + +type TaskStat interface { + Model + GetCreateTs() (ts time.Time) + SetCreateTs(ts time.Time) + GetStartTs() (ts time.Time) + SetStartTs(ts time.Time) + GetEndTs() (ts time.Time) + SetEndTs(ts time.Time) + GetWaitDuration() (d int64) + SetWaitDuration(d int64) + GetRuntimeDuration() (d int64) + SetRuntimeDuration(d int64) + GetTotalDuration() (d int64) + SetTotalDuration(d int64) + GetResultCount() (c int64) + SetResultCount(c int64) + GetErrorLogCount() (c int64) + SetErrorLogCount(c int64) +} diff --git a/core/interfaces/model_user.go b/core/interfaces/model_user.go new file mode 100644 index 000000000..d8f86c7cf --- /dev/null +++ b/core/interfaces/model_user.go @@ -0,0 +1,9 @@ +package interfaces + +type User interface { + Model + GetUsername() (name string) + GetPassword() (p string) + GetRole() (r string) + GetEmail() (email string) +} diff --git a/core/interfaces/model_user_group.go b/core/interfaces/model_user_group.go new file mode 100644 index 000000000..f3930a1e7 --- /dev/null +++ b/core/interfaces/model_user_group.go @@ -0,0 +1,6 @@ +package interfaces + +type UserGroup interface { + Model + GetUsers() (users []User, err error) +} diff --git a/core/interfaces/module.go b/core/interfaces/module.go new file mode 100644 index 000000000..4fc693037 --- /dev/null +++ b/core/interfaces/module.go @@ -0,0 +1,10 @@ +package interfaces + +type ModuleId int + +type Module interface { + Init() error + Start() + Wait() + Stop() +} diff --git a/core/interfaces/node_config_service.go b/core/interfaces/node_config_service.go new file mode 100644 index 000000000..80e5b5108 --- /dev/null +++ b/core/interfaces/node_config_service.go @@ -0,0 +1,13 @@ +package interfaces + +type NodeConfigService interface { + WithConfigPath + Init() error + Reload() error + GetBasicNodeInfo() Entity + GetNodeKey() string + GetNodeName() string + IsMaster() bool + GetAuthKey() string + GetMaxRunners() int +} diff --git a/core/interfaces/node_master_service.go b/core/interfaces/node_master_service.go new file mode 100644 index 000000000..dc9dcfc8c --- /dev/null +++ b/core/interfaces/node_master_service.go @@ -0,0 +1,14 @@ +package interfaces + +import ( + "time" +) + +type NodeMasterService interface { + NodeService + Monitor() + SetMonitorInterval(duration time.Duration) + Register() error + StopOnError() + GetServer() GrpcServer +} diff --git a/core/interfaces/node_service.go b/core/interfaces/node_service.go new file mode 100644 index 000000000..54cf7885c --- /dev/null +++ b/core/interfaces/node_service.go @@ -0,0 +1,8 @@ +package interfaces + +type NodeService interface { + Module + WithConfigPath + WithAddress + GetConfigService() NodeConfigService +} diff --git a/core/interfaces/node_service_option.go b/core/interfaces/node_service_option.go new file mode 100644 index 000000000..0402b349a --- /dev/null +++ b/core/interfaces/node_service_option.go @@ -0,0 +1,4 @@ +package interfaces + +type NodeServiceOption interface { +} diff --git a/core/interfaces/node_worker_service.go b/core/interfaces/node_worker_service.go new file mode 100644 index 000000000..b25c654a0 --- /dev/null +++ b/core/interfaces/node_worker_service.go @@ -0,0 +1,11 @@ +package interfaces + +import "time" + +type NodeWorkerService interface { + NodeService + Register() + Recv() + ReportStatus() + SetHeartbeatInterval(duration time.Duration) +} diff --git a/core/interfaces/options.go b/core/interfaces/options.go new file mode 100644 index 000000000..08badf2f9 --- /dev/null +++ b/core/interfaces/options.go @@ -0,0 +1 @@ +package interfaces diff --git a/core/interfaces/process_daemon.go b/core/interfaces/process_daemon.go new file mode 100644 index 000000000..aea65df70 --- /dev/null +++ b/core/interfaces/process_daemon.go @@ -0,0 +1,17 @@ +package interfaces + +import ( + "os/exec" + "time" +) + +type ProcessDaemon interface { + Start() (err error) + Stop() + GetMaxErrors() (maxErrors int) + SetMaxErrors(maxErrors int) + GetExitTimeout() (timeout time.Duration) + SetExitTimeout(timeout time.Duration) + GetCmd() (cmd *exec.Cmd) + GetCh() (ch chan int) +} diff --git a/core/interfaces/provide.go b/core/interfaces/provide.go new file mode 100644 index 000000000..e83929485 --- /dev/null +++ b/core/interfaces/provide.go @@ -0,0 +1,3 @@ +package interfaces + +type Provide func(env string) diff --git a/core/interfaces/result_service.go b/core/interfaces/result_service.go new file mode 100644 index 000000000..1d8616671 --- /dev/null +++ b/core/interfaces/result_service.go @@ -0,0 +1,15 @@ +package interfaces + +import ( + "github.com/crawlab-team/crawlab-db/generic" + "time" +) + +type ResultService interface { + Insert(records ...interface{}) (err error) + List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) + Count(query generic.ListQuery) (n int, err error) + Index(fields []string) + SetTime(t time.Time) + GetTime() (t time.Time) +} diff --git a/core/interfaces/result_service_mongo.go b/core/interfaces/result_service_mongo.go new file mode 100644 index 000000000..37c807ae7 --- /dev/null +++ b/core/interfaces/result_service_mongo.go @@ -0,0 +1,15 @@ +package interfaces + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type ResultServiceMongo interface { + GetId() (id primitive.ObjectID) + SetId(id primitive.ObjectID) + List(query bson.M, opts *mongo.FindOptions) (results []Result, err error) + Count(query bson.M) (total int, err error) + Insert(docs ...interface{}) (err error) +} diff --git a/core/interfaces/result_service_registry.go b/core/interfaces/result_service_registry.go new file mode 100644 index 000000000..e989ffe85 --- /dev/null +++ b/core/interfaces/result_service_registry.go @@ -0,0 +1,11 @@ +package interfaces + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type ResultServiceRegistry interface { + Register(key string, fn ResultServiceRegistryFn) + Unregister(key string) + Get(key string) (fn ResultServiceRegistryFn) +} + +type ResultServiceRegistryFn func(colId primitive.ObjectID, dsId primitive.ObjectID) (ResultService, error) diff --git a/core/interfaces/schedule_service.go b/core/interfaces/schedule_service.go new file mode 100644 index 000000000..9e535dbac --- /dev/null +++ b/core/interfaces/schedule_service.go @@ -0,0 +1,23 @@ +package interfaces + +import ( + "github.com/robfig/cron/v3" + "time" +) + +type ScheduleService interface { + WithConfigPath + Module + GetLocation() (loc *time.Location) + SetLocation(loc *time.Location) + GetDelay() (delay bool) + SetDelay(delay bool) + GetSkip() (skip bool) + SetSkip(skip bool) + GetUpdateInterval() (interval time.Duration) + SetUpdateInterval(interval time.Duration) + Enable(s Schedule, args ...interface{}) (err error) + Disable(s Schedule, args ...interface{}) (err error) + Update() + GetCron() (c *cron.Cron) +} diff --git a/core/interfaces/spider_admin_service.go b/core/interfaces/spider_admin_service.go new file mode 100644 index 000000000..d410e5cb2 --- /dev/null +++ b/core/interfaces/spider_admin_service.go @@ -0,0 +1,22 @@ +package interfaces + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type SpiderAdminService interface { + WithConfigPath + Start() (err error) + // Schedule a new task of the spider + Schedule(id primitive.ObjectID, opts *SpiderRunOptions) (taskIds []primitive.ObjectID, err error) + // Clone the spider + Clone(id primitive.ObjectID, opts *SpiderCloneOptions) (err error) + // Delete the spider + Delete(id primitive.ObjectID) (err error) + // SyncGit syncs all git repositories + SyncGit() (err error) + // SyncGitOne syncs one git repository + SyncGitOne(g Git) (err error) + // Export exports the spider and return zip file path + Export(id primitive.ObjectID) (filePath string, err error) +} diff --git a/core/interfaces/spider_service_options.go b/core/interfaces/spider_service_options.go new file mode 100644 index 000000000..d92726d35 --- /dev/null +++ b/core/interfaces/spider_service_options.go @@ -0,0 +1,17 @@ +package interfaces + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type SpiderRunOptions struct { + Mode string `json:"mode"` + NodeIds []primitive.ObjectID `json:"node_ids"` + Cmd string `json:"cmd"` + Param string `json:"param"` + ScheduleId primitive.ObjectID `json:"schedule_id"` + Priority int `json:"priority"` + UserId primitive.ObjectID `json:"-"` +} + +type SpiderCloneOptions struct { + Name string +} diff --git a/core/interfaces/stats_service.go b/core/interfaces/stats_service.go new file mode 100644 index 000000000..aef3f5f50 --- /dev/null +++ b/core/interfaces/stats_service.go @@ -0,0 +1,9 @@ +package interfaces + +import "go.mongodb.org/mongo-driver/bson" + +type StatsService interface { + GetOverviewStats(query bson.M) (data interface{}, err error) + GetDailyStats(query bson.M) (data interface{}, err error) + GetTaskStats(query bson.M) (data interface{}, err error) +} diff --git a/core/interfaces/task_base_service.go b/core/interfaces/task_base_service.go new file mode 100644 index 000000000..951613161 --- /dev/null +++ b/core/interfaces/task_base_service.go @@ -0,0 +1,11 @@ +package interfaces + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type TaskBaseService interface { + WithConfigPath + Module + SaveTask(t Task, status string) (err error) + IsStopped() (res bool) + GetQueue(nodeId primitive.ObjectID) (queue string) +} diff --git a/core/interfaces/task_handler_service.go b/core/interfaces/task_handler_service.go new file mode 100644 index 000000000..382764ce9 --- /dev/null +++ b/core/interfaces/task_handler_service.go @@ -0,0 +1,60 @@ +package interfaces + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type TaskHandlerService interface { + TaskBaseService + // Run task and execute locally + Run(taskId primitive.ObjectID) (err error) + // Cancel task locally + Cancel(taskId primitive.ObjectID) (err error) + // Fetch tasks and run + Fetch() + // ReportStatus periodically report handler status to master + ReportStatus() + // Reset reset internals to default + Reset() + // IsSyncLocked whether the given task is locked for files sync + IsSyncLocked(path string) (ok bool) + // LockSync lock files sync for given task + LockSync(path string) + // UnlockSync unlock files sync for given task + UnlockSync(path string) + // GetExitWatchDuration get max runners + GetExitWatchDuration() (duration time.Duration) + // SetExitWatchDuration set max runners + SetExitWatchDuration(duration time.Duration) + // GetFetchInterval get report interval + GetFetchInterval() (interval time.Duration) + // SetFetchInterval set report interval + SetFetchInterval(interval time.Duration) + // GetReportInterval get report interval + GetReportInterval() (interval time.Duration) + // SetReportInterval set report interval + SetReportInterval(interval time.Duration) + // GetCancelTimeout get report interval + GetCancelTimeout() (timeout time.Duration) + // SetCancelTimeout set report interval + SetCancelTimeout(timeout time.Duration) + // GetModelService get model service + GetModelService() (modelSvc GrpcClientModelService) + // GetModelSpiderService get model spider service + GetModelSpiderService() (modelSpiderSvc GrpcClientModelSpiderService) + // GetModelTaskService get model task service + GetModelTaskService() (modelTaskSvc GrpcClientModelTaskService) + // GetModelTaskStatService get model task stat service + GetModelTaskStatService() (modelTaskStatSvc GrpcClientModelTaskStatService) + // GetModelEnvironmentService get model environment service + GetModelEnvironmentService() (modelEnvironmentSvc GrpcClientModelEnvironmentService) + // GetNodeConfigService get node config service + GetNodeConfigService() (cfgSvc NodeConfigService) + // GetCurrentNode get node of the handler + GetCurrentNode() (n Node, err error) + // GetTaskById get task by id + GetTaskById(id primitive.ObjectID) (t Task, err error) + // GetSpiderById get task by id + GetSpiderById(id primitive.ObjectID) (t Spider, err error) +} diff --git a/core/interfaces/task_hook_service.go b/core/interfaces/task_hook_service.go new file mode 100644 index 000000000..f7c18082e --- /dev/null +++ b/core/interfaces/task_hook_service.go @@ -0,0 +1,6 @@ +package interfaces + +type TaskHookService interface { + PreActions(Task, Spider, FsServiceV2, TaskHandlerService) (err error) + PostActions(Task, Spider, FsServiceV2, TaskHandlerService) (err error) +} diff --git a/core/interfaces/task_runner.go b/core/interfaces/task_runner.go new file mode 100644 index 000000000..4194764c0 --- /dev/null +++ b/core/interfaces/task_runner.go @@ -0,0 +1,15 @@ +package interfaces + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type TaskRunner interface { + Init() (err error) + Run() (err error) + Cancel() (err error) + SetSubscribeTimeout(timeout time.Duration) + GetTaskId() (id primitive.ObjectID) + CleanUp() (err error) +} diff --git a/core/interfaces/task_scheduler_service.go b/core/interfaces/task_scheduler_service.go new file mode 100644 index 000000000..b287440f2 --- /dev/null +++ b/core/interfaces/task_scheduler_service.go @@ -0,0 +1,16 @@ +package interfaces + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type TaskSchedulerService interface { + TaskBaseService + // Enqueue task into the task queue + Enqueue(t Task) (t2 Task, err error) + // Cancel task to corresponding node + Cancel(id primitive.ObjectID, args ...interface{}) (err error) + // SetInterval set the interval or duration between two adjacent fetches + SetInterval(interval time.Duration) +} diff --git a/core/interfaces/task_stats_service.go b/core/interfaces/task_stats_service.go new file mode 100644 index 000000000..9d1604ac2 --- /dev/null +++ b/core/interfaces/task_stats_service.go @@ -0,0 +1,9 @@ +package interfaces + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type TaskStatsService interface { + TaskBaseService + InsertData(id primitive.ObjectID, records ...interface{}) (err error) + InsertLogs(id primitive.ObjectID, logs ...string) (err error) +} diff --git a/core/interfaces/test.go b/core/interfaces/test.go new file mode 100644 index 000000000..bc107b42b --- /dev/null +++ b/core/interfaces/test.go @@ -0,0 +1,8 @@ +package interfaces + +import "testing" + +type Test interface { + Setup(*testing.T) + Cleanup() +} diff --git a/core/interfaces/translation.go b/core/interfaces/translation.go new file mode 100644 index 000000000..c5c4c6b66 --- /dev/null +++ b/core/interfaces/translation.go @@ -0,0 +1,5 @@ +package interfaces + +type Translation interface { + GetLang() (l string) +} diff --git a/core/interfaces/user_service.go b/core/interfaces/user_service.go new file mode 100644 index 000000000..d04952344 --- /dev/null +++ b/core/interfaces/user_service.go @@ -0,0 +1,19 @@ +package interfaces + +import ( + "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt/v5" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type UserService interface { + Init() (err error) + SetJwtSecret(secret string) + SetJwtSigningMethod(method jwt.SigningMethod) + Create(opts *UserCreateOptions, args ...interface{}) (err error) + Login(opts *UserLoginOptions) (token string, u User, err error) + CheckToken(token string) (u User, err error) + ChangePassword(id primitive.ObjectID, password string, args ...interface{}) (err error) + MakeToken(user User) (tokenStr string, err error) + GetCurrentUser(c *gin.Context) (u User, err error) +} diff --git a/core/interfaces/user_service_options.go b/core/interfaces/user_service_options.go new file mode 100644 index 000000000..abf29774e --- /dev/null +++ b/core/interfaces/user_service_options.go @@ -0,0 +1,13 @@ +package interfaces + +type UserCreateOptions struct { + Username string + Password string + Email string + Role string +} + +type UserLoginOptions struct { + Username string + Password string +} diff --git a/core/interfaces/with_address.go b/core/interfaces/with_address.go new file mode 100644 index 000000000..805eff184 --- /dev/null +++ b/core/interfaces/with_address.go @@ -0,0 +1,6 @@ +package interfaces + +type WithAddress interface { + GetAddress() (address Address) + SetAddress(address Address) +} diff --git a/core/interfaces/with_config_path.go b/core/interfaces/with_config_path.go new file mode 100644 index 000000000..b2aac6b82 --- /dev/null +++ b/core/interfaces/with_config_path.go @@ -0,0 +1,6 @@ +package interfaces + +type WithConfigPath interface { + GetConfigPath() (path string) + SetConfigPath(path string) +} diff --git a/core/interfaces/with_model_id.go b/core/interfaces/with_model_id.go new file mode 100644 index 000000000..f2f59f8cc --- /dev/null +++ b/core/interfaces/with_model_id.go @@ -0,0 +1,6 @@ +package interfaces + +type WithModelId interface { + GetModelId() (id ModelId) + SetModelId(id ModelId) +} diff --git a/core/main.go b/core/main.go new file mode 100644 index 000000000..da29a2cad --- /dev/null +++ b/core/main.go @@ -0,0 +1,4 @@ +package main + +func main() { +} diff --git a/core/middlewares/auth.go b/core/middlewares/auth.go new file mode 100644 index 000000000..201695cfc --- /dev/null +++ b/core/middlewares/auth.go @@ -0,0 +1,50 @@ +package middlewares + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/user" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" +) + +func AuthorizationMiddleware() gin.HandlerFunc { + userSvc, _ := user.GetUserService() + return func(c *gin.Context) { + // disable auth for test + if viper.GetBool("auth.disabled") { + modelSvc, err := service.GetService() + if err != nil { + utils.HandleErrorInternalServerError(c, err) + return + } + u, err := modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil) + if err != nil { + utils.HandleErrorInternalServerError(c, err) + return + } + c.Set(constants.UserContextKey, u) + c.Next() + return + } + + // token string + tokenStr := c.GetHeader("Authorization") + + // validate token + u, err := userSvc.CheckToken(tokenStr) + if err != nil { + // validation failed, return error response + utils.HandleErrorUnauthorized(c, errors.ErrorHttpUnauthorized) + return + } + + // set user in context + c.Set(constants.UserContextKey, u) + + // validation success + c.Next() + } +} diff --git a/core/middlewares/auth_v2.go b/core/middlewares/auth_v2.go new file mode 100644 index 000000000..61b8954b7 --- /dev/null +++ b/core/middlewares/auth_v2.go @@ -0,0 +1,50 @@ +package middlewares + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/user" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" +) + +func AuthorizationMiddlewareV2() gin.HandlerFunc { + userSvc, _ := user.GetUserServiceV2() + return func(c *gin.Context) { + // disable auth for test + if viper.GetBool("auth.disabled") { + modelSvc, err := service.GetService() + if err != nil { + utils.HandleErrorInternalServerError(c, err) + return + } + u, err := modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil) + if err != nil { + utils.HandleErrorInternalServerError(c, err) + return + } + c.Set(constants.UserContextKey, u) + c.Next() + return + } + + // token string + tokenStr := c.GetHeader("Authorization") + + // validate token + u, err := userSvc.CheckToken(tokenStr) + if err != nil { + // validation failed, return error response + utils.HandleErrorUnauthorized(c, errors.ErrorHttpUnauthorized) + return + } + + // set user in context + c.Set(constants.UserContextKey, u) + + // validation success + c.Next() + } +} diff --git a/core/middlewares/cors.go b/core/middlewares/cors.go new file mode 100644 index 000000000..5397251d1 --- /dev/null +++ b/core/middlewares/cors.go @@ -0,0 +1,19 @@ +package middlewares + +import "github.com/gin-gonic/gin" + +func CORSMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + c.Writer.Header().Set("Access-Control-Allow-Origin", "*") + c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") + c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With") + c.Writer.Header().Set("Access-Control-Allow-Methods", "DELETE, POST, OPTIONS, GET, PUT") + + if c.Request.Method == "OPTIONS" { + c.AbortWithStatus(204) + return + } + + c.Next() + } +} diff --git a/core/middlewares/filer_auth.go b/core/middlewares/filer_auth.go new file mode 100644 index 000000000..1535f2b80 --- /dev/null +++ b/core/middlewares/filer_auth.go @@ -0,0 +1,34 @@ +package middlewares + +import ( + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" +) + +func FilerAuthorizationMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + // auth key + authKey := c.GetHeader("Authorization") + + // server auth key + svrAuthKey := viper.GetString("fs.filer.authKey") + + // skip to next if no server auth key is provided + if svrAuthKey == "" { + c.Next() + return + } + + // validate + if authKey != svrAuthKey { + // validation failed, return error response + utils.HandleErrorUnauthorized(c, errors.ErrorHttpUnauthorized) + return + } + + // validation success + c.Next() + } +} diff --git a/core/middlewares/middlewares.go b/core/middlewares/middlewares.go new file mode 100644 index 000000000..8ead457df --- /dev/null +++ b/core/middlewares/middlewares.go @@ -0,0 +1,16 @@ +package middlewares + +import "github.com/gin-gonic/gin" + +func InitMiddlewares(app *gin.Engine) (err error) { + // default logger + app.Use(gin.Logger()) + + // recovery from panics + app.Use(gin.Recovery()) + + // cors + app.Use(CORSMiddleware()) + + return nil +} diff --git a/core/models/client/binder_basic.go b/core/models/client/binder_basic.go new file mode 100644 index 000000000..06e675a74 --- /dev/null +++ b/core/models/client/binder_basic.go @@ -0,0 +1,92 @@ +package client + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" +) + +func NewBasicBinder(id interfaces.ModelId, res *grpc.Response) (b interfaces.GrpcModelBinder) { + return &BasicBinder{ + id: id, + res: res, + } +} + +type BasicBinder struct { + id interfaces.ModelId + res *grpc.Response +} + +func (b *BasicBinder) Bind() (res interfaces.Model, err error) { + m := models.NewModelMap() + + switch b.id { + case interfaces.ModelIdArtifact: + return b.Process(&m.Artifact) + case interfaces.ModelIdTag: + return b.Process(&m.Tag) + case interfaces.ModelIdNode: + return b.Process(&m.Node) + case interfaces.ModelIdProject: + return b.Process(&m.Project) + case interfaces.ModelIdSpider: + return b.Process(&m.Spider) + case interfaces.ModelIdTask: + return b.Process(&m.Task) + case interfaces.ModelIdJob: + return b.Process(&m.Job) + case interfaces.ModelIdSchedule: + return b.Process(&m.Schedule) + case interfaces.ModelIdUser: + return b.Process(&m.User) + case interfaces.ModelIdSetting: + return b.Process(&m.Setting) + case interfaces.ModelIdToken: + return b.Process(&m.Token) + case interfaces.ModelIdVariable: + return b.Process(&m.Variable) + case interfaces.ModelIdTaskQueue: + return b.Process(&m.TaskQueueItem) + case interfaces.ModelIdTaskStat: + return b.Process(&m.TaskStat) + case interfaces.ModelIdSpiderStat: + return b.Process(&m.SpiderStat) + case interfaces.ModelIdDataSource: + return b.Process(&m.DataSource) + case interfaces.ModelIdDataCollection: + return b.Process(&m.DataCollection) + case interfaces.ModelIdResult: + return b.Process(&m.Result) + case interfaces.ModelIdPassword: + return b.Process(&m.Password) + case interfaces.ModelIdExtraValue: + return b.Process(&m.ExtraValue) + case interfaces.ModelIdGit: + return b.Process(&m.Git) + case interfaces.ModelIdRole: + return b.Process(&m.Role) + case interfaces.ModelIdUserRole: + return b.Process(&m.UserRole) + case interfaces.ModelIdPermission: + return b.Process(&m.Permission) + case interfaces.ModelIdRolePermission: + return b.Process(&m.RolePermission) + case interfaces.ModelIdEnvironment: + return b.Process(&m.Environment) + case interfaces.ModelIdDependencySetting: + return b.Process(&m.DependencySetting) + default: + return nil, errors.ErrorModelInvalidModelId + } +} + +func (b *BasicBinder) Process(d interfaces.Model) (res interfaces.Model, err error) { + if err := json.Unmarshal(b.res.Data, d); err != nil { + return nil, trace.TraceError(err) + } + return d, nil +} diff --git a/core/models/client/binder_list.go b/core/models/client/binder_list.go new file mode 100644 index 000000000..4f81980c6 --- /dev/null +++ b/core/models/client/binder_list.go @@ -0,0 +1,100 @@ +package client + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" +) + +func NewListBinder(id interfaces.ModelId, res *grpc.Response) (b interfaces.GrpcModelListBinder) { + return &ListBinder{ + id: id, + res: res, + } +} + +type ListBinder struct { + id interfaces.ModelId + res *grpc.Response +} + +func (b *ListBinder) Bind() (l interfaces.List, err error) { + m := models.NewModelListMap() + + switch b.id { + case interfaces.ModelIdArtifact: + return b.Process(&m.Artifacts) + case interfaces.ModelIdTag: + return b.Process(&m.Tags) + case interfaces.ModelIdNode: + return b.Process(&m.Nodes) + case interfaces.ModelIdProject: + return b.Process(&m.Projects) + case interfaces.ModelIdSpider: + return b.Process(&m.Spiders) + case interfaces.ModelIdTask: + return b.Process(&m.Tasks) + case interfaces.ModelIdJob: + return b.Process(&m.Jobs) + case interfaces.ModelIdSchedule: + return b.Process(&m.Schedules) + case interfaces.ModelIdUser: + return b.Process(&m.Users) + case interfaces.ModelIdSetting: + return b.Process(&m.Settings) + case interfaces.ModelIdToken: + return b.Process(&m.Tokens) + case interfaces.ModelIdVariable: + return b.Process(&m.Variables) + case interfaces.ModelIdTaskQueue: + return b.Process(&m.TaskQueueItems) + case interfaces.ModelIdTaskStat: + return b.Process(&m.TaskStats) + case interfaces.ModelIdSpiderStat: + return b.Process(&m.SpiderStats) + case interfaces.ModelIdDataSource: + return b.Process(&m.DataSources) + case interfaces.ModelIdDataCollection: + return b.Process(&m.DataCollections) + case interfaces.ModelIdResult: + return b.Process(&m.Results) + case interfaces.ModelIdPassword: + return b.Process(&m.Passwords) + case interfaces.ModelIdExtraValue: + return b.Process(&m.ExtraValues) + case interfaces.ModelIdGit: + return b.Process(&m.Gits) + case interfaces.ModelIdRole: + return b.Process(&m.Roles) + case interfaces.ModelIdUserRole: + return b.Process(&m.UserRoles) + case interfaces.ModelIdPermission: + return b.Process(&m.PermissionList) + case interfaces.ModelIdRolePermission: + return b.Process(&m.RolePermissionList) + case interfaces.ModelIdEnvironment: + return b.Process(&m.Environments) + case interfaces.ModelIdDependencySetting: + return b.Process(&m.DependencySettings) + default: + return l, errors.ErrorModelInvalidModelId + } +} + +func (b *ListBinder) MustBind() (res interface{}) { + res, err := b.Bind() + if err != nil { + panic(err) + } + return res +} + +func (b *ListBinder) Process(d interface{}) (l interfaces.List, err error) { + if err := json.Unmarshal(b.res.Data, d); err != nil { + return l, trace.TraceError(err) + } + return d.(interfaces.List), nil +} diff --git a/core/models/client/model_base_service.go b/core/models/client/model_base_service.go new file mode 100644 index 000000000..e604c891e --- /dev/null +++ b/core/models/client/model_base_service.go @@ -0,0 +1,251 @@ +package client + +import ( + "encoding/json" + "github.com/apex/log" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + errors2 "github.com/pkg/errors" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type BaseServiceDelegate struct { + // settings + cfgPath string + + // internals + id interfaces.ModelId + c interfaces.GrpcClient +} + +func (d *BaseServiceDelegate) GetModelId() (id interfaces.ModelId) { + return d.id +} + +func (d *BaseServiceDelegate) SetModelId(id interfaces.ModelId) { + d.id = id +} + +func (d *BaseServiceDelegate) GetConfigPath() (path string) { + return d.cfgPath +} + +func (d *BaseServiceDelegate) SetConfigPath(path string) { + d.cfgPath = path +} + +func (d *BaseServiceDelegate) GetById(id primitive.ObjectID) (doc interfaces.Model, err error) { + log.Debugf("[BaseServiceDelegate] get by id[%s]", id.Hex()) + ctx, cancel := d.c.Context() + defer cancel() + req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id}) + c := d.getModelBaseServiceClient() + if c == nil { + return nil, trace.TraceError(errors.ErrorModelNilPointer) + } + log.Debugf("[BaseServiceDelegate] get by id[%s] req: %v", id.Hex(), req) + res, err := c.GetById(ctx, req) + if err != nil { + return nil, trace.TraceError(err) + } + log.Debugf("[BaseServiceDelegate] get by id[%s] res: %v", id.Hex(), res) + return NewBasicBinder(d.id, res).Bind() +} + +func (d *BaseServiceDelegate) Get(query bson.M, opts *mongo.FindOptions) (doc interfaces.Model, err error) { + ctx, cancel := d.c.Context() + defer cancel() + req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, FindOptions: opts}) + res, err := d.getModelBaseServiceClient().Get(ctx, req) + if err != nil { + return nil, err + } + return NewBasicBinder(d.id, res).Bind() +} + +func (d *BaseServiceDelegate) GetList(query bson.M, opts *mongo.FindOptions) (l interfaces.List, err error) { + ctx, cancel := d.c.Context() + defer cancel() + req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, FindOptions: opts}) + res, err := d.getModelBaseServiceClient().GetList(ctx, req) + if err != nil { + return l, err + } + return NewListBinder(d.id, res).Bind() +} + +func (d *BaseServiceDelegate) DeleteById(id primitive.ObjectID, args ...interface{}) (err error) { + u := utils.GetUserFromArgs(args...) + ctx, cancel := d.c.Context() + defer cancel() + req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id, User: u}) + _, err = d.getModelBaseServiceClient().DeleteById(ctx, req) + if err != nil { + return err + } + return nil +} + +func (d *BaseServiceDelegate) Delete(query bson.M, args ...interface{}) (err error) { + u := utils.GetUserFromArgs(args...) + ctx, cancel := d.c.Context() + defer cancel() + req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u}) + _, err = d.getModelBaseServiceClient().Delete(ctx, req) + if err != nil { + return err + } + return nil +} + +func (d *BaseServiceDelegate) DeleteList(query bson.M, args ...interface{}) (err error) { + u := utils.GetUserFromArgs(args...) + ctx, cancel := d.c.Context() + defer cancel() + req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u}) + _, err = d.getModelBaseServiceClient().DeleteList(ctx, req) + if err != nil { + return err + } + return nil +} + +func (d *BaseServiceDelegate) ForceDeleteList(query bson.M, args ...interface{}) (err error) { + u := utils.GetUserFromArgs(args...) + ctx, cancel := d.c.Context() + defer cancel() + req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, User: u}) + _, err = d.getModelBaseServiceClient().ForceDeleteList(ctx, req) + if err != nil { + return err + } + return nil +} + +func (d *BaseServiceDelegate) UpdateById(id primitive.ObjectID, update bson.M, args ...interface{}) (err error) { + u := utils.GetUserFromArgs(args...) + ctx, cancel := d.c.Context() + defer cancel() + req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Id: id, Update: update, User: u}) + _, err = d.getModelBaseServiceClient().UpdateById(ctx, req) + if err != nil { + return err + } + return nil +} + +func (d *BaseServiceDelegate) Update(query bson.M, update bson.M, fields []string, args ...interface{}) (err error) { + u := utils.GetUserFromArgs(args...) + ctx, cancel := d.c.Context() + defer cancel() + req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, Update: update, Fields: fields, User: u}) + _, err = d.getModelBaseServiceClient().Update(ctx, req) + if err != nil { + return err + } + return nil +} + +func (d *BaseServiceDelegate) UpdateDoc(query bson.M, doc interfaces.Model, fields []string, args ...interface{}) (err error) { + u := utils.GetUserFromArgs(args...) + ctx, cancel := d.c.Context() + defer cancel() + req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query, Doc: doc, Fields: fields, User: u}) + _, err = d.getModelBaseServiceClient().UpdateDoc(ctx, req) + if err != nil { + return err + } + return nil +} + +func (d *BaseServiceDelegate) Insert(u interfaces.User, docs ...interface{}) (err error) { + ctx, cancel := d.c.Context() + defer cancel() + req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Docs: docs, User: u}) + _, err = d.getModelBaseServiceClient().Insert(ctx, req) + if err != nil { + return err + } + return nil +} + +func (d *BaseServiceDelegate) Count(query bson.M) (total int, err error) { + ctx, cancel := d.c.Context() + defer cancel() + req := d.mustNewRequest(&entity.GrpcBaseServiceParams{Query: query}) + res, err := d.getModelBaseServiceClient().Count(ctx, req) + if err != nil { + return total, err + } + if err := json.Unmarshal(res.Data, &total); err != nil { + return total, err + } + return total, nil +} + +func (d *BaseServiceDelegate) newRequest(params interfaces.GrpcBaseServiceParams) (req *grpc.Request, err error) { + return d.c.NewModelBaseServiceRequest(d.id, params) +} + +func (d *BaseServiceDelegate) mustNewRequest(params *entity.GrpcBaseServiceParams) (req *grpc.Request) { + req, err := d.newRequest(params) + if err != nil { + panic(err) + } + return req +} + +func (d *BaseServiceDelegate) getModelBaseServiceClient() (c grpc.ModelBaseServiceClient) { + if err := backoff.Retry(func() (err error) { + c = d.c.GetModelBaseServiceClient() + if c == nil { + err = errors2.New("unable to get model base service client") + log.Debugf("[BaseServiceDelegate] err: %v", err) + return err + } + return nil + }, backoff.NewConstantBackOff(1*time.Second)); err != nil { + trace.PrintError(err) + } + return c +} + +func NewBaseServiceDelegate(opts ...ModelBaseServiceDelegateOption) (svc2 interfaces.GrpcClientModelBaseService, err error) { + // base service + svc := &BaseServiceDelegate{} + + // apply options + for _, opt := range opts { + opt(svc) + } + + // config path + if viper.GetString("config.path") != "" { + svc.cfgPath = viper.GetString("config.path") + } + + // dependency injection + if err := container.GetContainer().Invoke(func(client interfaces.GrpcClient) { + svc.c = client + }); err != nil { + return nil, err + } + + return svc, nil +} + +func ProvideBaseServiceDelegate(id interfaces.ModelId) func() (svc interfaces.GrpcClientModelBaseService, err error) { + return func() (svc interfaces.GrpcClientModelBaseService, err error) { + return NewBaseServiceDelegate(WithBaseServiceModelId(id)) + } +} diff --git a/core/models/client/model_delegate.go b/core/models/client/model_delegate.go new file mode 100644 index 000000000..a42db1cd3 --- /dev/null +++ b/core/models/client/model_delegate.go @@ -0,0 +1,329 @@ +package client + +import ( + "encoding/json" + config2 "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/grpc/client" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "github.com/spf13/viper" +) + +func NewModelDelegate(doc interfaces.Model, opts ...ModelDelegateOption) interfaces.GrpcClientModelDelegate { + switch doc.(type) { + case *models.Artifact: + return newModelDelegate(interfaces.ModelIdArtifact, doc, opts...) + case *models.Tag: + return newModelDelegate(interfaces.ModelIdTag, doc, opts...) + case *models.Node: + return newModelDelegate(interfaces.ModelIdNode, doc, opts...) + case *models.Project: + return newModelDelegate(interfaces.ModelIdProject, doc, opts...) + case *models.Spider: + return newModelDelegate(interfaces.ModelIdSpider, doc, opts...) + case *models.Task: + return newModelDelegate(interfaces.ModelIdTask, doc, opts...) + case *models.Job: + return newModelDelegate(interfaces.ModelIdJob, doc, opts...) + case *models.Schedule: + return newModelDelegate(interfaces.ModelIdSchedule, doc, opts...) + case *models.User: + return newModelDelegate(interfaces.ModelIdUser, doc, opts...) + case *models.Setting: + return newModelDelegate(interfaces.ModelIdSetting, doc, opts...) + case *models.Token: + return newModelDelegate(interfaces.ModelIdToken, doc, opts...) + case *models.Variable: + return newModelDelegate(interfaces.ModelIdVariable, doc, opts...) + case *models.TaskQueueItem: + return newModelDelegate(interfaces.ModelIdTaskQueue, doc, opts...) + case *models.TaskStat: + return newModelDelegate(interfaces.ModelIdTaskStat, doc, opts...) + case *models.SpiderStat: + return newModelDelegate(interfaces.ModelIdSpiderStat, doc, opts...) + case *models.DataSource: + return newModelDelegate(interfaces.ModelIdDataSource, doc, opts...) + case *models.DataCollection: + return newModelDelegate(interfaces.ModelIdDataCollection, doc, opts...) + case *models.Result: + return newModelDelegate(interfaces.ModelIdResult, doc, opts...) + case *models.Password: + return newModelDelegate(interfaces.ModelIdPassword, doc, opts...) + case *models.ExtraValue: + return newModelDelegate(interfaces.ModelIdExtraValue, doc, opts...) + case *models.Git: + return newModelDelegate(interfaces.ModelIdGit, doc, opts...) + case *models.UserRole: + return newModelDelegate(interfaces.ModelIdUserRole, doc, opts...) + case *models.Permission: + return newModelDelegate(interfaces.ModelIdPermission, doc, opts...) + case *models.RolePermission: + return newModelDelegate(interfaces.ModelIdRolePermission, doc, opts...) + case *models.Environment: + return newModelDelegate(interfaces.ModelIdEnvironment, doc, opts...) + case *models.DependencySetting: + return newModelDelegate(interfaces.ModelIdDependencySetting, doc, opts...) + default: + _ = trace.TraceError(errors.ErrorModelInvalidType) + return nil + } +} + +func newModelDelegate(id interfaces.ModelId, doc interfaces.Model, opts ...ModelDelegateOption) interfaces.GrpcClientModelDelegate { + var err error + + // collection name + colName := models.GetModelColName(id) + + // model delegate + d := &ModelDelegate{ + id: id, + colName: colName, + doc: doc, + cfgPath: config2.GetConfigPath(), + a: &models.Artifact{ + Col: colName, + }, + } + + // config path + if viper.GetString("config.path") != "" { + d.cfgPath = viper.GetString("config.path") + } + + // apply options + for _, opt := range opts { + opt(d) + } + + // grpc client + d.c, err = client.GetClient() + if err != nil { + trace.PrintError(errors.ErrorModelInvalidType) + return nil + } + if !d.c.IsStarted() { + if err := d.c.Start(); err != nil { + trace.PrintError(err) + return nil + } + } else if d.c.IsClosed() { + if err := d.c.Restart(); err != nil { + trace.PrintError(err) + return nil + } + } + + return d +} + +type ModelDelegate struct { + // settings + cfgPath string + + // internals + id interfaces.ModelId + colName string + c interfaces.GrpcClient + doc interfaces.Model + a interfaces.ModelArtifact +} + +func (d *ModelDelegate) Add() (err error) { + return d.do(interfaces.ModelDelegateMethodAdd) +} + +func (d *ModelDelegate) Save() (err error) { + return d.do(interfaces.ModelDelegateMethodSave) +} + +func (d *ModelDelegate) Delete() (err error) { + return d.do(interfaces.ModelDelegateMethodDelete) +} + +func (d *ModelDelegate) GetArtifact() (res interfaces.ModelArtifact, err error) { + if err := d.do(interfaces.ModelDelegateMethodGetArtifact); err != nil { + return nil, err + } + return d.a, nil +} + +func (d *ModelDelegate) GetModel() (res interfaces.Model) { + return d.doc +} + +func (d *ModelDelegate) Refresh() (err error) { + return d.refresh() +} + +func (d *ModelDelegate) GetConfigPath() (path string) { + return d.cfgPath +} + +func (d *ModelDelegate) SetConfigPath(path string) { + d.cfgPath = path +} + +func (d *ModelDelegate) Close() (err error) { + return d.c.Stop() +} + +func (d *ModelDelegate) ToBytes(m interface{}) (bytes []byte, err error) { + if m != nil { + return utils.JsonToBytes(m) + } + return json.Marshal(d.doc) +} + +func (d *ModelDelegate) do(method interfaces.ModelDelegateMethod) (err error) { + switch method { + case interfaces.ModelDelegateMethodAdd: + err = d.add() + case interfaces.ModelDelegateMethodSave: + err = d.save() + case interfaces.ModelDelegateMethodDelete: + err = d.delete() + case interfaces.ModelDelegateMethodGetArtifact, interfaces.ModelDelegateMethodRefresh: + return d.refresh() + default: + return trace.TraceError(errors.ErrorModelInvalidType) + } + + if err != nil { + return err + } + + return nil +} + +func (d *ModelDelegate) add() (err error) { + ctx, cancel := d.c.Context() + defer cancel() + method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodAdd) + res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{ + ModelId: d.id, + Method: method, + Data: d.mustGetData(), + })) + if err != nil { + return trace.TraceError(err) + } + if err := d.deserialize(res, method); err != nil { + return err + } + return d.refreshArtifact() +} + +func (d *ModelDelegate) save() (err error) { + ctx, cancel := d.c.Context() + defer cancel() + method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodSave) + res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{ + ModelId: d.id, + Method: method, + Data: d.mustGetData(), + })) + if err != nil { + return trace.TraceError(err) + } + if err := d.deserialize(res, method); err != nil { + return err + } + return d.refreshArtifact() +} + +func (d *ModelDelegate) delete() (err error) { + ctx, cancel := d.c.Context() + defer cancel() + method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodDelete) + res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{ + ModelId: d.id, + Method: method, + Data: d.mustGetData(), + })) + if err != nil { + return trace.TraceError(err) + } + if err := d.deserialize(res, method); err != nil { + return err + } + return d.refreshArtifact() +} + +func (d *ModelDelegate) refresh() (err error) { + ctx, cancel := d.c.Context() + defer cancel() + method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodRefresh) + res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{ + ModelId: d.id, + Method: method, + Data: d.mustGetData(), + })) + if err != nil { + return trace.TraceError(err) + } + if err := d.deserialize(res, method); err != nil { + return err + } + return nil +} + +func (d *ModelDelegate) refreshArtifact() (err error) { + _, err = d.getArtifact() + return err +} + +func (d *ModelDelegate) getArtifact() (res2 interfaces.ModelArtifact, err error) { + ctx, cancel := d.c.Context() + defer cancel() + method := interfaces.ModelDelegateMethod(interfaces.ModelDelegateMethodGetArtifact) + res, err := d.c.GetModelDelegateClient().Do(ctx, d.c.NewRequest(entity.GrpcDelegateMessage{ + ModelId: d.id, + Method: method, + Data: d.mustGetData(), + })) + if err != nil { + return nil, err + } + if err := d.deserialize(res, method); err != nil { + return nil, err + } + return d.a, nil +} + +func (d *ModelDelegate) mustGetData() (data []byte) { + data, err := d.getData() + if err != nil { + panic(err) + } + return data +} + +func (d *ModelDelegate) getData() (data []byte, err error) { + return json.Marshal(d.doc) +} + +func (d *ModelDelegate) deserialize(res *grpc.Response, method interfaces.ModelDelegateMethod) (err error) { + if method == interfaces.ModelDelegateMethodGetArtifact { + res, err := NewBasicBinder(interfaces.ModelIdArtifact, res).Bind() + if err != nil { + return err + } + a, ok := res.(interfaces.ModelArtifact) + if !ok { + return trace.TraceError(errors.ErrorModelInvalidType) + } + d.a = a + } else { + d.doc, err = NewBasicBinder(d.id, res).Bind() + if err != nil { + return err + } + } + return nil +} diff --git a/core/models/client/model_environment_service.go b/core/models/client/model_environment_service.go new file mode 100644 index 000000000..57aef5cc5 --- /dev/null +++ b/core/models/client/model_environment_service.go @@ -0,0 +1,70 @@ +package client + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type EnvironmentServiceDelegate struct { + interfaces.GrpcClientModelBaseService +} + +func (svc *EnvironmentServiceDelegate) GetEnvironmentById(id primitive.ObjectID) (e interfaces.Environment, err error) { + res, err := svc.GetById(id) + if err != nil { + return nil, err + } + s, ok := res.(interfaces.Environment) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return s, nil +} + +func (svc *EnvironmentServiceDelegate) GetEnvironment(query bson.M, opts *mongo.FindOptions) (e interfaces.Environment, err error) { + res, err := svc.Get(query, opts) + if err != nil { + return nil, err + } + s, ok := res.(interfaces.Environment) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return s, nil +} + +func (svc *EnvironmentServiceDelegate) GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Environment, err error) { + list, err := svc.GetList(query, opts) + if err != nil { + return nil, err + } + for _, item := range list.GetModels() { + s, ok := item.(interfaces.Environment) + if !ok { + return nil, errors.ErrorModelInvalidType + } + res = append(res, s) + } + return res, nil +} + +func NewEnvironmentServiceDelegate() (svc2 interfaces.GrpcClientModelEnvironmentService, err error) { + var opts []ModelBaseServiceDelegateOption + + // apply options + opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdEnvironment)) + + // base service + baseSvc, err := NewBaseServiceDelegate(opts...) + if err != nil { + return nil, err + } + + // service + svc := &EnvironmentServiceDelegate{baseSvc} + + return svc, nil +} diff --git a/core/models/client/model_node_delegate.go b/core/models/client/model_node_delegate.go new file mode 100644 index 000000000..4d7d8eb45 --- /dev/null +++ b/core/models/client/model_node_delegate.go @@ -0,0 +1,37 @@ +package client + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type ModelNodeDelegate struct { + n interfaces.Node + interfaces.GrpcClientModelDelegate +} + +func (d *ModelNodeDelegate) UpdateStatus(active bool, activeTs *time.Time, status string) (err error) { + d.n.SetActive(active) + if activeTs != nil { + d.n.SetActiveTs(*activeTs) + } + d.n.SetStatus(status) + return d.Save() +} + +func (d *ModelNodeDelegate) UpdateStatusOnline() (err error) { + now := time.Now() + return d.UpdateStatus(true, &now, constants.NodeStatusOnline) +} + +func (d *ModelNodeDelegate) UpdateStatusOffline() (err error) { + return d.UpdateStatus(false, nil, constants.NodeStatusOffline) +} + +func NewModelNodeDelegate(n interfaces.Node) interfaces.ModelNodeDelegate { + return &ModelNodeDelegate{ + n: n, + GrpcClientModelDelegate: NewModelDelegate(n), + } +} diff --git a/core/models/client/model_node_service.go b/core/models/client/model_node_service.go new file mode 100644 index 000000000..09bc396f4 --- /dev/null +++ b/core/models/client/model_node_service.go @@ -0,0 +1,74 @@ +package client + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type NodeServiceDelegate struct { + interfaces.GrpcClientModelBaseService +} + +func (svc *NodeServiceDelegate) GetNodeById(id primitive.ObjectID) (n interfaces.Node, err error) { + res, err := svc.GetById(id) + if err != nil { + return nil, err + } + s, ok := res.(interfaces.Node) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return s, nil +} + +func (svc *NodeServiceDelegate) GetNode(query bson.M, opts *mongo.FindOptions) (n interfaces.Node, err error) { + res, err := svc.Get(query, opts) + if err != nil { + return nil, err + } + s, ok := res.(interfaces.Node) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return s, nil +} + +func (svc *NodeServiceDelegate) GetNodeByKey(key string) (n interfaces.Node, err error) { + return svc.GetNode(bson.M{"key": key}, nil) +} + +func (svc *NodeServiceDelegate) GetNodeList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Node, err error) { + list, err := svc.GetList(query, opts) + if err != nil { + return nil, err + } + for _, item := range list.GetModels() { + s, ok := item.(interfaces.Node) + if !ok { + return nil, errors.ErrorModelInvalidType + } + res = append(res, s) + } + return res, nil +} + +func NewNodeServiceDelegate() (svc2 interfaces.GrpcClientModelNodeService, err error) { + var opts []ModelBaseServiceDelegateOption + + // apply options + opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdNode)) + + // base service + baseSvc, err := NewBaseServiceDelegate(opts...) + if err != nil { + return nil, err + } + + // service + svc := &NodeServiceDelegate{baseSvc} + + return svc, nil +} diff --git a/core/models/client/model_service.go b/core/models/client/model_service.go new file mode 100644 index 000000000..fde84d56b --- /dev/null +++ b/core/models/client/model_service.go @@ -0,0 +1,48 @@ +package client + +import ( + config2 "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/interfaces" +) + +type ServiceDelegate struct { + // settings + cfgPath string + + // internals + c interfaces.GrpcClient +} + +func (d *ServiceDelegate) GetConfigPath() string { + return d.cfgPath +} + +func (d *ServiceDelegate) SetConfigPath(path string) { + d.cfgPath = path +} + +func (d *ServiceDelegate) NewBaseServiceDelegate(id interfaces.ModelId) (svc interfaces.GrpcClientModelBaseService, err error) { + var opts []ModelBaseServiceDelegateOption + opts = append(opts, WithBaseServiceModelId(id)) + if d.cfgPath != "" { + opts = append(opts, WithBaseServiceConfigPath(d.cfgPath)) + } + return NewBaseServiceDelegate(opts...) +} + +func NewServiceDelegate() (svc2 interfaces.GrpcClientModelService, err error) { + // service delegate + svc := &ServiceDelegate{ + cfgPath: config2.GetConfigPath(), + } + + // dependency injection + if err := container.GetContainer().Invoke(func(client interfaces.GrpcClient) { + svc.c = client + }); err != nil { + return nil, err + } + + return svc, nil +} diff --git a/core/models/client/model_service_v2.go b/core/models/client/model_service_v2.go new file mode 100644 index 000000000..5a8b377e5 --- /dev/null +++ b/core/models/client/model_service_v2.go @@ -0,0 +1,360 @@ +package client + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/grpc/client" + "github.com/crawlab-team/crawlab/core/interfaces" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + grpc "github.com/crawlab-team/crawlab/grpc" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "reflect" + "sync" +) + +var ( + instanceMap = make(map[string]interface{}) + onceMap = make(map[string]*sync.Once) + mu sync.Mutex +) + +type ModelServiceV2[T any] struct { + cfg interfaces.NodeConfigService + c *client.GrpcClientV2 + modelType string +} + +func (svc *ModelServiceV2[T]) GetById(id primitive.ObjectID) (model *T, err error) { + ctx, cancel := svc.c.Context() + defer cancel() + res, err := svc.c.ModelBaseServiceV2Client.GetById(ctx, &grpc.ModelServiceV2GetByIdRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Id: id.Hex(), + }) + if err != nil { + return nil, err + } + return svc.deserializeOne(res) +} + +func (svc *ModelServiceV2[T]) GetOne(query bson.M, options *mongo.FindOptions) (model *T, err error) { + ctx, cancel := svc.c.Context() + defer cancel() + queryData, err := json.Marshal(query) + if err != nil { + return nil, err + } + findOptionsData, err := json.Marshal(options) + if err != nil { + return nil, err + } + res, err := svc.c.ModelBaseServiceV2Client.GetOne(ctx, &grpc.ModelServiceV2GetOneRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Query: queryData, + FindOptions: findOptionsData, + }) + if err != nil { + return nil, err + } + return svc.deserializeOne(res) +} + +func (svc *ModelServiceV2[T]) GetMany(query bson.M, options *mongo.FindOptions) (models []T, err error) { + ctx, cancel := svc.c.Context() + defer cancel() + queryData, err := json.Marshal(query) + if err != nil { + return nil, err + } + findOptionsData, err := json.Marshal(options) + if err != nil { + return nil, err + } + res, err := svc.c.ModelBaseServiceV2Client.GetMany(ctx, &grpc.ModelServiceV2GetManyRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Query: queryData, + FindOptions: findOptionsData, + }) + if err != nil { + return nil, err + } + return svc.deserializeMany(res) +} + +func (svc *ModelServiceV2[T]) DeleteById(id primitive.ObjectID) (err error) { + ctx, cancel := svc.c.Context() + defer cancel() + _, err = svc.c.ModelBaseServiceV2Client.DeleteById(ctx, &grpc.ModelServiceV2DeleteByIdRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Id: id.Hex(), + }) + if err != nil { + return err + } + return nil +} + +func (svc *ModelServiceV2[T]) DeleteOne(query bson.M) (err error) { + ctx, cancel := svc.c.Context() + defer cancel() + queryData, err := json.Marshal(query) + if err != nil { + return err + } + _, err = svc.c.ModelBaseServiceV2Client.DeleteOne(ctx, &grpc.ModelServiceV2DeleteOneRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Query: queryData, + }) + if err != nil { + return err + } + return nil +} + +func (svc *ModelServiceV2[T]) DeleteMany(query bson.M) (err error) { + ctx, cancel := svc.c.Context() + defer cancel() + queryData, err := json.Marshal(query) + if err != nil { + return err + } + _, err = svc.c.ModelBaseServiceV2Client.DeleteMany(ctx, &grpc.ModelServiceV2DeleteManyRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Query: queryData, + }) + if err != nil { + return err + } + return nil +} + +func (svc *ModelServiceV2[T]) UpdateById(id primitive.ObjectID, update bson.M) (err error) { + ctx, cancel := svc.c.Context() + defer cancel() + updateData, err := json.Marshal(update) + if err != nil { + return err + } + _, err = svc.c.ModelBaseServiceV2Client.UpdateById(ctx, &grpc.ModelServiceV2UpdateByIdRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Id: id.Hex(), + Update: updateData, + }) + if err != nil { + return err + } + return nil +} + +func (svc *ModelServiceV2[T]) UpdateOne(query bson.M, update bson.M) (err error) { + ctx, cancel := svc.c.Context() + defer cancel() + queryData, err := json.Marshal(query) + if err != nil { + return err + } + updateData, err := json.Marshal(update) + if err != nil { + return err + } + _, err = svc.c.ModelBaseServiceV2Client.UpdateOne(ctx, &grpc.ModelServiceV2UpdateOneRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Query: queryData, + Update: updateData, + }) + if err != nil { + return err + } + return nil +} + +func (svc *ModelServiceV2[T]) UpdateMany(query bson.M, update bson.M) (err error) { + ctx, cancel := svc.c.Context() + defer cancel() + queryData, err := json.Marshal(query) + if err != nil { + return err + } + updateData, err := json.Marshal(update) + if err != nil { + return err + } + _, err = svc.c.ModelBaseServiceV2Client.UpdateMany(ctx, &grpc.ModelServiceV2UpdateManyRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Query: queryData, + Update: updateData, + }) + return nil +} + +func (svc *ModelServiceV2[T]) ReplaceById(id primitive.ObjectID, model T) (err error) { + ctx, cancel := svc.c.Context() + defer cancel() + modelData, err := json.Marshal(model) + if err != nil { + return err + } + _, err = svc.c.ModelBaseServiceV2Client.ReplaceById(ctx, &grpc.ModelServiceV2ReplaceByIdRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Id: id.Hex(), + Model: modelData, + }) + if err != nil { + return err + } + return nil +} + +func (svc *ModelServiceV2[T]) ReplaceOne(query bson.M, model T) (err error) { + ctx, cancel := svc.c.Context() + defer cancel() + queryData, err := json.Marshal(query) + if err != nil { + return err + } + modelData, err := json.Marshal(model) + if err != nil { + return err + } + _, err = svc.c.ModelBaseServiceV2Client.ReplaceOne(ctx, &grpc.ModelServiceV2ReplaceOneRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Query: queryData, + Model: modelData, + }) + if err != nil { + return err + } + return nil +} + +func (svc *ModelServiceV2[T]) InsertOne(model T) (id primitive.ObjectID, err error) { + ctx, cancel := svc.c.Context() + defer cancel() + modelData, err := json.Marshal(model) + if err != nil { + return primitive.NilObjectID, err + } + res, err := svc.c.ModelBaseServiceV2Client.InsertOne(ctx, &grpc.ModelServiceV2InsertOneRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Model: modelData, + }) + if err != nil { + return primitive.NilObjectID, err + } + return deserialize[primitive.ObjectID](res) + //idStr, err := deserialize[string](res) + //if err != nil { + // return primitive.NilObjectID, err + //} + //return primitive.ObjectIDFromHex(idStr) +} + +func (svc *ModelServiceV2[T]) InsertMany(models []T) (ids []primitive.ObjectID, err error) { + ctx, cancel := svc.c.Context() + defer cancel() + modelsData, err := json.Marshal(models) + if err != nil { + return nil, err + } + res, err := svc.c.ModelBaseServiceV2Client.InsertMany(ctx, &grpc.ModelServiceV2InsertManyRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Models: modelsData, + }) + if err != nil { + return nil, err + } + return deserialize[[]primitive.ObjectID](res) +} + +func (svc *ModelServiceV2[T]) Count(query bson.M) (total int, err error) { + ctx, cancel := svc.c.Context() + defer cancel() + queryData, err := json.Marshal(query) + if err != nil { + return 0, err + } + res, err := svc.c.ModelBaseServiceV2Client.Count(ctx, &grpc.ModelServiceV2CountRequest{ + NodeKey: svc.cfg.GetNodeKey(), + ModelType: svc.modelType, + Query: queryData, + }) + if err != nil { + return 0, err + } + return deserialize[int](res) +} + +func (svc *ModelServiceV2[T]) GetCol() (col *mongo.Col) { + return nil +} + +func (svc *ModelServiceV2[T]) deserializeOne(res *grpc.Response) (result *T, err error) { + r, err := deserialize[T](res) + if err != nil { + return nil, err + } + return &r, err +} + +func (svc *ModelServiceV2[T]) deserializeMany(res *grpc.Response) (results []T, err error) { + return deserialize[[]T](res) +} + +func deserialize[T any](res *grpc.Response) (result T, err error) { + err = json.Unmarshal(res.Data, &result) + if err != nil { + return result, err + } + return result, nil +} + +func NewModelServiceV2[T any]() *ModelServiceV2[T] { + mu.Lock() + defer mu.Unlock() + + var v T + t := reflect.TypeOf(v) + typeName := t.Name() + + if _, exists := onceMap[typeName]; !exists { + onceMap[typeName] = &sync.Once{} + } + + var instance *ModelServiceV2[T] + + c, err := client.GetGrpcClientV2() + if err != nil { + panic(err) + } + if !c.IsStarted() { + err = c.Start() + if err != nil { + panic(err) + } + } + + onceMap[typeName].Do(func() { + instance = &ModelServiceV2[T]{ + cfg: nodeconfig.GetNodeConfigService(), + c: c, + modelType: typeName, + } + instanceMap[typeName] = instance + }) + + return instanceMap[typeName].(*ModelServiceV2[T]) +} diff --git a/core/models/client/model_service_v2_test.go b/core/models/client/model_service_v2_test.go new file mode 100644 index 000000000..59d0075d5 --- /dev/null +++ b/core/models/client/model_service_v2_test.go @@ -0,0 +1,439 @@ +package client_test + +import ( + "context" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/grpc/server" + "github.com/crawlab-team/crawlab/core/models/client" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "testing" + "time" +) + +type TestModel models.TestModel + +func setupTestDB() { + viper.Set("mongo.db", "testdb") +} + +func teardownTestDB() { + db := mongo.GetMongoDb("testdb") + db.Drop(context.Background()) +} + +func TestModelServiceV2_GetById(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + m := TestModel{ + Name: "Test Name", + } + modelSvc := service.NewModelServiceV2[TestModel]() + id, err := modelSvc.InsertOne(m) + require.Nil(t, err) + m.SetId(id) + time.Sleep(100 * time.Millisecond) + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + res, err := clientSvc.GetById(m.Id) + require.Nil(t, err) + assert.Equal(t, res.Id, m.Id) + assert.Equal(t, res.Name, m.Name) +} + +func TestModelServiceV2_GetOne(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + m := TestModel{ + Name: "Test Name", + } + modelSvc := service.NewModelServiceV2[TestModel]() + id, err := modelSvc.InsertOne(m) + require.Nil(t, err) + m.SetId(id) + time.Sleep(100 * time.Millisecond) + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + res, err := clientSvc.GetOne(bson.M{"name": m.Name}, nil) + require.Nil(t, err) + assert.Equal(t, res.Id, m.Id) + assert.Equal(t, res.Name, m.Name) +} + +func TestModelServiceV2_GetMany(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + m := TestModel{ + Name: "Test Name", + } + modelSvc := service.NewModelServiceV2[TestModel]() + id, err := modelSvc.InsertOne(m) + require.Nil(t, err) + m.SetId(id) + time.Sleep(100 * time.Millisecond) + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + res, err := clientSvc.GetMany(bson.M{"name": m.Name}, nil) + require.Nil(t, err) + assert.Equal(t, len(res), 1) + assert.Equal(t, res[0].Id, m.Id) + assert.Equal(t, res[0].Name, m.Name) +} + +func TestModelServiceV2_DeleteById(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + m := TestModel{ + Name: "Test Name", + } + modelSvc := service.NewModelServiceV2[TestModel]() + id, err := modelSvc.InsertOne(m) + require.Nil(t, err) + m.SetId(id) + time.Sleep(100 * time.Millisecond) + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + err = clientSvc.DeleteById(m.Id) + require.Nil(t, err) + + res, err := clientSvc.GetById(m.Id) + assert.NotNil(t, err) + require.Nil(t, res) +} + +func TestModelServiceV2_DeleteOne(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + m := TestModel{ + Name: "Test Name", + } + modelSvc := service.NewModelServiceV2[TestModel]() + id, err := modelSvc.InsertOne(m) + require.Nil(t, err) + m.SetId(id) + time.Sleep(100 * time.Millisecond) + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + err = clientSvc.DeleteOne(bson.M{"name": m.Name}) + require.Nil(t, err) + + res, err := clientSvc.GetOne(bson.M{"name": m.Name}, nil) + assert.NotNil(t, err) + require.Nil(t, res) +} + +func TestModelServiceV2_DeleteMany(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + m := TestModel{ + Name: "Test Name", + } + modelSvc := service.NewModelServiceV2[TestModel]() + id, err := modelSvc.InsertOne(m) + require.Nil(t, err) + m.SetId(id) + time.Sleep(100 * time.Millisecond) + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + err = clientSvc.DeleteMany(bson.M{"name": m.Name}) + require.Nil(t, err) + + res, err := clientSvc.GetMany(bson.M{"name": m.Name}, nil) + require.Nil(t, err) + assert.Equal(t, len(res), 0) +} + +func TestModelServiceV2_UpdateById(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + m := TestModel{ + Name: "Test Name", + } + modelSvc := service.NewModelServiceV2[TestModel]() + id, err := modelSvc.InsertOne(m) + require.Nil(t, err) + m.SetId(id) + time.Sleep(100 * time.Millisecond) + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + err = clientSvc.UpdateById(m.Id, bson.M{"$set": bson.M{"name": "New Name"}}) + require.Nil(t, err) + + res, err := clientSvc.GetById(m.Id) + require.Nil(t, err) + assert.Equal(t, res.Name, "New Name") +} + +func TestModelServiceV2_UpdateOne(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + m := TestModel{ + Name: "Test Name", + } + modelSvc := service.NewModelServiceV2[TestModel]() + id, err := modelSvc.InsertOne(m) + require.Nil(t, err) + m.SetId(id) + time.Sleep(100 * time.Millisecond) + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + err = clientSvc.UpdateOne(bson.M{"name": m.Name}, bson.M{"$set": bson.M{"name": "New Name"}}) + require.Nil(t, err) + + res, err := clientSvc.GetOne(bson.M{"name": "New Name"}, nil) + require.Nil(t, err) + assert.Equal(t, res.Name, "New Name") +} + +func TestModelServiceV2_UpdateMany(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + m1 := TestModel{ + Name: "Test Name", + } + m2 := TestModel{ + Name: "Test Name", + } + modelSvc := service.NewModelServiceV2[TestModel]() + _, err = modelSvc.InsertOne(m1) + require.Nil(t, err) + _, err = modelSvc.InsertOne(m2) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + err = clientSvc.UpdateMany(bson.M{"name": "Test Name"}, bson.M{"$set": bson.M{"name": "New Name"}}) + require.Nil(t, err) + + res, err := clientSvc.GetMany(bson.M{"name": "New Name"}, nil) + require.Nil(t, err) + assert.Equal(t, len(res), 2) +} + +func TestModelServiceV2_ReplaceById(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + m := TestModel{ + Name: "Test Name", + } + modelSvc := service.NewModelServiceV2[TestModel]() + id, err := modelSvc.InsertOne(m) + require.Nil(t, err) + m.SetId(id) + time.Sleep(100 * time.Millisecond) + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + m.Name = "New Name" + err = clientSvc.ReplaceById(m.Id, m) + require.Nil(t, err) + + res, err := clientSvc.GetById(m.Id) + require.Nil(t, err) + assert.Equal(t, res.Name, "New Name") +} + +func TestModelServiceV2_ReplaceOne(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + m := TestModel{ + Name: "Test Name", + } + modelSvc := service.NewModelServiceV2[TestModel]() + id, err := modelSvc.InsertOne(m) + require.Nil(t, err) + m.SetId(id) + time.Sleep(100 * time.Millisecond) + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + m.Name = "New Name" + err = clientSvc.ReplaceOne(bson.M{"name": "Test Name"}, m) + require.Nil(t, err) + + res, err := clientSvc.GetOne(bson.M{"name": "New Name"}, nil) + require.Nil(t, err) + assert.Equal(t, res.Name, "New Name") +} + +func TestModelServiceV2_InsertOne(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + m := TestModel{ + Name: "Test Name", + } + id, err := clientSvc.InsertOne(m) + require.Nil(t, err) + + res, err := clientSvc.GetById(id) + require.Nil(t, err) + assert.Equal(t, res.Name, m.Name) +} + +func TestModelServiceV2_InsertMany(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + models := []TestModel{ + {Name: "Test Name 1"}, + {Name: "Test Name 2"}, + } + ids, err := clientSvc.InsertMany(models) + require.Nil(t, err) + + for i, id := range ids { + res, err := clientSvc.GetById(id) + require.Nil(t, err) + assert.Equal(t, res.Name, models[i].Name) + } +} + +func TestModelServiceV2_Count(t *testing.T) { + setupTestDB() + defer teardownTestDB() + svr, err := server.NewGrpcServerV2() + require.Nil(t, err) + go svr.Start() + defer svr.Stop() + + modelSvc := service.NewModelServiceV2[TestModel]() + for i := 0; i < 5; i++ { + _, err = modelSvc.InsertOne(TestModel{ + Name: "Test Name", + }) + require.Nil(t, err) + } + time.Sleep(100 * time.Millisecond) + + c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + require.Nil(t, err) + c.Connect() + + clientSvc := client.NewModelServiceV2[TestModel]() + count, err := clientSvc.Count(bson.M{}) + require.Nil(t, err) + + assert.Equal(t, count, 5) +} diff --git a/core/models/client/model_spider_service.go b/core/models/client/model_spider_service.go new file mode 100644 index 000000000..53720498d --- /dev/null +++ b/core/models/client/model_spider_service.go @@ -0,0 +1,70 @@ +package client + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type SpiderServiceDelegate struct { + interfaces.GrpcClientModelBaseService +} + +func (svc *SpiderServiceDelegate) GetSpiderById(id primitive.ObjectID) (s interfaces.Spider, err error) { + res, err := svc.GetById(id) + if err != nil { + return nil, err + } + s, ok := res.(interfaces.Spider) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return s, nil +} + +func (svc *SpiderServiceDelegate) GetSpider(query bson.M, opts *mongo.FindOptions) (s interfaces.Spider, err error) { + res, err := svc.Get(query, opts) + if err != nil { + return nil, err + } + s, ok := res.(interfaces.Spider) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return s, nil +} + +func (svc *SpiderServiceDelegate) GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Spider, err error) { + list, err := svc.GetList(query, opts) + if err != nil { + return nil, err + } + for _, item := range list.GetModels() { + s, ok := item.(interfaces.Spider) + if !ok { + return nil, errors.ErrorModelInvalidType + } + res = append(res, s) + } + return res, nil +} + +func NewSpiderServiceDelegate() (svc2 interfaces.GrpcClientModelSpiderService, err error) { + var opts []ModelBaseServiceDelegateOption + + // apply options + opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdSpider)) + + // base service + baseSvc, err := NewBaseServiceDelegate(opts...) + if err != nil { + return nil, err + } + + // service + svc := &SpiderServiceDelegate{baseSvc} + + return svc, nil +} diff --git a/core/models/client/model_task_service.go b/core/models/client/model_task_service.go new file mode 100644 index 000000000..34b19e71d --- /dev/null +++ b/core/models/client/model_task_service.go @@ -0,0 +1,70 @@ +package client + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type TaskServiceDelegate struct { + interfaces.GrpcClientModelBaseService +} + +func (svc *TaskServiceDelegate) GetTaskById(id primitive.ObjectID) (t interfaces.Task, err error) { + res, err := svc.GetById(id) + if err != nil { + return nil, err + } + s, ok := res.(interfaces.Task) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return s, nil +} + +func (svc *TaskServiceDelegate) GetTask(query bson.M, opts *mongo.FindOptions) (t interfaces.Task, err error) { + res, err := svc.Get(query, opts) + if err != nil { + return nil, err + } + s, ok := res.(interfaces.Task) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return s, nil +} + +func (svc *TaskServiceDelegate) GetTaskList(query bson.M, opts *mongo.FindOptions) (res []interfaces.Task, err error) { + list, err := svc.GetList(query, opts) + if err != nil { + return nil, err + } + for _, item := range list.GetModels() { + s, ok := item.(interfaces.Task) + if !ok { + return nil, errors.ErrorModelInvalidType + } + res = append(res, s) + } + return res, nil +} + +func NewTaskServiceDelegate() (svc2 interfaces.GrpcClientModelTaskService, err error) { + var opts []ModelBaseServiceDelegateOption + + // apply options + opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdTask)) + + // base service + baseSvc, err := NewBaseServiceDelegate(opts...) + if err != nil { + return nil, err + } + + // service + svc := &TaskServiceDelegate{baseSvc} + + return svc, nil +} diff --git a/core/models/client/model_task_stat_service.go b/core/models/client/model_task_stat_service.go new file mode 100644 index 000000000..e9c69f56f --- /dev/null +++ b/core/models/client/model_task_stat_service.go @@ -0,0 +1,70 @@ +package client + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type TaskStatServiceDelegate struct { + interfaces.GrpcClientModelBaseService +} + +func (svc *TaskStatServiceDelegate) GetTaskStatById(id primitive.ObjectID) (t interfaces.TaskStat, err error) { + res, err := svc.GetById(id) + if err != nil { + return nil, err + } + s, ok := res.(interfaces.TaskStat) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return s, nil +} + +func (svc *TaskStatServiceDelegate) GetTaskStat(query bson.M, opts *mongo.FindOptions) (t interfaces.TaskStat, err error) { + res, err := svc.Get(query, opts) + if err != nil { + return nil, err + } + s, ok := res.(interfaces.TaskStat) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return s, nil +} + +func (svc *TaskStatServiceDelegate) GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []interfaces.TaskStat, err error) { + list, err := svc.GetList(query, opts) + if err != nil { + return nil, err + } + for _, item := range list.GetModels() { + s, ok := item.(interfaces.TaskStat) + if !ok { + return nil, errors.ErrorModelInvalidType + } + res = append(res, s) + } + return res, nil +} + +func NewTaskStatServiceDelegate() (svc2 interfaces.GrpcClientModelTaskStatService, err error) { + var opts []ModelBaseServiceDelegateOption + + // apply options + opts = append(opts, WithBaseServiceModelId(interfaces.ModelIdTaskStat)) + + // base service + baseSvc, err := NewBaseServiceDelegate(opts...) + if err != nil { + return nil, err + } + + // service + svc := &TaskStatServiceDelegate{baseSvc} + + return svc, nil +} diff --git a/core/models/client/options.go b/core/models/client/options.go new file mode 100644 index 000000000..e09668023 --- /dev/null +++ b/core/models/client/options.go @@ -0,0 +1,33 @@ +package client + +import "github.com/crawlab-team/crawlab/core/interfaces" + +type ModelDelegateOption func(delegate interfaces.GrpcClientModelDelegate) + +func WithDelegateConfigPath(path string) ModelDelegateOption { + return func(d interfaces.GrpcClientModelDelegate) { + d.SetConfigPath(path) + } +} + +type ModelServiceDelegateOption func(delegate interfaces.GrpcClientModelService) + +func WithServiceConfigPath(path string) ModelServiceDelegateOption { + return func(d interfaces.GrpcClientModelService) { + d.SetConfigPath(path) + } +} + +type ModelBaseServiceDelegateOption func(delegate interfaces.GrpcClientModelBaseService) + +func WithBaseServiceModelId(id interfaces.ModelId) ModelBaseServiceDelegateOption { + return func(d interfaces.GrpcClientModelBaseService) { + d.SetModelId(id) + } +} + +func WithBaseServiceConfigPath(path string) ModelBaseServiceDelegateOption { + return func(d interfaces.GrpcClientModelBaseService) { + d.SetConfigPath(path) + } +} diff --git a/core/models/common/index_service.go b/core/models/common/index_service.go new file mode 100644 index 000000000..44ddb0f5f --- /dev/null +++ b/core/models/common/index_service.go @@ -0,0 +1,145 @@ +package common + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func CreateIndexes() { + // artifacts + mongo.GetMongoCol(interfaces.ModelColNameArtifact).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"_col": 1}}, + {Keys: bson.M{"_del": 1}}, + {Keys: bson.M{"_tid": 1}}, + }) + + // tags + mongo.GetMongoCol(interfaces.ModelColNameTag).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"col": 1}}, + {Keys: bson.M{"name": 1}}, + }) + + // nodes + mongo.GetMongoCol(interfaces.ModelColNameNode).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"key": 1}}, // key + {Keys: bson.M{"name": 1}}, // name + {Keys: bson.M{"is_master": 1}}, // is_master + {Keys: bson.M{"status": 1}}, // status + {Keys: bson.M{"enabled": 1}}, // enabled + {Keys: bson.M{"active": 1}}, // active + }) + + // projects + mongo.GetMongoCol(interfaces.ModelColNameNode).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + }) + + // spiders + mongo.GetMongoCol(interfaces.ModelColNameSpider).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + {Keys: bson.M{"type": 1}}, + {Keys: bson.M{"col_id": 1}}, + {Keys: bson.M{"project_id": 1}}, + }) + + // tasks + mongo.GetMongoCol(interfaces.ModelColNameTask).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"spider_id": 1}}, + {Keys: bson.M{"status": 1}}, + {Keys: bson.M{"node_id": 1}}, + {Keys: bson.M{"schedule_id": 1}}, + {Keys: bson.M{"type": 1}}, + {Keys: bson.M{"mode": 1}}, + {Keys: bson.M{"priority": 1}}, + {Keys: bson.M{"parent_id": 1}}, + {Keys: bson.M{"has_sub": 1}}, + {Keys: bson.M{"create_ts": -1}}, + }) + + // task stats + mongo.GetMongoCol(interfaces.ModelColNameTaskStat).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"create_ts": 1}}, + }) + + // schedules + mongo.GetMongoCol(interfaces.ModelColNameSchedule).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + {Keys: bson.M{"spider_id": 1}}, + {Keys: bson.M{"enabled": 1}}, + }) + + // users + mongo.GetMongoCol(interfaces.ModelColNameUser).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"username": 1}}, + {Keys: bson.M{"role": 1}}, + {Keys: bson.M{"email": 1}}, + }) + + // settings + mongo.GetMongoCol(interfaces.ModelColNameSetting).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"key": 1}}, + }) + + // tokens + mongo.GetMongoCol(interfaces.ModelColNameToken).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + }) + + // variables + mongo.GetMongoCol(interfaces.ModelColNameVariable).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"key": 1}}, + }) + + // data sources + mongo.GetMongoCol(interfaces.ModelColNameDataSource).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + }) + + // data collections + mongo.GetMongoCol(interfaces.ModelColNameDataCollection).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + }) + + // extra values + mongo.GetMongoCol(interfaces.ModelColNameExtraValues).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"oid": 1}}, + {Keys: bson.M{"m": 1}}, + {Keys: bson.M{"t": 1}}, + {Keys: bson.M{"m": 1, "t": 1}}, + {Keys: bson.M{"oid": 1, "m": 1, "t": 1}}, + }) + + // roles + mongo.GetMongoCol(interfaces.ModelColNameRole).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, + }) + + // user role relations + mongo.GetMongoCol(interfaces.ModelColNameUserRole).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.D{{"user_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, + {Keys: bson.D{{"role_id", 1}, {"user_id", 1}}, Options: options.Index().SetUnique(true)}, + }) + + // permissions + mongo.GetMongoCol(interfaces.ModelColNamePermission).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, + }) + + // role permission relations + mongo.GetMongoCol(interfaces.ModelColNameRolePermission).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.D{{"role_id", 1}, {"permission_id", 1}}, Options: options.Index().SetUnique(true)}, + {Keys: bson.D{{"permission_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, + }) + + // cache + mongo.GetMongoCol(constants.CacheColName).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.M{constants.CacheColTime: 1}, + Options: options.Index().SetExpireAfterSeconds(3600 * 24), + }, + }) +} diff --git a/core/models/config_spider/common.go b/core/models/config_spider/common.go new file mode 100644 index 000000000..8ad3ad156 --- /dev/null +++ b/core/models/config_spider/common.go @@ -0,0 +1,24 @@ +package config_spider + +import "github.com/crawlab-team/crawlab/core/entity" + +func GetAllFields(data entity.ConfigSpiderData) []entity.Field { + var fields []entity.Field + for _, stage := range data.Stages { + fields = append(fields, stage.Fields...) + } + return fields +} + +func GetStartStageName(data entity.ConfigSpiderData) string { + // 如果 start_stage 设置了且在 stages 里，则返回 + if data.StartStage != "" { + return data.StartStage + } + + // 否则返回第一个 stage + for _, stage := range data.Stages { + return stage.Name + } + return "" +} diff --git a/core/models/config_spider/scrapy.go b/core/models/config_spider/scrapy.go new file mode 100644 index 000000000..8dab84616 --- /dev/null +++ b/core/models/config_spider/scrapy.go @@ -0,0 +1,263 @@ +package config_spider + +//import ( +// "errors" +// "fmt" +// "github.com/crawlab-team/crawlab/core/constants" +// "github.com/crawlab-team/crawlab/core/entity" +// "github.com/crawlab-team/crawlab/core/models" +// "github.com/crawlab-team/crawlab/core/utils" +// "path/filepath" +//) +// +//type ScrapyGenerator struct { +// Spider model.Spider +// ConfigData entity.ConfigSpiderData +//} +// +//// 生成爬虫文件 +//func (g ScrapyGenerator) Generate() error { +// // 生成 items.py +// if err := g.ProcessItems(); err != nil { +// return err +// } +// +// // 生成 spider.py +// if err := g.ProcessSpider(); err != nil { +// return err +// } +// return nil +//} +// +//// 生成 items.py +//func (g ScrapyGenerator) ProcessItems() error { +// // 待处理文件名 +// src := g.Spider.Src +// filePath := filepath.Join(src, "config_spider", "items.py") +// +// // 获取所有字段 +// fields := g.GetAllFields() +// +// // 字段名列表（包含默认字段名） +// fieldNames := []string{ +// "_id", +// "task_id", +// "ts", +// } +// +// // 加入字段 +// for _, field := range fields { +// fieldNames = append(fieldNames, field.Name) +// } +// +// // 将字段名转化为python代码 +// str := "" +// for _, fieldName := range fieldNames { +// line := g.PadCode(fmt.Sprintf("%s = scrapy.Field()", fieldName), 1) +// str += line +// } +// +// // 将占位符替换为代码 +// if err := utils.SetFileVariable(filePath, constants.AnchorItems, str); err != nil { +// return err +// } +// +// return nil +//} +// +//// 生成 spider.py +//func (g ScrapyGenerator) ProcessSpider() error { +// // 待处理文件名 +// src := g.Spider.Src +// filePath := filepath.Join(src, "config_spider", "spiders", "spider.py") +// +// // 替换 start_stage +// if err := utils.SetFileVariable(filePath, constants.AnchorStartStage, "parse_"+GetStartStageName(g.ConfigData)); err != nil { +// return err +// } +// +// // 替换 start_url +// if err := utils.SetFileVariable(filePath, constants.AnchorStartUrl, g.ConfigData.StartUrl); err != nil { +// return err +// } +// +// // 替换 parsers +// strParser := "" +// for _, stage := range g.ConfigData.Stages { +// stageName := stage.Name +// stageStr := g.GetParserString(stageName, stage) +// strParser += stageStr +// } +// if err := utils.SetFileVariable(filePath, constants.AnchorParsers, strParser); err != nil { +// return err +// } +// +// return nil +//} +// +//func (g ScrapyGenerator) GetParserString(stageName string, stage entity.Stage) string { +// // 构造函数定义行 +// strDef := g.PadCode(fmt.Sprintf("def parse_%s(self, response):", stageName), 1) +// +// strParse := "" +// if stage.IsList { +// // 列表逻辑 +// strParse = g.GetListParserString(stageName, stage) +// } else { +// // 非列表逻辑 +// strParse = g.GetNonListParserString(stageName, stage) +// } +// +// // 构造 +// str := fmt.Sprintf(`%s%s`, strDef, strParse) +// +// return str +//} +// +//func (g ScrapyGenerator) PadCode(str string, num int) string { +// res := "" +// for i := 0; i < num; i++ { +// res += " " +// } +// res += str +// res += "\n" +// return res +//} +// +//func (g ScrapyGenerator) GetNonListParserString(stageName string, stage entity.Stage) string { +// str := "" +// +// // 获取或构造item +// str += g.PadCode("item = Item() if response.meta.get('item') is None else response.meta.get('item')", 2) +// +// // 遍历字段列表 +// for _, f := range stage.Fields { +// line := fmt.Sprintf(`item['%s'] = response.%s.extract_first()`, f.Name, g.GetExtractStringFromField(f)) +// line = g.PadCode(line, 2) +// str += line +// } +// +// // next stage 字段 +// if f, err := g.GetNextStageField(stage); err == nil { +// // 如果找到 next stage 字段，进行下一个回调 +// str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url="get_real_url(response, item['%s'])", callback=self.parse_%s, meta={'item': item})`, f.Name, f.NextStage), 2) +// } else { +// // 如果没找到 next stage 字段，返回 item +// str += g.PadCode(fmt.Sprintf(`yield item`), 2) +// } +// +// // 加入末尾换行 +// str += g.PadCode("", 0) +// +// return str +//} +// +//func (g ScrapyGenerator) GetListParserString(stageName string, stage entity.Stage) string { +// str := "" +// +// // 获取前一个 stage 的 item +// str += g.PadCode(`prev_item = response.meta.get('item')`, 2) +// +// // for 循环遍历列表 +// str += g.PadCode(fmt.Sprintf(`for elem in response.%s:`, g.GetListString(stage)), 2) +// +// // 构造item +// str += g.PadCode(`item = Item()`, 3) +// +// // 遍历字段列表 +// for _, f := range stage.Fields { +// line := fmt.Sprintf(`item['%s'] = elem.%s.extract_first()`, f.Name, g.GetExtractStringFromField(f)) +// line = g.PadCode(line, 3) +// str += line +// } +// +// // 把前一个 stage 的 item 值赋给当前 item +// str += g.PadCode(`if prev_item is not None:`, 3) +// str += g.PadCode(`for key, value in prev_item.items():`, 4) +// str += g.PadCode(`item[key] = value`, 5) +// +// // next stage 字段 +// if f, err := g.GetNextStageField(stage); err == nil { +// // 如果 url 为空，则不进入下一个 stage +// str += g.PadCode(fmt.Sprintf(`if not item['%s']:`, f.Name), 3) +// str += g.PadCode(`continue`, 4) +// +// // 如果找到 next stage 字段，进行下一个回调 +// str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, item['%s']), callback=self.parse_%s, meta={'item': item})`, f.Name, f.NextStage), 3) +// } else { +// // 如果没找到 next stage 字段，返回 item +// str += g.PadCode(fmt.Sprintf(`yield item`), 3) +// } +// +// // 分页 +// if stage.PageCss != "" || stage.PageXpath != "" { +// str += g.PadCode(fmt.Sprintf(`next_url = response.%s.extract_first()`, g.GetExtractStringFromStage(stage)), 2) +// str += g.PadCode(fmt.Sprintf(`yield scrapy.Request(url=get_real_url(response, next_url), callback=self.parse_%s, meta={'item': prev_item})`, stageName), 2) +// } +// +// // 加入末尾换行 +// str += g.PadCode("", 0) +// +// return str +//} +// +//// 获取所有字段 +//func (g ScrapyGenerator) GetAllFields() []entity.Field { +// return GetAllFields(g.ConfigData) +//} +// +//// 获取包含 next stage 的字段 +//func (g ScrapyGenerator) GetNextStageField(stage entity.Stage) (entity.Field, error) { +// for _, field := range stage.Fields { +// if field.NextStage != "" { +// return field, nil +// } +// } +// return entity.Field{}, errors.New("cannot find next stage field") +//} +// +//func (g ScrapyGenerator) GetExtractStringFromField(f entity.Field) string { +// if f.Css != "" { +// // 如果为CSS +// if f.Attr == "" { +// // 文本 +// return fmt.Sprintf(`css('%s::text')`, f.Css) +// } else { +// // 属性 +// return fmt.Sprintf(`css('%s::attr("%s")')`, f.Css, f.Attr) +// } +// } else { +// // 如果为XPath +// if f.Attr == "" { +// // 文本 +// return fmt.Sprintf(`xpath('string(%s)')`, f.Xpath) +// } else { +// // 属性 +// return fmt.Sprintf(`xpath('%s/@%s')`, f.Xpath, f.Attr) +// } +// } +//} +// +//func (g ScrapyGenerator) GetExtractStringFromStage(stage entity.Stage) string { +// // 分页元素属性，默认为 href +// pageAttr := "href" +// if stage.PageAttr != "" { +// pageAttr = stage.PageAttr +// } +// +// if stage.PageCss != "" { +// // 如果为CSS +// return fmt.Sprintf(`css('%s::attr("%s")')`, stage.PageCss, pageAttr) +// } else { +// // 如果为XPath +// return fmt.Sprintf(`xpath('%s/@%s')`, stage.PageXpath, pageAttr) +// } +//} +// +//func (g ScrapyGenerator) GetListString(stage entity.Stage) string { +// if stage.ListCss != "" { +// return fmt.Sprintf(`css('%s')`, stage.ListCss) +// } else { +// return fmt.Sprintf(`xpath('%s')`, stage.ListXpath) +// } +//} diff --git a/core/models/delegate/base_test.go b/core/models/delegate/base_test.go new file mode 100644 index 000000000..3ad962bc5 --- /dev/null +++ b/core/models/delegate/base_test.go @@ -0,0 +1,25 @@ +package delegate_test + +import ( + "context" + "github.com/crawlab-team/crawlab-db/mongo" + "go.mongodb.org/mongo-driver/bson" + "testing" + "time" +) + +func SetupTest(t *testing.T) { + CleanupTest() + t.Cleanup(CleanupTest) +} + +func CleanupTest() { + db := mongo.GetMongoDb("") + names, _ := db.ListCollectionNames(context.Background(), bson.M{}) + for _, n := range names { + _, _ = db.Collection(n).DeleteMany(context.Background(), bson.M{}) + } + + // avoid caching + time.Sleep(200 * time.Millisecond) +} diff --git a/core/models/delegate/model.go b/core/models/delegate/model.go new file mode 100644 index 000000000..a25ee5d05 --- /dev/null +++ b/core/models/delegate/model.go @@ -0,0 +1,362 @@ +package delegate + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab-db/errors" + "github.com/crawlab-team/crawlab-db/mongo" + errors2 "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/event" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "reflect" + "time" +) + +func NewModelDelegate(doc interfaces.Model, args ...interface{}) interfaces.ModelDelegate { + switch doc.(type) { + case *models.Artifact: + return newModelDelegate(interfaces.ModelIdArtifact, doc, args...) + case *models.Tag: + return newModelDelegate(interfaces.ModelIdTag, doc, args...) + case *models.Node: + return newModelDelegate(interfaces.ModelIdNode, doc, args...) + case *models.Project: + return newModelDelegate(interfaces.ModelIdProject, doc, args...) + case *models.Spider: + return newModelDelegate(interfaces.ModelIdSpider, doc, args...) + case *models.Task: + return newModelDelegate(interfaces.ModelIdTask, doc, args...) + case *models.Job: + return newModelDelegate(interfaces.ModelIdJob, doc, args...) + case *models.Schedule: + return newModelDelegate(interfaces.ModelIdSchedule, doc, args...) + case *models.User: + return newModelDelegate(interfaces.ModelIdUser, doc, args...) + case *models.Setting: + return newModelDelegate(interfaces.ModelIdSetting, doc, args...) + case *models.Token: + return newModelDelegate(interfaces.ModelIdToken, doc, args...) + case *models.Variable: + return newModelDelegate(interfaces.ModelIdVariable, doc, args...) + case *models.TaskQueueItem: + return newModelDelegate(interfaces.ModelIdTaskQueue, doc, args...) + case *models.TaskStat: + return newModelDelegate(interfaces.ModelIdTaskStat, doc, args...) + case *models.SpiderStat: + return newModelDelegate(interfaces.ModelIdSpiderStat, doc, args...) + case *models.DataSource: + return newModelDelegate(interfaces.ModelIdDataSource, doc, args...) + case *models.DataCollection: + return newModelDelegate(interfaces.ModelIdDataCollection, doc, args...) + case *models.Result: + return newModelDelegate(interfaces.ModelIdResult, doc, args...) + case *models.Password: + return newModelDelegate(interfaces.ModelIdPassword, doc, args...) + case *models.ExtraValue: + return newModelDelegate(interfaces.ModelIdExtraValue, doc, args...) + case *models.Git: + return newModelDelegate(interfaces.ModelIdGit, doc, args...) + case *models.Role: + return newModelDelegate(interfaces.ModelIdRole, doc, args...) + case *models.UserRole: + return newModelDelegate(interfaces.ModelIdUserRole, doc, args...) + case *models.Permission: + return newModelDelegate(interfaces.ModelIdPermission, doc, args...) + case *models.RolePermission: + return newModelDelegate(interfaces.ModelIdRolePermission, doc, args...) + case *models.Environment: + return newModelDelegate(interfaces.ModelIdEnvironment, doc, args...) + case *models.DependencySetting: + return newModelDelegate(interfaces.ModelIdDependencySetting, doc, args...) + default: + _ = trace.TraceError(errors2.ErrorModelInvalidType) + return nil + } +} + +func newModelDelegate(id interfaces.ModelId, doc interfaces.Model, args ...interface{}) interfaces.ModelDelegate { + // user + u := utils.GetUserFromArgs(args...) + + // collection name + colName := models.GetModelColName(id) + + // model delegate + d := &ModelDelegate{ + id: id, + colName: colName, + doc: doc, + a: &models.Artifact{ + Col: colName, + }, + u: u, + } + + return d +} + +type ModelDelegate struct { + id interfaces.ModelId + colName string + doc interfaces.Model // doc to delegate + cd bson.M // current doc + od bson.M // original doc + a interfaces.ModelArtifact // artifact + u interfaces.User // user +} + +// Add model +func (d *ModelDelegate) Add() (err error) { + return d.do(interfaces.ModelDelegateMethodAdd) +} + +// Save model +func (d *ModelDelegate) Save() (err error) { + return d.do(interfaces.ModelDelegateMethodSave) +} + +// Delete model +func (d *ModelDelegate) Delete() (err error) { + return d.do(interfaces.ModelDelegateMethodDelete) +} + +// GetArtifact refresh artifact and return it +func (d *ModelDelegate) GetArtifact() (res interfaces.ModelArtifact, err error) { + if err := d.do(interfaces.ModelDelegateMethodGetArtifact); err != nil { + return nil, err + } + return d.a, nil +} + +// Refresh model +func (d *ModelDelegate) Refresh() (err error) { + return d.refresh() +} + +// GetModel return model +func (d *ModelDelegate) GetModel() (res interfaces.Model) { + return d.doc +} + +func (d *ModelDelegate) ToBytes(m interface{}) (bytes []byte, err error) { + if m != nil { + return utils.JsonToBytes(m) + } + return json.Marshal(d.doc) +} + +// do action given the model delegate method +func (d *ModelDelegate) do(method interfaces.ModelDelegateMethod) (err error) { + switch method { + case interfaces.ModelDelegateMethodAdd: + err = d.add() + case interfaces.ModelDelegateMethodSave: + err = d.save() + case interfaces.ModelDelegateMethodDelete: + err = d.delete() + case interfaces.ModelDelegateMethodGetArtifact, interfaces.ModelDelegateMethodRefresh: + err = d.refresh() + default: + return trace.TraceError(errors2.ErrorModelInvalidType) + } + + if err != nil { + return err + } + + // trigger event + eventName := GetEventName(d, method) + go event.SendEvent(eventName, d.doc) + + return nil +} + +// add model +func (d *ModelDelegate) add() (err error) { + if d.doc == nil { + return trace.TraceError(errors.ErrMissingValue) + } + if d.doc.GetId().IsZero() { + d.doc.SetId(primitive.NewObjectID()) + } + col := mongo.GetMongoCol(d.colName) + if _, err = col.Insert(d.doc); err != nil { + return trace.TraceError(err) + } + if err := d.upsertArtifact(); err != nil { + return trace.TraceError(err) + } + return d.refresh() +} + +// save model +func (d *ModelDelegate) save() (err error) { + // validate + if d.doc == nil || d.doc.GetId().IsZero() { + return trace.TraceError(errors.ErrMissingValue) + } + + // collection + col := mongo.GetMongoCol(d.colName) + + // current doc + docData, err := bson.Marshal(d.doc) + if err != nil { + trace.PrintError(err) + } else { + if err := bson.Unmarshal(docData, &d.cd); err != nil { + trace.PrintError(err) + } + } + + // original doc + if err := col.FindId(d.doc.GetId()).One(&d.od); err != nil { + trace.PrintError(err) + } + + // replace + if err := col.ReplaceId(d.doc.GetId(), d.doc); err != nil { + return trace.TraceError(err) + } + + // upsert artifact + if err := d.upsertArtifact(); err != nil { + return trace.TraceError(err) + } + + return d.refresh() +} + +// delete model +func (d *ModelDelegate) delete() (err error) { + if d.doc.GetId().IsZero() { + return trace.TraceError(errors2.ErrorModelMissingId) + } + col := mongo.GetMongoCol(d.colName) + if err := col.FindId(d.doc.GetId()).One(d.doc); err != nil { + return trace.TraceError(err) + } + if err := col.DeleteId(d.doc.GetId()); err != nil { + return trace.TraceError(err) + } + return d.deleteArtifact() +} + +// refresh model and artifact +func (d *ModelDelegate) refresh() (err error) { + if d.doc.GetId().IsZero() { + return trace.TraceError(errors2.ErrorModelMissingId) + } + col := mongo.GetMongoCol(d.colName) + fr := col.FindId(d.doc.GetId()) + if err := fr.One(d.doc); err != nil { + return trace.TraceError(err) + } + return d.refreshArtifact() +} + +// refresh artifact +func (d *ModelDelegate) refreshArtifact() (err error) { + if d.doc.GetId().IsZero() { + return trace.TraceError(errors2.ErrorModelMissingId) + } + col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) + if err := col.FindId(d.doc.GetId()).One(d.a); err != nil { + return trace.TraceError(err) + } + return nil +} + +// upsertArtifact +func (d *ModelDelegate) upsertArtifact() (err error) { + // skip + if d._skip() { + return nil + } + + // validate id + if d.doc.GetId().IsZero() { + return trace.TraceError(errors.ErrMissingValue) + } + + // mongo collection + col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) + + // assign id to artifact + d.a.SetId(d.doc.GetId()) + + // attempt to find artifact + if err := col.FindId(d.doc.GetId()).One(d.a); err != nil { + if err == mongo2.ErrNoDocuments { + // new artifact + d.a.GetSys().SetCreateTs(time.Now()) + d.a.GetSys().SetUpdateTs(time.Now()) + if d.u != nil && !reflect.ValueOf(d.u).IsZero() { + d.a.GetSys().SetCreateUid(d.u.GetId()) + d.a.GetSys().SetUpdateUid(d.u.GetId()) + } + _, err = col.Insert(d.a) + if err != nil { + return trace.TraceError(err) + } + return nil + } else { + // error + return trace.TraceError(err) + } + } + + // existing artifact + d.a.GetSys().SetUpdateTs(time.Now()) + if d.u != nil { + d.a.GetSys().SetUpdateUid(d.u.GetId()) + } + + // save new artifact + return col.ReplaceId(d.a.GetId(), d.a) +} + +// deleteArtifact +func (d *ModelDelegate) deleteArtifact() (err error) { + // skip + if d._skip() { + return nil + } + + if d.doc.GetId().IsZero() { + return trace.TraceError(errors.ErrMissingValue) + } + col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) + d.a.SetId(d.doc.GetId()) + d.a.SetObj(d.doc) + d.a.SetDel(true) + d.a.GetSys().SetDeleteTs(time.Now()) + if d.u != nil { + d.a.GetSys().SetDeleteUid(d.u.GetId()) + } + return col.ReplaceId(d.doc.GetId(), d.a) +} + +func (d *ModelDelegate) hasChange() (ok bool) { + return !utils.BsonMEqual(d.cd, d.od) +} + +func (d *ModelDelegate) _skip() (ok bool) { + switch d.id { + case + interfaces.ModelIdArtifact, + interfaces.ModelIdTaskQueue, + interfaces.ModelIdTaskStat, + interfaces.ModelIdSpiderStat, + interfaces.ModelIdResult, + interfaces.ModelIdPassword: + return true + default: + return false + } +} diff --git a/core/models/delegate/model_node.go b/core/models/delegate/model_node.go new file mode 100644 index 000000000..99bdd11d2 --- /dev/null +++ b/core/models/delegate/model_node.go @@ -0,0 +1,37 @@ +package delegate + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type ModelNodeDelegate struct { + n interfaces.Node + interfaces.ModelDelegate +} + +func (d *ModelNodeDelegate) UpdateStatus(active bool, activeTs *time.Time, status string) (err error) { + d.n.SetActive(active) + if activeTs != nil { + d.n.SetActiveTs(*activeTs) + } + d.n.SetStatus(status) + return d.Save() +} + +func (d *ModelNodeDelegate) UpdateStatusOnline() (err error) { + now := time.Now() + return d.UpdateStatus(true, &now, constants.NodeStatusOnline) +} + +func (d *ModelNodeDelegate) UpdateStatusOffline() (err error) { + return d.UpdateStatus(false, nil, constants.NodeStatusOffline) +} + +func NewModelNodeDelegate(n interfaces.Node) interfaces.ModelNodeDelegate { + return &ModelNodeDelegate{ + n: n, + ModelDelegate: NewModelDelegate(n), + } +} diff --git a/core/models/delegate/model_node_test.go b/core/models/delegate/model_node_test.go new file mode 100644 index 000000000..0a95c87e9 --- /dev/null +++ b/core/models/delegate/model_node_test.go @@ -0,0 +1,65 @@ +package delegate_test + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "github.com/stretchr/testify/require" + "testing" +) + +func TestNode_Add(t *testing.T) { + SetupTest(t) + + n := &models2.Node{} + + err := delegate.NewModelDelegate(n).Add() + require.Nil(t, err) + require.NotNil(t, n.Id) + + // validate artifact + a, err := delegate.NewModelDelegate(n).GetArtifact() + require.Nil(t, err) + require.Equal(t, n.Id, a.GetId()) + require.NotNil(t, a.GetSys().GetCreateTs()) + require.NotNil(t, a.GetSys().GetUpdateTs()) +} + +func TestNode_Save(t *testing.T) { + SetupTest(t) + + n := &models2.Node{} + + err := delegate.NewModelDelegate(n).Add() + + name := "test_node" + n.Name = name + err = delegate.NewModelDelegate(n).Save() + require.Nil(t, err) + + err = mongo.GetMongoCol(interfaces.ModelColNameNode).FindId(n.Id).One(&n) + require.Nil(t, err) + require.Equal(t, name, n.Name) +} + +func TestNode_Delete(t *testing.T) { + SetupTest(t) + + n := &models2.Node{ + Name: "test_node", + } + + err := delegate.NewModelDelegate(n).Add() + require.Nil(t, err) + + err = delegate.NewModelDelegate(n).Delete() + require.Nil(t, err) + + var a models2.Artifact + col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) + err = col.FindId(n.Id).One(&a) + require.Nil(t, err) + require.NotNil(t, a.Obj) + require.True(t, a.Del) +} diff --git a/core/models/delegate/model_role_test.go b/core/models/delegate/model_role_test.go new file mode 100644 index 000000000..2d4a1c5e4 --- /dev/null +++ b/core/models/delegate/model_role_test.go @@ -0,0 +1,65 @@ +package delegate_test + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "github.com/stretchr/testify/require" + "testing" +) + +func TestRole_Add(t *testing.T) { + SetupTest(t) + + p := &models2.Role{} + + err := delegate.NewModelDelegate(p).Add() + require.Nil(t, err) + require.NotNil(t, p.Id) + + a, err := delegate.NewModelDelegate(p).GetArtifact() + require.Nil(t, err) + require.Equal(t, p.Id, a.GetId()) + require.NotNil(t, a.GetSys().GetCreateTs()) + require.NotNil(t, a.GetSys().GetUpdateTs()) +} + +func TestRole_Save(t *testing.T) { + SetupTest(t) + + p := &models2.Role{} + + err := delegate.NewModelDelegate(p).Add() + require.Nil(t, err) + + name := "test_role" + p.Name = name + err = delegate.NewModelDelegate(p).Save() + require.Nil(t, err) + + err = mongo.GetMongoCol(interfaces.ModelColNameRole).FindId(p.Id).One(&p) + require.Nil(t, err) + require.Equal(t, name, p.Name) +} + +func TestRole_Delete(t *testing.T) { + SetupTest(t) + + p := &models2.Role{ + Name: "test_role", + } + + err := delegate.NewModelDelegate(p).Add() + require.Nil(t, err) + + err = delegate.NewModelDelegate(p).Delete() + require.Nil(t, err) + + var a models2.Artifact + col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) + err = col.FindId(p.Id).One(&a) + require.Nil(t, err) + require.NotNil(t, a.Obj) + require.True(t, a.Del) +} diff --git a/core/models/delegate/model_test.go b/core/models/delegate/model_test.go new file mode 100644 index 000000000..c23aaa953 --- /dev/null +++ b/core/models/delegate/model_test.go @@ -0,0 +1,65 @@ +package delegate_test + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "github.com/stretchr/testify/require" + "testing" +) + +func TestProject_Add(t *testing.T) { + SetupTest(t) + + p := &models2.Project{} + + err := delegate.NewModelDelegate(p).Add() + require.Nil(t, err) + require.NotNil(t, p.Id) + + a, err := delegate.NewModelDelegate(p).GetArtifact() + require.Nil(t, err) + require.Equal(t, p.Id, a.GetId()) + require.NotNil(t, a.GetSys().GetCreateTs()) + require.NotNil(t, a.GetSys().GetUpdateTs()) +} + +func TestProject_Save(t *testing.T) { + SetupTest(t) + + p := &models2.Project{} + + err := delegate.NewModelDelegate(p).Add() + require.Nil(t, err) + + name := "test_project" + p.Name = name + err = delegate.NewModelDelegate(p).Save() + require.Nil(t, err) + + err = mongo.GetMongoCol(interfaces.ModelColNameProject).FindId(p.Id).One(&p) + require.Nil(t, err) + require.Equal(t, name, p.Name) +} + +func TestProject_Delete(t *testing.T) { + SetupTest(t) + + p := &models2.Project{ + Name: "test_project", + } + + err := delegate.NewModelDelegate(p).Add() + require.Nil(t, err) + + err = delegate.NewModelDelegate(p).Delete() + require.Nil(t, err) + + var a models2.Artifact + col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) + err = col.FindId(p.Id).One(&a) + require.Nil(t, err) + require.NotNil(t, a.Obj) + require.True(t, a.Del) +} diff --git a/core/models/delegate/model_user_role_test.go b/core/models/delegate/model_user_role_test.go new file mode 100644 index 000000000..d1c87a96d --- /dev/null +++ b/core/models/delegate/model_user_role_test.go @@ -0,0 +1,97 @@ +package delegate_test + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/common" + "github.com/crawlab-team/crawlab/core/models/delegate" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson/primitive" + "testing" +) + +func init() { + viper.Set("mongo.db", "crawlab_test") + common.CreateIndexes() +} + +func TestUserRole_Add(t *testing.T) { + SetupTest(t) + + p := &models2.UserRole{} + + err := delegate.NewModelDelegate(p).Add() + require.Nil(t, err) + require.NotNil(t, p.Id) + + a, err := delegate.NewModelDelegate(p).GetArtifact() + require.Nil(t, err) + require.Equal(t, p.Id, a.GetId()) + require.NotNil(t, a.GetSys().GetCreateTs()) + require.NotNil(t, a.GetSys().GetUpdateTs()) +} + +func TestUserRole_Save(t *testing.T) { + SetupTest(t) + + p := &models2.UserRole{ + UserId: primitive.NewObjectID(), + RoleId: primitive.NewObjectID(), + } + + err := delegate.NewModelDelegate(p).Add() + require.Nil(t, err) + + uid := primitive.NewObjectID() + rid := primitive.NewObjectID() + p.UserId = uid + p.RoleId = rid + err = delegate.NewModelDelegate(p).Save() + require.Nil(t, err) + + err = mongo.GetMongoCol(interfaces.ModelColNameUserRole).FindId(p.Id).One(&p) + require.Nil(t, err) + require.Equal(t, uid, p.UserId) + require.Equal(t, rid, p.RoleId) +} + +func TestUserRole_Delete(t *testing.T) { + SetupTest(t) + + p := &models2.UserRole{} + + err := delegate.NewModelDelegate(p).Add() + require.Nil(t, err) + + err = delegate.NewModelDelegate(p).Delete() + require.Nil(t, err) + + var a models2.Artifact + col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) + err = col.FindId(p.Id).One(&a) + require.Nil(t, err) + require.NotNil(t, a.Obj) + require.True(t, a.Del) +} + +func TestUserRole_AddDuplicates(t *testing.T) { + SetupTest(t) + + uid := primitive.NewObjectID() + rid := primitive.NewObjectID() + p := &models2.UserRole{ + UserId: uid, + RoleId: rid, + } + p2 := &models2.UserRole{ + UserId: uid, + RoleId: rid, + } + + err := delegate.NewModelDelegate(p).Add() + require.Nil(t, err) + err = delegate.NewModelDelegate(p2).Add() + require.NotNil(t, err) +} diff --git a/core/models/delegate/utils_event.go b/core/models/delegate/utils_event.go new file mode 100644 index 000000000..d5cd3f89b --- /dev/null +++ b/core/models/delegate/utils_event.go @@ -0,0 +1,20 @@ +package delegate + +import ( + "fmt" + "github.com/crawlab-team/crawlab/core/interfaces" +) + +func GetEventName(d *ModelDelegate, method interfaces.ModelDelegateMethod) (eventName string) { + return getEventName(d, method) +} + +func getEventName(d *ModelDelegate, method interfaces.ModelDelegateMethod) (eventName string) { + if method == interfaces.ModelDelegateMethodSave { + hasChange := d.hasChange() + if hasChange { + method = interfaces.ModelDelegateMethodChange + } + } + return fmt.Sprintf("model:%s:%s", d.colName, method) +} diff --git a/core/models/models/artifact.go b/core/models/models/artifact.go new file mode 100644 index 000000000..55e07af87 --- /dev/null +++ b/core/models/models/artifact.go @@ -0,0 +1,56 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Artifact struct { + Id primitive.ObjectID `bson:"_id" json:"_id"` + Col string `bson:"_col" json:"_col"` + Del bool `bson:"_del" json:"_del"` + TagIds []primitive.ObjectID `bson:"_tid" json:"_tid"` + Sys *ArtifactSys `bson:"_sys" json:"_sys"` + Obj interface{} `bson:"_obj" json:"_obj"` +} + +func (a *Artifact) GetId() (id primitive.ObjectID) { + return a.Id +} + +func (a *Artifact) SetId(id primitive.ObjectID) { + a.Id = id +} + +func (a *Artifact) GetSys() (sys interfaces.ModelArtifactSys) { + if a.Sys == nil { + a.Sys = &ArtifactSys{} + } + return a.Sys +} + +func (a *Artifact) GetTagIds() (ids []primitive.ObjectID) { + return a.TagIds +} + +func (a *Artifact) SetTagIds(ids []primitive.ObjectID) { + a.TagIds = ids +} + +func (a *Artifact) SetObj(obj interfaces.Model) { + a.Obj = obj +} + +func (a *Artifact) SetDel(del bool) { + a.Del = del +} + +type ArtifactList []Artifact + +func (l *ArtifactList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/artifact_sys.go b/core/models/models/artifact_sys.go new file mode 100644 index 000000000..539d8fb00 --- /dev/null +++ b/core/models/models/artifact_sys.go @@ -0,0 +1,63 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type ArtifactSys struct { + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + CreateUid primitive.ObjectID `json:"create_uid" bson:"create_uid"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` + UpdateUid primitive.ObjectID `json:"update_uid" bson:"update_uid"` + DeleteTs time.Time `json:"delete_ts" bson:"delete_ts"` + DeleteUid primitive.ObjectID `json:"delete_uid" bson:"delete_uid"` +} + +func (sys *ArtifactSys) GetCreateTs() time.Time { + return sys.CreateTs +} + +func (sys *ArtifactSys) SetCreateTs(ts time.Time) { + sys.CreateTs = ts +} + +func (sys *ArtifactSys) GetUpdateTs() time.Time { + return sys.UpdateTs +} + +func (sys *ArtifactSys) SetUpdateTs(ts time.Time) { + sys.UpdateTs = ts +} + +func (sys *ArtifactSys) GetDeleteTs() time.Time { + return sys.DeleteTs +} + +func (sys *ArtifactSys) SetDeleteTs(ts time.Time) { + sys.DeleteTs = ts +} + +func (sys *ArtifactSys) GetCreateUid() primitive.ObjectID { + return sys.CreateUid +} + +func (sys *ArtifactSys) SetCreateUid(id primitive.ObjectID) { + sys.CreateUid = id +} + +func (sys *ArtifactSys) GetUpdateUid() primitive.ObjectID { + return sys.UpdateUid +} + +func (sys *ArtifactSys) SetUpdateUid(id primitive.ObjectID) { + sys.UpdateUid = id +} + +func (sys *ArtifactSys) GetDeleteUid() primitive.ObjectID { + return sys.DeleteUid +} + +func (sys *ArtifactSys) SetDeleteUid(id primitive.ObjectID) { + sys.DeleteUid = id +} diff --git a/core/models/models/base.go b/core/models/models/base.go new file mode 100644 index 000000000..a17bb98f3 --- /dev/null +++ b/core/models/models/base.go @@ -0,0 +1,13 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type BaseModel struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` +} + +func (d *BaseModel) GetId() (id primitive.ObjectID) { + return d.Id +} diff --git a/core/models/models/base_v2.go b/core/models/models/base_v2.go new file mode 100644 index 000000000..cc1468e67 --- /dev/null +++ b/core/models/models/base_v2.go @@ -0,0 +1,64 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type BaseModelV2[T any] struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + CreatedAt time.Time `json:"created_ts" bson:"created_ts"` + CreatedBy primitive.ObjectID `json:"created_by" bson:"created_by"` + UpdatedAt time.Time `json:"updated_ts" bson:"updated_ts"` + UpdatedBy primitive.ObjectID `json:"updated_by" bson:"updated_by"` +} + +func (m *BaseModelV2[T]) GetId() primitive.ObjectID { + return m.Id +} + +func (m *BaseModelV2[T]) SetId(id primitive.ObjectID) { + m.Id = id +} + +func (m *BaseModelV2[T]) GetCreatedAt() time.Time { + return m.CreatedAt +} + +func (m *BaseModelV2[T]) SetCreatedAt(t time.Time) { + m.CreatedAt = t +} + +func (m *BaseModelV2[T]) GetCreatedBy() primitive.ObjectID { + return m.CreatedBy +} + +func (m *BaseModelV2[T]) SetCreatedBy(id primitive.ObjectID) { + m.CreatedBy = id +} + +func (m *BaseModelV2[T]) GetUpdatedAt() time.Time { + return m.UpdatedAt +} + +func (m *BaseModelV2[T]) SetUpdatedAt(t time.Time) { + m.UpdatedAt = t +} + +func (m *BaseModelV2[T]) GetUpdatedBy() primitive.ObjectID { + return m.UpdatedBy +} + +func (m *BaseModelV2[T]) SetUpdatedBy(id primitive.ObjectID) { + m.UpdatedBy = id +} + +func (m *BaseModelV2[T]) SetCreated(id primitive.ObjectID) { + m.SetCreatedAt(time.Now()) + m.SetCreatedBy(id) +} + +func (m *BaseModelV2[T]) SetUpdated(id primitive.ObjectID) { + m.SetUpdatedAt(time.Now()) + m.SetUpdatedBy(id) +} diff --git a/core/models/models/data_collection.go b/core/models/models/data_collection.go new file mode 100644 index 000000000..dfc6fb238 --- /dev/null +++ b/core/models/models/data_collection.go @@ -0,0 +1,36 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type DataCollection struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Name string `json:"name" bson:"name"` + Fields []entity.DataField `json:"fields" bson:"fields"` + Dedup struct { + Enabled bool `json:"enabled" bson:"enabled"` + Keys []string `json:"keys" bson:"keys"` + Type string `json:"type" bson:"type"` + } `json:"dedup" bson:"dedup"` +} + +func (dc *DataCollection) GetId() (id primitive.ObjectID) { + return dc.Id +} + +func (dc *DataCollection) SetId(id primitive.ObjectID) { + dc.Id = id +} + +type DataCollectionList []DataCollection + +func (l *DataCollectionList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/data_collection_v2.go b/core/models/models/data_collection_v2.go new file mode 100644 index 000000000..1a3c724e3 --- /dev/null +++ b/core/models/models/data_collection_v2.go @@ -0,0 +1,17 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/entity" +) + +type DataCollectionV2 struct { + any `collection:"data_collections"` + BaseModelV2[DataCollection] `bson:",inline"` + Name string `json:"name" bson:"name"` + Fields []entity.DataField `json:"fields" bson:"fields"` + Dedup struct { + Enabled bool `json:"enabled" bson:"enabled"` + Keys []string `json:"keys" bson:"keys"` + Type string `json:"type" bson:"type"` + } `json:"dedup" bson:"dedup"` +} diff --git a/core/models/models/data_source.go b/core/models/models/data_source.go new file mode 100644 index 000000000..f444266f5 --- /dev/null +++ b/core/models/models/data_source.go @@ -0,0 +1,42 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type DataSource struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Name string `json:"name" bson:"name"` + Type string `json:"type" bson:"type"` + Description string `json:"description" bson:"description"` + Host string `json:"host" bson:"host"` + Port string `json:"port" bson:"port"` + Url string `json:"url" bson:"url"` + Hosts []string `json:"hosts" bson:"hosts"` + Database string `json:"database" bson:"database"` + Username string `json:"username" bson:"username"` + Password string `json:"password,omitempty" bson:"-"` + ConnectType string `json:"connect_type" bson:"connect_type"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"` +} + +func (ds *DataSource) GetId() (id primitive.ObjectID) { + return ds.Id +} + +func (ds *DataSource) SetId(id primitive.ObjectID) { + ds.Id = id +} + +type DataSourceList []DataSource + +func (l *DataSourceList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/data_source_v2.go b/core/models/models/data_source_v2.go new file mode 100644 index 000000000..eb81e199f --- /dev/null +++ b/core/models/models/data_source_v2.go @@ -0,0 +1,20 @@ +package models + +type DataSourceV2 struct { + any `collection:"data_sources"` + BaseModelV2[DataSource] `bson:",inline"` + Name string `json:"name" bson:"name"` + Type string `json:"type" bson:"type"` + Description string `json:"description" bson:"description"` + Host string `json:"host" bson:"host"` + Port string `json:"port" bson:"port"` + Url string `json:"url" bson:"url"` + Hosts []string `json:"hosts" bson:"hosts"` + Database string `json:"database" bson:"database"` + Username string `json:"username" bson:"username"` + Password string `json:"password,omitempty" bson:"-"` + ConnectType string `json:"connect_type" bson:"connect_type"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"` +} diff --git a/core/models/models/dependency_setting.go b/core/models/models/dependency_setting.go new file mode 100644 index 000000000..d6232ede6 --- /dev/null +++ b/core/models/models/dependency_setting.go @@ -0,0 +1,36 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type DependencySetting struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + Cmd string `json:"cmd" bson:"cmd"` + Proxy string `json:"proxy" bson:"proxy"` + LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"` +} + +func (j *DependencySetting) GetId() (id primitive.ObjectID) { + return j.Id +} + +func (j *DependencySetting) SetId(id primitive.ObjectID) { + j.Id = id +} + +type DependencySettingList []DependencySetting + +func (l *DependencySettingList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/dependency_setting_v2.go b/core/models/models/dependency_setting_v2.go new file mode 100644 index 000000000..d4c241e69 --- /dev/null +++ b/core/models/models/dependency_setting_v2.go @@ -0,0 +1,17 @@ +package models + +import ( + "time" +) + +type DependencySettingV2 struct { + any `collection:"dependency_settings"` + BaseModelV2[DependencySetting] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + Cmd string `json:"cmd" bson:"cmd"` + Proxy string `json:"proxy" bson:"proxy"` + LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"` +} diff --git a/core/models/models/environment.go b/core/models/models/environment.go new file mode 100644 index 000000000..1598f1305 --- /dev/null +++ b/core/models/models/environment.go @@ -0,0 +1,46 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Environment struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Key string `json:"key" bson:"key"` + Value string `json:"value" bson:"value"` +} + +func (e *Environment) GetId() (id primitive.ObjectID) { + return e.Id +} + +func (e *Environment) SetId(id primitive.ObjectID) { + e.Id = id +} + +func (e *Environment) GetKey() (key string) { + return e.Key +} + +func (e *Environment) SetKey(key string) { + e.Key = key +} + +func (e *Environment) GetValue() (value string) { + return e.Value +} + +func (e *Environment) SetValue(value string) { + e.Value = value +} + +type EnvironmentList []Environment + +func (l *EnvironmentList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/environment_v2.go b/core/models/models/environment_v2.go new file mode 100644 index 000000000..776a11422 --- /dev/null +++ b/core/models/models/environment_v2.go @@ -0,0 +1,8 @@ +package models + +type EnvironmentV2 struct { + any `collection:"environments"` + BaseModelV2[EnvironmentV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Value string `json:"value" bson:"value"` +} diff --git a/core/models/models/extra_value.go b/core/models/models/extra_value.go new file mode 100644 index 000000000..4d65bf5fd --- /dev/null +++ b/core/models/models/extra_value.go @@ -0,0 +1,64 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type ExtraValue struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + ObjectId primitive.ObjectID `json:"oid" bson:"oid"` + Model string `json:"model" bson:"m"` + Type string `json:"type" bson:"t"` + Value interface{} `json:"value" bson:"v"` +} + +func (ev *ExtraValue) GetId() (id primitive.ObjectID) { + return ev.Id +} + +func (ev *ExtraValue) SetId(id primitive.ObjectID) { + ev.Id = id +} + +func (ev *ExtraValue) GetValue() (v interface{}) { + return ev.Value +} + +func (ev *ExtraValue) SetValue(v interface{}) { + ev.Value = v +} + +func (ev *ExtraValue) GetObjectId() (oid primitive.ObjectID) { + return ev.ObjectId +} + +func (ev *ExtraValue) SetObjectId(oid primitive.ObjectID) { + ev.ObjectId = oid +} + +func (ev *ExtraValue) GetModel() (m string) { + return ev.Model +} + +func (ev *ExtraValue) SetModel(m string) { + ev.Model = m +} + +func (ev *ExtraValue) GetType() (t string) { + return ev.Type +} + +func (ev *ExtraValue) SetType(t string) { + ev.Type = t +} + +type ExtraValueList []ExtraValue + +func (l *ExtraValueList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/git.go b/core/models/models/git.go new file mode 100644 index 000000000..1a9a0a587 --- /dev/null +++ b/core/models/models/git.go @@ -0,0 +1,82 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Git struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Url string `json:"url" bson:"url"` + AuthType string `json:"auth_type" bson:"auth_type"` + Username string `json:"username" bson:"username"` + Password string `json:"password" bson:"password"` + CurrentBranch string `json:"current_branch" bson:"current_branch"` + AutoPull bool `json:"auto_pull" bson:"auto_pull"` +} + +func (g *Git) GetId() (id primitive.ObjectID) { + return g.Id +} + +func (g *Git) SetId(id primitive.ObjectID) { + g.Id = id +} + +func (g *Git) GetUrl() (url string) { + return g.Url +} + +func (g *Git) SetUrl(url string) { + g.Url = url +} + +func (g *Git) GetAuthType() (authType string) { + return g.AuthType +} + +func (g *Git) SetAuthType(authType string) { + g.AuthType = authType +} + +func (g *Git) GetUsername() (username string) { + return g.Username +} + +func (g *Git) SetUsername(username string) { + g.Username = username +} + +func (g *Git) GetPassword() (password string) { + return g.Password +} + +func (g *Git) SetPassword(password string) { + g.Password = password +} + +func (g *Git) GetCurrentBranch() (currentBranch string) { + return g.CurrentBranch +} + +func (g *Git) SetCurrentBranch(currentBranch string) { + g.CurrentBranch = currentBranch +} + +func (g *Git) GetAutoPull() (autoPull bool) { + return g.AutoPull +} + +func (g *Git) SetAutoPull(autoPull bool) { + g.AutoPull = autoPull +} + +type GitList []Git + +func (l *GitList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/git_v2.go b/core/models/models/git_v2.go new file mode 100644 index 000000000..b72c130a9 --- /dev/null +++ b/core/models/models/git_v2.go @@ -0,0 +1,12 @@ +package models + +type GitV2 struct { + any `collection:"gits"` + BaseModelV2[GitV2] `bson:",inline"` + Url string `json:"url" bson:"url"` + AuthType string `json:"auth_type" bson:"auth_type"` + Username string `json:"username" bson:"username"` + Password string `json:"password" bson:"password"` + CurrentBranch string `json:"current_branch" bson:"current_branch"` + AutoPull bool `json:"auto_pull" bson:"auto_pull"` +} diff --git a/core/models/models/job.go b/core/models/models/job.go new file mode 100644 index 000000000..66c17720a --- /dev/null +++ b/core/models/models/job.go @@ -0,0 +1,29 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Job struct { + Id primitive.ObjectID `bson:"_id" json:"_id"` + TaskId primitive.ObjectID `bson:"task_id" json:"task_id"` +} + +func (j *Job) GetId() (id primitive.ObjectID) { + return j.Id +} + +func (j *Job) SetId(id primitive.ObjectID) { + j.Id = id +} + +type JobList []Job + +func (l *JobList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/node.go b/core/models/models/node.go new file mode 100644 index 000000000..c9698106c --- /dev/null +++ b/core/models/models/node.go @@ -0,0 +1,119 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type Node struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Ip string `json:"ip" bson:"ip"` + Port string `json:"port" bson:"port"` + Mac string `json:"mac" bson:"mac"` + Hostname string `json:"hostname" bson:"hostname"` + Description string `json:"description" bson:"description"` + IsMaster bool `json:"is_master" bson:"is_master"` + Status string `json:"status" bson:"status"` + Enabled bool `json:"enabled" bson:"enabled"` + Active bool `json:"active" bson:"active"` + ActiveTs time.Time `json:"active_ts" bson:"active_ts"` + AvailableRunners int `json:"available_runners" bson:"available_runners"` + MaxRunners int `json:"max_runners" bson:"max_runners"` +} + +func (n *Node) GetId() (id primitive.ObjectID) { + return n.Id +} + +func (n *Node) SetId(id primitive.ObjectID) { + n.Id = id +} + +func (n *Node) GetName() (name string) { + return n.Name +} + +func (n *Node) SetName(name string) { + n.Name = name +} + +func (n *Node) GetDescription() (description string) { + return n.Description +} + +func (n *Node) SetDescription(description string) { + n.Description = description +} + +func (n *Node) GetKey() (key string) { + return n.Key +} + +func (n *Node) GetIsMaster() (ok bool) { + return n.IsMaster +} + +func (n *Node) GetActive() (active bool) { + return n.Active +} + +func (n *Node) SetActive(active bool) { + n.Active = active +} + +func (n *Node) SetActiveTs(activeTs time.Time) { + n.ActiveTs = activeTs +} + +func (n *Node) GetStatus() (status string) { + return n.Status +} + +func (n *Node) SetStatus(status string) { + n.Status = status +} + +func (n *Node) GetEnabled() (enabled bool) { + return n.Enabled +} + +func (n *Node) SetEnabled(enabled bool) { + n.Enabled = enabled +} + +func (n *Node) GetAvailableRunners() (runners int) { + return n.AvailableRunners +} + +func (n *Node) SetAvailableRunners(runners int) { + n.AvailableRunners = runners +} + +func (n *Node) GetMaxRunners() (runners int) { + return n.MaxRunners +} + +func (n *Node) SetMaxRunners(runners int) { + n.MaxRunners = runners +} + +func (n *Node) IncrementAvailableRunners() { + n.AvailableRunners++ +} + +func (n *Node) DecrementAvailableRunners() { + n.AvailableRunners-- +} + +type NodeList []Node + +func (l *NodeList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/node_v2.go b/core/models/models/node_v2.go new file mode 100644 index 000000000..6d5a84421 --- /dev/null +++ b/core/models/models/node_v2.go @@ -0,0 +1,24 @@ +package models + +import ( + "time" +) + +type NodeV2 struct { + any `collection:"nodes"` + BaseModelV2[NodeV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Ip string `json:"ip" bson:"ip"` + Port string `json:"port" bson:"port"` + Mac string `json:"mac" bson:"mac"` + Hostname string `json:"hostname" bson:"hostname"` + Description string `json:"description" bson:"description"` + IsMaster bool `json:"is_master" bson:"is_master"` + Status string `json:"status" bson:"status"` + Enabled bool `json:"enabled" bson:"enabled"` + Active bool `json:"active" bson:"active"` + ActiveAt time.Time `json:"active_at" bson:"active_ts"` + AvailableRunners int `json:"available_runners" bson:"available_runners"` + MaxRunners int `json:"max_runners" bson:"max_runners"` +} diff --git a/core/models/models/notification_setting_v2.go b/core/models/models/notification_setting_v2.go new file mode 100644 index 000000000..6b5654109 --- /dev/null +++ b/core/models/models/notification_setting_v2.go @@ -0,0 +1,32 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type NotificationSettingV2 struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Type string `json:"type" bson:"type"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + Global bool `json:"global" bson:"global"` + Title string `json:"title,omitempty" bson:"title,omitempty"` + Template string `json:"template,omitempty" bson:"template,omitempty"` + TaskTrigger string `json:"task_trigger" bson:"task_trigger"` + Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` + Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` +} + +type NotificationSettingMail struct { + Server string `json:"server" bson:"server"` + Port string `json:"port,omitempty" bson:"port,omitempty"` + User string `json:"user,omitempty" bson:"user,omitempty"` + Password string `json:"password,omitempty" bson:"password,omitempty"` + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` + SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"` + To string `json:"to,omitempty" bson:"to,omitempty"` + Cc string `json:"cc,omitempty" bson:"cc,omitempty"` +} + +type NotificationSettingMobile struct { + Webhook string `json:"webhook" bson:"webhook"` +} diff --git a/core/models/models/password.go b/core/models/models/password.go new file mode 100644 index 000000000..4be613ef1 --- /dev/null +++ b/core/models/models/password.go @@ -0,0 +1,29 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Password struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Password string `json:"password" bson:"p"` +} + +func (p *Password) GetId() (id primitive.ObjectID) { + return p.Id +} + +func (p *Password) SetId(id primitive.ObjectID) { + p.Id = id +} + +type PasswordList []Password + +func (l *PasswordList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/permission.go b/core/models/models/permission.go new file mode 100644 index 000000000..51c42d370 --- /dev/null +++ b/core/models/models/permission.go @@ -0,0 +1,91 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Permission struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Type string `json:"type" bson:"type"` + Target []string `json:"target" bson:"target"` + Allow []string `json:"allow" bson:"allow"` + Deny []string `json:"deny" bson:"deny"` +} + +func (p *Permission) GetId() (id primitive.ObjectID) { + return p.Id +} + +func (p *Permission) SetId(id primitive.ObjectID) { + p.Id = id +} + +func (p *Permission) GetKey() (key string) { + return p.Key +} + +func (p *Permission) SetKey(key string) { + p.Key = key +} + +func (p *Permission) GetName() (name string) { + return p.Name +} + +func (p *Permission) SetName(name string) { + p.Name = name +} + +func (p *Permission) GetDescription() (description string) { + return p.Description +} + +func (p *Permission) SetDescription(description string) { + p.Description = description +} + +func (p *Permission) GetType() (t string) { + return p.Type +} + +func (p *Permission) SetType(t string) { + p.Type = t +} + +func (p *Permission) GetTarget() (target []string) { + return p.Target +} + +func (p *Permission) SetTarget(target []string) { + p.Target = target +} + +func (p *Permission) GetAllow() (include []string) { + return p.Allow +} + +func (p *Permission) SetAllow(include []string) { + p.Allow = include +} + +func (p *Permission) GetDeny() (exclude []string) { + return p.Deny +} + +func (p *Permission) SetDeny(exclude []string) { + p.Deny = exclude +} + +type PermissionList []Permission + +func (l *PermissionList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/permission_v2.go b/core/models/models/permission_v2.go new file mode 100644 index 000000000..77c830a38 --- /dev/null +++ b/core/models/models/permission_v2.go @@ -0,0 +1,13 @@ +package models + +type PermissionV2 struct { + any `collection:"permissions"` + BaseModelV2[PermissionV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Type string `json:"type" bson:"type"` + Target []string `json:"target" bson:"target"` + Allow []string `json:"allow" bson:"allow"` + Deny []string `json:"deny" bson:"deny"` +} diff --git a/core/models/models/project.go b/core/models/models/project.go new file mode 100644 index 000000000..59b01212f --- /dev/null +++ b/core/models/models/project.go @@ -0,0 +1,47 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Project struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Spiders int `json:"spiders" bson:"-"` +} + +func (p *Project) GetId() (id primitive.ObjectID) { + return p.Id +} + +func (p *Project) SetId(id primitive.ObjectID) { + p.Id = id +} + +func (p *Project) GetName() (name string) { + return p.Name +} + +func (p *Project) SetName(name string) { + p.Name = name +} + +func (p *Project) GetDescription() (description string) { + return p.Description +} + +func (p *Project) SetDescription(description string) { + p.Description = description +} + +type ProjectList []Project + +func (l *ProjectList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/project_v2.go b/core/models/models/project_v2.go new file mode 100644 index 000000000..ee07e705a --- /dev/null +++ b/core/models/models/project_v2.go @@ -0,0 +1,9 @@ +package models + +type ProjectV2 struct { + any `collection:"projects"` + BaseModelV2[ProjectV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Spiders int `json:"spiders" bson:"-"` +} diff --git a/core/models/models/result.go b/core/models/models/result.go new file mode 100644 index 000000000..9c2957b04 --- /dev/null +++ b/core/models/models/result.go @@ -0,0 +1,60 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Result bson.M + +func (r *Result) GetId() (id primitive.ObjectID) { + res, ok := r.Value()["_id"] + if ok { + id, ok = res.(primitive.ObjectID) + if ok { + return id + } + } + return id +} + +func (r *Result) SetId(id primitive.ObjectID) { + (*r)["_id"] = id +} + +func (r *Result) Value() map[string]interface{} { + return *r +} + +func (r *Result) SetValue(key string, value interface{}) { + (*r)[key] = value +} + +func (r *Result) GetValue(key string) (value interface{}) { + return (*r)[key] +} + +func (r *Result) GetTaskId() (id primitive.ObjectID) { + res := r.GetValue(constants.TaskKey) + if res == nil { + return id + } + id, _ = res.(primitive.ObjectID) + return id +} + +func (r *Result) SetTaskId(id primitive.ObjectID) { + r.SetValue(constants.TaskKey, id) +} + +type ResultList []Result + +func (l *ResultList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/role.go b/core/models/models/role.go new file mode 100644 index 000000000..3a73d8f67 --- /dev/null +++ b/core/models/models/role.go @@ -0,0 +1,55 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Role struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` +} + +func (r *Role) GetId() (id primitive.ObjectID) { + return r.Id +} + +func (r *Role) SetId(id primitive.ObjectID) { + r.Id = id +} + +func (r *Role) GetKey() (key string) { + return r.Key +} + +func (r *Role) SetKey(key string) { + r.Key = key +} + +func (r *Role) GetName() (name string) { + return r.Name +} + +func (r *Role) SetName(name string) { + r.Name = name +} + +func (r *Role) GetDescription() (description string) { + return r.Description +} + +func (r *Role) SetDescription(description string) { + r.Description = description +} + +type RoleList []Role + +func (l *RoleList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/role_permission.go b/core/models/models/role_permission.go new file mode 100644 index 000000000..d603c1437 --- /dev/null +++ b/core/models/models/role_permission.go @@ -0,0 +1,30 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type RolePermission struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"` +} + +func (ur *RolePermission) GetId() (id primitive.ObjectID) { + return ur.Id +} + +func (ur *RolePermission) SetId(id primitive.ObjectID) { + ur.Id = id +} + +type RolePermissionList []RolePermission + +func (l *RolePermissionList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/role_permission_v2.go b/core/models/models/role_permission_v2.go new file mode 100644 index 000000000..386fa0529 --- /dev/null +++ b/core/models/models/role_permission_v2.go @@ -0,0 +1,12 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type RolePermissionV2 struct { + any `collection:"role_permissions"` + BaseModelV2[RolePermissionV2] `bson:",inline"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"` +} diff --git a/core/models/models/role_v2.go b/core/models/models/role_v2.go new file mode 100644 index 000000000..de287a61f --- /dev/null +++ b/core/models/models/role_v2.go @@ -0,0 +1,9 @@ +package models + +type RoleV2 struct { + any `collection:"roles"` + BaseModelV2[RoleV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` +} diff --git a/core/models/models/schedule.go b/core/models/models/schedule.go new file mode 100644 index 000000000..96c455b35 --- /dev/null +++ b/core/models/models/schedule.go @@ -0,0 +1,113 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/robfig/cron/v3" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Schedule struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Cron string `json:"cron" bson:"cron"` + EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Mode string `json:"mode" bson:"mode"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` + Priority int `json:"priority" bson:"priority"` + Enabled bool `json:"enabled" bson:"enabled"` + UserId primitive.ObjectID `json:"user_id" bson:"user_id"` +} + +func (s *Schedule) GetId() (id primitive.ObjectID) { + return s.Id +} + +func (s *Schedule) SetId(id primitive.ObjectID) { + s.Id = id +} + +func (s *Schedule) GetEnabled() (enabled bool) { + return s.Enabled +} + +func (s *Schedule) SetEnabled(enabled bool) { + s.Enabled = enabled +} + +func (s *Schedule) GetEntryId() (id cron.EntryID) { + return s.EntryId +} + +func (s *Schedule) SetEntryId(id cron.EntryID) { + s.EntryId = id +} + +func (s *Schedule) GetCron() (c string) { + return s.Cron +} + +func (s *Schedule) SetCron(c string) { + s.Cron = c +} + +func (s *Schedule) GetSpiderId() (id primitive.ObjectID) { + return s.SpiderId +} + +func (s *Schedule) SetSpiderId(id primitive.ObjectID) { + s.SpiderId = id +} + +func (s *Schedule) GetMode() (mode string) { + return s.Mode +} + +func (s *Schedule) SetMode(mode string) { + s.Mode = mode +} + +func (s *Schedule) GetNodeIds() (ids []primitive.ObjectID) { + return s.NodeIds +} + +func (s *Schedule) SetNodeIds(ids []primitive.ObjectID) { + s.NodeIds = ids +} + +func (s *Schedule) GetCmd() (cmd string) { + return s.Cmd +} + +func (s *Schedule) SetCmd(cmd string) { + s.Cmd = cmd +} + +func (s *Schedule) GetParam() (param string) { + return s.Param +} + +func (s *Schedule) SetParam(param string) { + s.Param = param +} + +func (s *Schedule) GetPriority() (p int) { + return s.Priority +} + +func (s *Schedule) SetPriority(p int) { + s.Priority = p +} + +type ScheduleList []Schedule + +func (l *ScheduleList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/schedule_v2.go b/core/models/models/schedule_v2.go new file mode 100644 index 000000000..a52f1b920 --- /dev/null +++ b/core/models/models/schedule_v2.go @@ -0,0 +1,23 @@ +package models + +import ( + "github.com/robfig/cron/v3" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type ScheduleV2 struct { + any `collection:"schedules"` + BaseModelV2[ScheduleV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Cron string `json:"cron" bson:"cron"` + EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Mode string `json:"mode" bson:"mode"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` + Priority int `json:"priority" bson:"priority"` + Enabled bool `json:"enabled" bson:"enabled"` + UserId primitive.ObjectID `json:"user_id" bson:"user_id"` +} diff --git a/core/models/models/setting.go b/core/models/models/setting.go new file mode 100644 index 000000000..b0fc9d88f --- /dev/null +++ b/core/models/models/setting.go @@ -0,0 +1,31 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Setting struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Key string `json:"key" bson:"key"` + Value bson.M `json:"value" bson:"value"` +} + +func (s *Setting) GetId() (id primitive.ObjectID) { + return s.Id +} + +func (s *Setting) SetId(id primitive.ObjectID) { + s.Id = id +} + +type SettingList []Setting + +func (l *SettingList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/setting_v2.go b/core/models/models/setting_v2.go new file mode 100644 index 000000000..86481607e --- /dev/null +++ b/core/models/models/setting_v2.go @@ -0,0 +1,12 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson" +) + +type SettingV2 struct { + any `collection:"settings"` + BaseModelV2[SettingV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Value bson.M `json:"value" bson:"value"` +} diff --git a/core/models/models/spider.go b/core/models/models/spider.go new file mode 100644 index 000000000..49b89418f --- /dev/null +++ b/core/models/models/spider.go @@ -0,0 +1,137 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Env struct { + Name string `json:"name" bson:"name"` + Value string `json:"value" bson:"value"` +} + +type Spider struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` // spider id + Name string `json:"name" bson:"name"` // spider name + Type string `json:"type" bson:"type"` // spider type + ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id + ColName string `json:"col_name,omitempty" bson:"-"` // data collection name + DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id + DataSource *DataSource `json:"data_source,omitempty" bson:"-"` // data source + Description string `json:"description" bson:"description"` // description + ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id + Mode string `json:"mode" bson:"mode"` // default Task.Mode + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds + Stat *SpiderStat `json:"stat,omitempty" bson:"-"` + + // execution + Cmd string `json:"cmd" bson:"cmd"` // execute command + Param string `json:"param" bson:"param"` // default task param + Priority int `json:"priority" bson:"priority"` + AutoInstall bool `json:"auto_install" bson:"auto_install"` + + // settings + IncrementalSync bool `json:"incremental_sync" bson:"incremental_sync"` // whether to incrementally sync files +} + +func (s *Spider) GetId() (id primitive.ObjectID) { + return s.Id +} + +func (s *Spider) SetId(id primitive.ObjectID) { + s.Id = id +} + +func (s *Spider) GetName() (name string) { + return s.Name +} + +func (s *Spider) SetName(name string) { + s.Name = name +} + +func (s *Spider) GetDescription() (description string) { + return s.Description +} + +func (s *Spider) SetDescription(description string) { + s.Description = description +} + +func (s *Spider) GetType() (ty string) { + return s.Type +} + +func (s *Spider) GetMode() (mode string) { + return s.Mode +} + +func (s *Spider) SetMode(mode string) { + s.Mode = mode +} + +func (s *Spider) GetNodeIds() (ids []primitive.ObjectID) { + return s.NodeIds +} + +func (s *Spider) SetNodeIds(ids []primitive.ObjectID) { + s.NodeIds = ids +} + +func (s *Spider) GetCmd() (cmd string) { + return s.Cmd +} + +func (s *Spider) SetCmd(cmd string) { + s.Cmd = cmd +} + +func (s *Spider) GetParam() (param string) { + return s.Param +} + +func (s *Spider) SetParam(param string) { + s.Param = param +} + +func (s *Spider) GetPriority() (p int) { + return s.Priority +} + +func (s *Spider) SetPriority(p int) { + s.Priority = p +} + +func (s *Spider) GetColId() (id primitive.ObjectID) { + return s.ColId +} + +func (s *Spider) SetColId(id primitive.ObjectID) { + s.ColId = id +} + +func (s *Spider) GetIncrementalSync() (incrementalSync bool) { + return s.IncrementalSync +} + +func (s *Spider) SetIncrementalSync(incrementalSync bool) { + s.IncrementalSync = incrementalSync +} + +func (s *Spider) GetAutoInstall() (autoInstall bool) { + return s.AutoInstall +} + +func (s *Spider) SetAutoInstall(autoInstall bool) { + s.AutoInstall = autoInstall +} + +type SpiderList []Spider + +func (l *SpiderList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/spider_stat.go b/core/models/models/spider_stat.go new file mode 100644 index 000000000..09047b90f --- /dev/null +++ b/core/models/models/spider_stat.go @@ -0,0 +1,38 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type SpiderStat struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"` + LastTask *Task `json:"last_task,omitempty" bson:"-"` + Tasks int `json:"tasks" bson:"tasks"` + Results int `json:"results" bson:"results"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second + AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second + AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second + AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second +} + +func (s *SpiderStat) GetId() (id primitive.ObjectID) { + return s.Id +} + +func (s *SpiderStat) SetId(id primitive.ObjectID) { + s.Id = id +} + +type SpiderStatList []SpiderStat + +func (l *SpiderStatList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/spider_stat_v2.go b/core/models/models/spider_stat_v2.go new file mode 100644 index 000000000..ea01c0cc6 --- /dev/null +++ b/core/models/models/spider_stat_v2.go @@ -0,0 +1,20 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type SpiderStatV2 struct { + any `collection:"spider_stats"` + BaseModelV2[SpiderStatV2] `bson:",inline"` + LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"` + LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"` + Tasks int `json:"tasks" bson:"tasks"` + Results int `json:"results" bson:"results"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second + AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second + AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second + AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second +} diff --git a/core/models/models/spider_v2.go b/core/models/models/spider_v2.go new file mode 100644 index 000000000..4f98f6e79 --- /dev/null +++ b/core/models/models/spider_v2.go @@ -0,0 +1,30 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type SpiderV2 struct { + any `collection:"spiders"` // spider id + BaseModelV2[SpiderV2] `bson:",inline"` + Name string `json:"name" bson:"name"` // spider name + Type string `json:"type" bson:"type"` // spider type + ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id + ColName string `json:"col_name,omitempty" bson:"-"` // data collection name + DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id + DataSource *DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source + Description string `json:"description" bson:"description"` // description + ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id + Mode string `json:"mode" bson:"mode"` // default Task.Mode + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds + Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"` + + // execution + Cmd string `json:"cmd" bson:"cmd"` // execute command + Param string `json:"param" bson:"param"` // default task param + Priority int `json:"priority" bson:"priority"` + AutoInstall bool `json:"auto_install" bson:"auto_install"` + + // settings + IncrementalSync bool `json:"incremental_sync" bson:"incremental_sync"` // whether to incrementally sync files +} diff --git a/core/models/models/tag.go b/core/models/models/tag.go new file mode 100644 index 000000000..86708240c --- /dev/null +++ b/core/models/models/tag.go @@ -0,0 +1,44 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Tag struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Name string `json:"name" bson:"name"` + Color string `json:"color" bson:"color"` + Description string `json:"description" bson:"description"` + Col string `json:"col" bson:"col"` +} + +func (t *Tag) GetId() (id primitive.ObjectID) { + return t.Id +} + +func (t *Tag) SetId(id primitive.ObjectID) { + t.Id = id +} + +func (t *Tag) GetName() (res string) { + return t.Name +} + +func (t *Tag) GetColor() (res string) { + return t.Color +} + +func (t *Tag) SetCol(col string) { + t.Col = col +} + +type TagList []Tag + +func (l *TagList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/task.go b/core/models/models/task.go new file mode 100644 index 000000000..6dcc926af --- /dev/null +++ b/core/models/models/task.go @@ -0,0 +1,118 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type Task struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Status string `json:"status" bson:"status"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Error string `json:"error" bson:"error"` + Pid int `json:"pid" bson:"pid"` + ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` // Schedule.Id + Type string `json:"type" bson:"type"` + Mode string `json:"mode" bson:"mode"` // running mode of Task + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // list of Node.Id + ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"` // parent Task.Id if it'Spider a sub-task + Priority int `json:"priority" bson:"priority"` + Stat *TaskStat `json:"stat,omitempty" bson:"-"` + HasSub bool `json:"has_sub" json:"has_sub"` // whether to have sub-tasks + SubTasks []Task `json:"sub_tasks,omitempty" bson:"-"` + Spider *Spider `json:"spider,omitempty" bson:"-"` + UserId primitive.ObjectID `json:"-" bson:"-"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` +} + +func (t *Task) GetId() (id primitive.ObjectID) { + return t.Id +} + +func (t *Task) SetId(id primitive.ObjectID) { + t.Id = id +} + +func (t *Task) GetNodeId() (id primitive.ObjectID) { + return t.NodeId +} + +func (t *Task) SetNodeId(id primitive.ObjectID) { + t.NodeId = id +} + +func (t *Task) GetNodeIds() (ids []primitive.ObjectID) { + return t.NodeIds +} + +func (t *Task) GetStatus() (status string) { + return t.Status +} + +func (t *Task) SetStatus(status string) { + t.Status = status +} + +func (t *Task) GetError() (error string) { + return t.Error +} + +func (t *Task) SetError(error string) { + t.Error = error +} + +func (t *Task) GetPid() (pid int) { + return t.Pid +} + +func (t *Task) SetPid(pid int) { + t.Pid = pid +} + +func (t *Task) GetSpiderId() (id primitive.ObjectID) { + return t.SpiderId +} + +func (t *Task) GetType() (ty string) { + return t.Type +} + +func (t *Task) GetCmd() (cmd string) { + return t.Cmd +} + +func (t *Task) GetParam() (param string) { + return t.Param +} + +func (t *Task) GetPriority() (p int) { + return t.Priority +} + +func (t *Task) GetUserId() (id primitive.ObjectID) { + return t.UserId +} + +func (t *Task) SetUserId(id primitive.ObjectID) { + t.UserId = id +} + +type TaskList []Task + +func (l *TaskList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} + +type TaskDailyItem struct { + Date string `json:"date" bson:"_id"` + TaskCount int `json:"task_count" bson:"task_count"` + AvgRuntimeDuration float64 `json:"avg_runtime_duration" bson:"avg_runtime_duration"` +} diff --git a/core/models/models/task_queue_item.go b/core/models/models/task_queue_item.go new file mode 100644 index 000000000..f2f08d49c --- /dev/null +++ b/core/models/models/task_queue_item.go @@ -0,0 +1,30 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type TaskQueueItem struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Priority int `json:"p" bson:"p"` + NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"` +} + +func (t *TaskQueueItem) GetId() (id primitive.ObjectID) { + return t.Id +} + +func (t *TaskQueueItem) SetId(id primitive.ObjectID) { + t.Id = id +} + +type TaskQueueItemList []TaskQueueItem + +func (l *TaskQueueItemList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/task_queue_item_v2.go b/core/models/models/task_queue_item_v2.go new file mode 100644 index 000000000..f222aef3e --- /dev/null +++ b/core/models/models/task_queue_item_v2.go @@ -0,0 +1,12 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type TaskQueueItemV2 struct { + any `collection:"task_queue"` + BaseModelV2[TaskQueueItemV2] `bson:",inline"` + Priority int `json:"p" bson:"p"` + NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"` +} diff --git a/core/models/models/task_stat.go b/core/models/models/task_stat.go new file mode 100644 index 000000000..d8f509058 --- /dev/null +++ b/core/models/models/task_stat.go @@ -0,0 +1,101 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type TaskStat struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"` + StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` + EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond + ResultCount int64 `json:"result_count" bson:"result_count"` + ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"` +} + +func (s *TaskStat) GetId() (id primitive.ObjectID) { + return s.Id +} + +func (s *TaskStat) SetId(id primitive.ObjectID) { + s.Id = id +} + +func (s *TaskStat) GetCreateTs() (ts time.Time) { + return s.CreateTs +} + +func (s *TaskStat) SetCreateTs(ts time.Time) { + s.CreateTs = ts +} + +func (s *TaskStat) GetStartTs() (ts time.Time) { + return s.StartTs +} + +func (s *TaskStat) SetStartTs(ts time.Time) { + s.StartTs = ts +} + +func (s *TaskStat) GetEndTs() (ts time.Time) { + return s.EndTs +} + +func (s *TaskStat) SetEndTs(ts time.Time) { + s.EndTs = ts +} + +func (s *TaskStat) GetWaitDuration() (d int64) { + return s.WaitDuration +} + +func (s *TaskStat) SetWaitDuration(d int64) { + s.WaitDuration = d +} + +func (s *TaskStat) GetRuntimeDuration() (d int64) { + return s.RuntimeDuration +} + +func (s *TaskStat) SetRuntimeDuration(d int64) { + s.RuntimeDuration = d +} + +func (s *TaskStat) GetTotalDuration() (d int64) { + return s.WaitDuration + s.RuntimeDuration +} + +func (s *TaskStat) SetTotalDuration(d int64) { + s.TotalDuration = d +} + +func (s *TaskStat) GetResultCount() (c int64) { + return s.ResultCount +} + +func (s *TaskStat) SetResultCount(c int64) { + s.ResultCount = c +} + +func (s *TaskStat) GetErrorLogCount() (c int64) { + return s.ErrorLogCount +} + +func (s *TaskStat) SetErrorLogCount(c int64) { + s.ErrorLogCount = c +} + +type TaskStatList []TaskStat + +func (l *TaskStatList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/task_stat_v2.go b/core/models/models/task_stat_v2.go new file mode 100644 index 000000000..5456946d4 --- /dev/null +++ b/core/models/models/task_stat_v2.go @@ -0,0 +1,18 @@ +package models + +import ( + "time" +) + +type TaskStatV2 struct { + any `collection:"task_stats"` + BaseModelV2[TaskStatV2] `bson:",inline"` + CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"` + StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` + EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond + ResultCount int64 `json:"result_count" bson:"result_count"` + ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"` +} diff --git a/core/models/models/task_v2.go b/core/models/models/task_v2.go new file mode 100644 index 000000000..3473b6dba --- /dev/null +++ b/core/models/models/task_v2.go @@ -0,0 +1,30 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type TaskV2 struct { + any `collection:"tasks"` + BaseModelV2[TaskV2] `bson:",inline"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Status string `json:"status" bson:"status"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Error string `json:"error" bson:"error"` + Pid int `json:"pid" bson:"pid"` + ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` + Type string `json:"type" bson:"type"` + Mode string `json:"mode" bson:"mode"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` + ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"` + Priority int `json:"priority" bson:"priority"` + Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"` + HasSub bool `json:"has_sub" json:"has_sub"` + SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"` + Spider *SpiderV2 `json:"spider,omitempty" bson:"-"` + UserId primitive.ObjectID `json:"-" bson:"-"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` +} diff --git a/core/models/models/test.go b/core/models/models/test.go new file mode 100644 index 000000000..67027d19c --- /dev/null +++ b/core/models/models/test.go @@ -0,0 +1,7 @@ +package models + +type TestModel struct { + any `collection:"testmodels"` + BaseModelV2[TestModel] `bson:",inline"` + Name string `json:"name" bson:"name"` +} diff --git a/core/models/models/token.go b/core/models/models/token.go new file mode 100644 index 000000000..7a957e3e9 --- /dev/null +++ b/core/models/models/token.go @@ -0,0 +1,30 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Token struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Name string `json:"name" bson:"name"` + Token string `json:"token" bson:"token"` +} + +func (t *Token) GetId() (id primitive.ObjectID) { + return t.Id +} + +func (t *Token) SetId(id primitive.ObjectID) { + t.Id = id +} + +type TokenList []Token + +func (l *TokenList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/token_v2.go b/core/models/models/token_v2.go new file mode 100644 index 000000000..c792c9ee4 --- /dev/null +++ b/core/models/models/token_v2.go @@ -0,0 +1,8 @@ +package models + +type TokenV2 struct { + any `collection:"tokens"` + BaseModelV2[TokenV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Token string `json:"token" bson:"token"` +} diff --git a/core/models/models/user.go b/core/models/models/user.go new file mode 100644 index 000000000..c03bf388c --- /dev/null +++ b/core/models/models/user.go @@ -0,0 +1,59 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type User struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Username string `json:"username" bson:"username"` + Password string `json:"password,omitempty" bson:"-"` + Role string `json:"role" bson:"role"` + Email string `json:"email" bson:"email"` + //Setting UserSetting `json:"setting" bson:"setting"` +} + +func (u *User) GetId() (id primitive.ObjectID) { + return u.Id +} + +func (u *User) SetId(id primitive.ObjectID) { + u.Id = id +} + +func (u *User) GetUsername() (name string) { + return u.Username +} + +func (u *User) GetPassword() (p string) { + return u.Password +} + +func (u *User) GetRole() (r string) { + return u.Role +} + +func (u *User) GetEmail() (email string) { + return u.Email +} + +//type UserSetting struct { +// NotificationTrigger string `json:"notification_trigger" bson:"notification_trigger"` +// DingTalkRobotWebhook string `json:"ding_talk_robot_webhook" bson:"ding_talk_robot_webhook"` +// WechatRobotWebhook string `json:"wechat_robot_webhook" bson:"wechat_robot_webhook"` +// EnabledNotifications []string `json:"enabled_notifications" bson:"enabled_notifications"` +// ErrorRegexPattern string `json:"error_regex_pattern" bson:"error_regex_pattern"` +// MaxErrorLog int `json:"max_error_log" bson:"max_error_log"` +// LogExpireDuration int64 `json:"log_expire_duration" bson:"log_expire_duration"` +//} + +type UserList []User + +func (l *UserList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/user_role.go b/core/models/models/user_role.go new file mode 100644 index 000000000..a706bec62 --- /dev/null +++ b/core/models/models/user_role.go @@ -0,0 +1,30 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type UserRole struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + UserId primitive.ObjectID `json:"user_id" bson:"user_id"` +} + +func (ur *UserRole) GetId() (id primitive.ObjectID) { + return ur.Id +} + +func (ur *UserRole) SetId(id primitive.ObjectID) { + ur.Id = id +} + +type UserRoleList []UserRole + +func (l *UserRoleList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/user_role_v2.go b/core/models/models/user_role_v2.go new file mode 100644 index 000000000..aa0ac34a0 --- /dev/null +++ b/core/models/models/user_role_v2.go @@ -0,0 +1,12 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type UserRoleV2 struct { + any `collection:"user_roles"` + BaseModelV2[UserRoleV2] `bson:",inline"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + UserId primitive.ObjectID `json:"user_id" bson:"user_id"` +} diff --git a/core/models/models/user_v2.go b/core/models/models/user_v2.go new file mode 100644 index 000000000..acfbe2b64 --- /dev/null +++ b/core/models/models/user_v2.go @@ -0,0 +1,10 @@ +package models + +type UserV2 struct { + any `collection:"users"` + BaseModelV2[UserV2] `bson:",inline"` + Username string `json:"username" bson:"username"` + Password string `json:"-,omitempty" bson:"password"` + Role string `json:"role" bson:"role"` + Email string `json:"email" bson:"email"` +} diff --git a/core/models/models/utils_binder_legacy.go b/core/models/models/utils_binder_legacy.go new file mode 100644 index 000000000..0c46d791b --- /dev/null +++ b/core/models/models/utils_binder_legacy.go @@ -0,0 +1,96 @@ +package models + +//func AssignFields(d interface{}, fieldIds ...interfaces.ModelId) (res interface{}, err error) { +// return assignFields(d, fieldIds...) +//} +// +//func AssignListFields(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) { +// return assignListFields(list, fieldIds...) +//} +// +//func AssignListFieldsAsPtr(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) { +// return assignListFieldsAsPtr(list, fieldIds...) +//} +// +//func assignFields(d interface{}, fieldIds ...interfaces.ModelId) (res interface{}, err error) { +// doc, ok := d.(interfaces.Model) +// if !ok { +// return nil, errors.ErrorModelInvalidType +// } +// if len(fieldIds) == 0 { +// return doc, nil +// } +// for _, fid := range fieldIds { +// switch fid { +// case interfaces.ModelIdTag: +// // convert interface +// d, ok := doc.(interfaces.ModelWithTags) +// if !ok { +// return nil, errors.ErrorModelInvalidType +// } +// +// // attempt to get artifact +// a, err := doc.GetArtifact() +// if err != nil { +// return nil, err +// } +// +// // skip if no artifact found +// if a == nil { +// return d, nil +// } +// +// // assign tags +// tags, err := a.GetTags() +// if err != nil { +// return nil, err +// } +// d.SetTags(tags) +// +// return d, nil +// } +// } +// return doc, nil +//} +// +//func _assignListFields(asPtr bool, list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) { +// vList := reflect.ValueOf(list) +// if vList.Kind() != reflect.Array && +// vList.Kind() != reflect.Slice { +// return res, errors.ErrorModelInvalidType +// } +// for i := 0; i < vList.Len(); i++ { +// vItem := vList.Index(i) +// var item interface{} +// if vItem.CanAddr() { +// item = vItem.Addr().Interface() +// } else { +// item = vItem.Interface() +// } +// doc, ok := item.(interfaces.Model) +// if !ok { +// return res, errors.ErrorModelInvalidType +// } +// ptr, err := assignFields(doc, fieldIds...) +// if err != nil { +// return res, err +// } +// v := reflect.ValueOf(ptr) +// if !asPtr { +// // non-pointer item +// res.Add(v.Elem().Interface()) +// } else { +// // pointer item +// res.Add(v.Interface()) +// } +// } +// return res, nil +//} +// +//func assignListFields(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) { +// return _assignListFields(false, list, fieldIds...) +//} +// +//func assignListFieldsAsPtr(list interface{}, fieldIds ...interfaces.ModelId) (res arraylist.List, err error) { +// return _assignListFields(true, list, fieldIds...) +//} diff --git a/core/models/models/utils_col.go b/core/models/models/utils_col.go new file mode 100644 index 000000000..b59717963 --- /dev/null +++ b/core/models/models/utils_col.go @@ -0,0 +1,10 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/utils/binders" +) + +func GetModelColName(id interfaces.ModelId) (colName string) { + return binders.NewColNameBinder(id).MustBindString() +} diff --git a/core/models/models/utils_model_map.go b/core/models/models/utils_model_map.go new file mode 100644 index 000000000..f03ce11e7 --- /dev/null +++ b/core/models/models/utils_model_map.go @@ -0,0 +1,95 @@ +package models + +type ModelMap struct { + Artifact Artifact + Tag Tag + Node Node + Project Project + Spider Spider + Task Task + Job Job + Schedule Schedule + User User + Setting Setting + Token Token + Variable Variable + TaskQueueItem TaskQueueItem + TaskStat TaskStat + SpiderStat SpiderStat + DataSource DataSource + DataCollection DataCollection + Result Result + Password Password + ExtraValue ExtraValue + Git Git + Role Role + UserRole UserRole + Permission Permission + RolePermission RolePermission + Environment Environment + DependencySetting DependencySetting +} + +type ModelListMap struct { + Artifacts ArtifactList + Tags TagList + Nodes NodeList + Projects ProjectList + Spiders SpiderList + Tasks TaskList + Jobs JobList + Schedules ScheduleList + Users UserList + Settings SettingList + Tokens TokenList + Variables VariableList + TaskQueueItems TaskQueueItemList + TaskStats TaskStatList + SpiderStats SpiderStatList + DataSources DataSourceList + DataCollections DataCollectionList + Results ResultList + Passwords PasswordList + ExtraValues ExtraValueList + Gits GitList + Roles RoleList + UserRoles UserRoleList + PermissionList PermissionList + RolePermissionList RolePermissionList + Environments EnvironmentList + DependencySettings DependencySettingList +} + +func NewModelMap() (m *ModelMap) { + return &ModelMap{} +} + +func NewModelListMap() (m *ModelListMap) { + return &ModelListMap{ + Artifacts: ArtifactList{}, + Tags: TagList{}, + Nodes: NodeList{}, + Projects: ProjectList{}, + Spiders: SpiderList{}, + Tasks: TaskList{}, + Jobs: JobList{}, + Schedules: ScheduleList{}, + Users: UserList{}, + Settings: SettingList{}, + Tokens: TokenList{}, + Variables: VariableList{}, + TaskQueueItems: TaskQueueItemList{}, + TaskStats: TaskStatList{}, + SpiderStats: SpiderStatList{}, + DataSources: DataSourceList{}, + DataCollections: DataCollectionList{}, + Results: ResultList{}, + Passwords: PasswordList{}, + ExtraValues: ExtraValueList{}, + Gits: GitList{}, + Roles: RoleList{}, + PermissionList: PermissionList{}, + RolePermissionList: RolePermissionList{}, + Environments: EnvironmentList{}, + } +} diff --git a/core/models/models/utils_tag.go b/core/models/models/utils_tag.go new file mode 100644 index 000000000..bdcb80461 --- /dev/null +++ b/core/models/models/utils_tag.go @@ -0,0 +1,34 @@ +package models + +import ( + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/go-trace" +) + +func convertInterfacesToTags(tags []interfaces.Tag) (res []Tag) { + if tags == nil { + return nil + } + for _, t := range tags { + tag, ok := t.(*Tag) + if !ok { + log.Warnf("%v: cannot convert tag", trace.TraceError(errors.ErrorModelInvalidType)) + return nil + } + if tag == nil { + log.Warnf("%v: cannot convert tag", trace.TraceError(errors.ErrorModelInvalidType)) + return nil + } + res = append(res, *tag) + } + return res +} + +func convertTagsToInterfaces(tags []Tag) (res []interfaces.Tag) { + for _, t := range tags { + res = append(res, &t) + } + return res +} diff --git a/core/models/models/variable.go b/core/models/models/variable.go new file mode 100644 index 000000000..08e0d15d0 --- /dev/null +++ b/core/models/models/variable.go @@ -0,0 +1,31 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Variable struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Key string `json:"key" bson:"key"` + Value string `json:"value" bson:"value"` + Remark string `json:"remark" bson:"remark"` +} + +func (v *Variable) GetId() (id primitive.ObjectID) { + return v.Id +} + +func (v *Variable) SetId(id primitive.ObjectID) { + v.Id = id +} + +type VariableList []Variable + +func (l *VariableList) GetModels() (res []interfaces.Model) { + for i := range *l { + d := (*l)[i] + res = append(res, &d) + } + return res +} diff --git a/core/models/models/variable_v2.go b/core/models/models/variable_v2.go new file mode 100644 index 000000000..497888322 --- /dev/null +++ b/core/models/models/variable_v2.go @@ -0,0 +1,9 @@ +package models + +type VariableV2 struct { + any `collection:"variables"` + BaseModelV2[VariableV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Value string `json:"value" bson:"value"` + Remark string `json:"remark" bson:"remark"` +} diff --git a/core/models/service/artifact_service.go b/core/models/service/artifact_service.go new file mode 100644 index 000000000..afa0219db --- /dev/null +++ b/core/models/service/artifact_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeArtifact(d interface{}, err error) (res *models2.Artifact, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Artifact) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetArtifactById(id primitive.ObjectID) (res *models2.Artifact, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdArtifact).GetById(id) + return convertTypeArtifact(d, err) +} + +func (svc *Service) GetArtifact(query bson.M, opts *mongo.FindOptions) (res *models2.Artifact, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdArtifact).Get(query, opts) + return convertTypeArtifact(d, err) +} + +func (svc *Service) GetArtifactList(query bson.M, opts *mongo.FindOptions) (res []models2.Artifact, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdArtifact).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.Artifact) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/base_service.go b/core/models/service/base_service.go new file mode 100644 index 000000000..1b58c46cf --- /dev/null +++ b/core/models/service/base_service.go @@ -0,0 +1,420 @@ +package service + +import ( + "encoding/json" + "github.com/apex/log" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "reflect" + "strings" + "sync" + "time" +) + +type BaseService struct { + id interfaces.ModelId + col *mongo.Col +} + +func (svc *BaseService) GetModelId() (id interfaces.ModelId) { + return svc.id +} + +func (svc *BaseService) SetModelId(id interfaces.ModelId) { + svc.id = id +} + +func (svc *BaseService) GetCol() (col *mongo.Col) { + return svc.col +} + +func (svc *BaseService) SetCol(col *mongo.Col) { + svc.col = col +} + +func (svc *BaseService) GetById(id primitive.ObjectID) (res interfaces.Model, err error) { + // find result + fr := svc.findId(id) + + // bind + return NewBasicBinder(svc.id, fr).Bind() +} + +func (svc *BaseService) Get(query bson.M, opts *mongo.FindOptions) (res interfaces.Model, err error) { + // find result + fr := svc.find(query, opts) + + // bind + return NewBasicBinder(svc.id, fr).Bind() +} + +func (svc *BaseService) GetList(query bson.M, opts *mongo.FindOptions) (l interfaces.List, err error) { + // find result + tic := time.Now() + log.Debugf("baseService.GetMany -> svc.find:start") + log.Debugf("baseService.GetMany -> svc.id: %v", svc.id) + log.Debugf("baseService.GetMany -> svc.col.GetName(): %v", svc.col.GetName()) + log.Debugf("baseService.GetMany -> query: %v", query) + log.Debugf("baseService.GetMany -> opts: %v", opts) + fr := svc.find(query, opts) + log.Debugf("baseService.GetMany -> svc.find:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds()) + + // bind + return NewListBinder(svc.id, fr).Bind() +} + +func (svc *BaseService) DeleteById(id primitive.ObjectID, args ...interface{}) (err error) { + return svc.deleteId(id, args...) +} + +func (svc *BaseService) Delete(query bson.M, args ...interface{}) (err error) { + return svc.delete(query) +} + +func (svc *BaseService) DeleteList(query bson.M, args ...interface{}) (err error) { + return svc.deleteList(query) +} + +func (svc *BaseService) ForceDeleteList(query bson.M, args ...interface{}) (err error) { + return svc.forceDeleteList(query) +} + +func (svc *BaseService) UpdateById(id primitive.ObjectID, update bson.M, args ...interface{}) (err error) { + return svc.updateId(id, update) +} + +func (svc *BaseService) Update(query bson.M, update bson.M, fields []string, args ...interface{}) (err error) { + return svc.update(query, update, fields) +} + +func (svc *BaseService) UpdateDoc(query bson.M, doc interfaces.Model, fields []string, args ...interface{}) (err error) { + return svc.update(query, doc, fields) +} + +func (svc *BaseService) Insert(u interfaces.User, docs ...interface{}) (err error) { + log.Debugf("baseService.Insert -> svc.col.GetName(): %v", svc.col.GetName()) + log.Debugf("baseService.Insert -> docs: %v", docs) + return svc.insert(u, docs...) +} + +func (svc *BaseService) Count(query bson.M) (total int, err error) { + return svc.count(query) +} + +func (svc *BaseService) findId(id primitive.ObjectID) (fr *mongo.FindResult) { + if svc.col == nil { + return mongo.NewFindResultWithError(constants.ErrMissingCol) + } + return svc.col.FindId(id) +} + +func (svc *BaseService) find(query bson.M, opts *mongo.FindOptions) (fr *mongo.FindResult) { + if svc.col == nil { + return mongo.NewFindResultWithError(constants.ErrMissingCol) + } + return svc.col.Find(query, opts) +} + +func (svc *BaseService) deleteId(id primitive.ObjectID, args ...interface{}) (err error) { + if svc.col == nil { + return trace.TraceError(constants.ErrMissingCol) + } + fr := svc.findId(id) + doc, err := NewBasicBinder(svc.id, fr).Bind() + if err != nil { + return err + } + return delegate.NewModelDelegate(doc, svc._getUserFromArgs(args...)).Delete() +} + +func (svc *BaseService) delete(query bson.M, args ...interface{}) (err error) { + if svc.col == nil { + return trace.TraceError(constants.ErrMissingCol) + } + var doc models2.BaseModel + if err := svc.find(query, nil).One(&doc); err != nil { + return err + } + return svc.deleteId(doc.Id, svc._getUserFromArgs(args...)) +} + +func (svc *BaseService) deleteList(query bson.M, args ...interface{}) (err error) { + if svc.col == nil { + return trace.TraceError(constants.ErrMissingCol) + } + fr := svc.find(query, nil) + list, err := NewListBinder(svc.id, fr).Bind() + if err != nil { + return err + } + for _, doc := range list.GetModels() { + if err := delegate.NewModelDelegate(doc, svc._getUserFromArgs(args...)).Delete(); err != nil { + return err + } + } + return nil +} + +func (svc *BaseService) forceDeleteList(query bson.M, args ...interface{}) (err error) { + return svc.col.Delete(query) +} + +func (svc *BaseService) count(query bson.M) (total int, err error) { + if svc.col == nil { + return total, trace.TraceError(constants.ErrMissingCol) + } + return svc.col.Count(query) +} + +func (svc *BaseService) update(query bson.M, update interface{}, fields []string, args ...interface{}) (err error) { + update, err = svc._getUpdateBsonM(update, fields) + if err != nil { + return err + } + return svc._update(query, update, svc._getUserFromArgs(args...)) +} + +func (svc *BaseService) updateId(id primitive.ObjectID, update interface{}, args ...interface{}) (err error) { + update, err = svc._getUpdateBsonM(update, nil) + if err != nil { + return err + } + return svc._updateById(id, update, svc._getUserFromArgs(args...)) +} + +func (svc *BaseService) insert(u interfaces.User, docs ...interface{}) (err error) { + // validate col + if svc.col == nil { + return trace.TraceError(constants.ErrMissingCol) + } + + // iterate docs + for i, doc := range docs { + switch doc.(type) { + case map[string]interface{}: + // doc type: map[string]interface{}, need to handle _id + d := doc.(map[string]interface{}) + vId, ok := d["_id"] + if !ok { + // _id not exists + d["_id"] = primitive.NewObjectID() + } else { + // _id exists + switch vId.(type) { + case string: + // _id type: string + sId, ok := vId.(string) + if ok { + d["_id"], err = primitive.ObjectIDFromHex(sId) + if err != nil { + return trace.TraceError(err) + } + } + case primitive.ObjectID: + // _id type: primitive.ObjectID + // do nothing + default: + return trace.TraceError(errors.ErrorModelInvalidType) + } + } + } + docs[i] = doc + } + + // perform insert + ids, err := svc.col.InsertMany(docs) + if err != nil { + return err + } + + // upsert artifacts + query := bson.M{ + "_id": bson.M{ + "$in": ids, + }, + } + fr := svc.col.Find(query, nil) + list, err := NewListBinder(svc.id, fr).Bind() + for _, doc := range list.GetModels() { + // upsert artifact when performing model delegate save + if err := delegate.NewModelDelegate(doc, u).Save(); err != nil { + return err + } + } + + return nil +} + +func (svc *BaseService) _update(query bson.M, update interface{}, args ...interface{}) (err error) { + // ids of query + var ids []primitive.ObjectID + list, err := NewListBinder(svc.id, svc.find(query, nil)).Bind() + if err != nil { + return err + } + for _, doc := range list.GetModels() { + ids = append(ids, doc.GetId()) + } + + // update model objects + if err := svc.col.Update(query, update); err != nil { + return err + } + + // update artifacts + u := svc._getUserFromArgs(args...) + return mongo.GetMongoCol(interfaces.ModelColNameArtifact).Update(query, svc._getUpdateArtifactUpdate(u)) +} + +func (svc *BaseService) _updateById(id primitive.ObjectID, update interface{}, args ...interface{}) (err error) { + // update model object + if err := svc.col.UpdateId(id, update); err != nil { + return err + } + + // update artifact + u := svc._getUserFromArgs(args...) + return mongo.GetMongoCol(interfaces.ModelColNameArtifact).UpdateId(id, svc._getUpdateArtifactUpdate(u)) +} + +func (svc *BaseService) _getUpdateBsonM(update interface{}, fields []string) (res bson.M, err error) { + switch update.(type) { + case interfaces.Model: + // convert to bson.M + var updateBsonM bson.M + bytes, err := json.Marshal(&update) + if err != nil { + return nil, err + } + if err := json.Unmarshal(bytes, &updateBsonM); err != nil { + return nil, err + } + return svc._getUpdateBsonM(updateBsonM, fields) + + case bson.M: + // convert to bson.M + updateBsonM := update.(bson.M) + + // filter fields if not nil + if fields != nil { + // fields map + fieldsMap := map[string]bool{} + for _, f := range fields { + fieldsMap[f] = true + } + + // remove unselected fields + for k := range updateBsonM { + if _, ok := fieldsMap[k]; !ok { + delete(updateBsonM, k) + } + } + } + + // normalize update bson.M + if !svc._containsDollar(updateBsonM) { + if _, ok := updateBsonM["$set"]; !ok { + updateBsonM = bson.M{ + "$set": updateBsonM, + } + } + } + + return updateBsonM, nil + } + + v := reflect.ValueOf(update) + switch v.Kind() { + case reflect.Struct: + if v.CanAddr() { + update = v.Addr().Interface() + return svc._getUpdateBsonM(update, fields) + } + return nil, errors.ErrorModelInvalidType + default: + return nil, errors.ErrorModelInvalidType + } +} + +func (svc *BaseService) _getUpdateArtifactUpdate(u interfaces.User) (res bson.M) { + var uid primitive.ObjectID + if u != nil { + uid = u.GetId() + } + return bson.M{ + "$set": bson.M{ + "_sys.update_ts": time.Now(), + "_sys.update_uid": uid, + }, + } +} + +func (svc *BaseService) _getUserFromArgs(args ...interface{}) (u interfaces.User) { + return utils.GetUserFromArgs(args...) +} + +func (svc *BaseService) _containsDollar(updateBsonM bson.M) (ok bool) { + for k := range updateBsonM { + if strings.HasPrefix(k, "$") { + return true + } + } + return false +} + +func NewBaseService(id interfaces.ModelId, opts ...BaseServiceOption) (svc2 interfaces.ModelBaseService) { + // service + svc := &BaseService{ + id: id, + } + + // apply options + for _, opt := range opts { + opt(svc) + } + + // get collection name if not set + if svc.GetCol() == nil { + colName := models2.GetModelColName(id) + svc.SetCol(mongo.GetMongoCol(colName)) + } + + return svc +} + +var store = sync.Map{} + +func GetBaseService(id interfaces.ModelId) (svc interfaces.ModelBaseService) { + res, ok := store.Load(id) + if ok { + svc, ok = res.(interfaces.ModelBaseService) + if ok { + return svc + } + } + svc = NewBaseService(id) + store.Store(id, svc) + return svc +} + +func GetBaseServiceByColName(id interfaces.ModelId, colName string) (svc interfaces.ModelBaseService) { + res, ok := store.Load(colName) + if ok { + svc, ok = res.(interfaces.ModelBaseService) + if ok { + return svc + } + } + col := mongo.GetMongoCol(colName) + svc = NewBaseService(id, WithBaseServiceCol(col)) + store.Store(colName, svc) + return svc +} diff --git a/core/models/service/base_service_v2.go b/core/models/service/base_service_v2.go new file mode 100644 index 000000000..1197c9bf0 --- /dev/null +++ b/core/models/service/base_service_v2.go @@ -0,0 +1,181 @@ +package service + +import ( + "fmt" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "reflect" + "sync" +) + +var ( + instanceMap = make(map[string]any) + onceMap = make(map[string]*sync.Once) + onceColNameMap = make(map[string]*sync.Once) + mu sync.Mutex +) + +type ModelServiceV2[T any] struct { + col *mongo.Col +} + +func (svc *ModelServiceV2[T]) GetById(id primitive.ObjectID) (model *T, err error) { + var result T + err = svc.col.FindId(id).One(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (svc *ModelServiceV2[T]) GetOne(query bson.M, options *mongo.FindOptions) (model *T, err error) { + var result T + err = svc.col.Find(query, options).One(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (svc *ModelServiceV2[T]) GetMany(query bson.M, options *mongo.FindOptions) (models []T, err error) { + var result []T + err = svc.col.Find(query, options).All(&result) + if err != nil { + return nil, err + } + return result, nil +} + +func (svc *ModelServiceV2[T]) DeleteById(id primitive.ObjectID) (err error) { + return svc.col.DeleteId(id) +} + +func (svc *ModelServiceV2[T]) DeleteOne(query bson.M) (err error) { + _, err = svc.col.GetCollection().DeleteOne(svc.col.GetContext(), query) + return err +} + +func (svc *ModelServiceV2[T]) DeleteMany(query bson.M) (err error) { + _, err = svc.col.GetCollection().DeleteMany(svc.col.GetContext(), query, nil) + return err +} + +func (svc *ModelServiceV2[T]) UpdateById(id primitive.ObjectID, update bson.M) (err error) { + return svc.col.UpdateId(id, update) +} + +func (svc *ModelServiceV2[T]) UpdateOne(query bson.M, update bson.M) (err error) { + _, err = svc.col.GetCollection().UpdateOne(svc.col.GetContext(), query, update) + return err +} + +func (svc *ModelServiceV2[T]) UpdateMany(query bson.M, update bson.M) (err error) { + _, err = svc.col.GetCollection().UpdateMany(svc.col.GetContext(), query, update) + return err +} + +func (svc *ModelServiceV2[T]) ReplaceById(id primitive.ObjectID, model T) (err error) { + _, err = svc.col.GetCollection().ReplaceOne(svc.col.GetContext(), bson.M{"_id": id}, model) + return err +} + +func (svc *ModelServiceV2[T]) ReplaceOne(query bson.M, model T) (err error) { + _, err = svc.col.GetCollection().ReplaceOne(svc.col.GetContext(), query, model) + return err +} + +func (svc *ModelServiceV2[T]) InsertOne(model T) (id primitive.ObjectID, err error) { + m := any(&model).(interfaces.Model) + if m.GetId().IsZero() { + m.SetId(primitive.NewObjectID()) + } + res, err := svc.col.GetCollection().InsertOne(svc.col.GetContext(), m) + if err != nil { + return primitive.NilObjectID, err + } + return res.InsertedID.(primitive.ObjectID), nil +} + +func (svc *ModelServiceV2[T]) InsertMany(models []T) (ids []primitive.ObjectID, err error) { + var _models []any + for _, model := range models { + m := any(&model).(interfaces.Model) + if m.GetId().IsZero() { + m.SetId(primitive.NewObjectID()) + } + _models = append(_models, m) + } + res, err := svc.col.GetCollection().InsertMany(svc.col.GetContext(), _models) + if err != nil { + return nil, err + } + for _, v := range res.InsertedIDs { + ids = append(ids, v.(primitive.ObjectID)) + } + return ids, nil +} + +func (svc *ModelServiceV2[T]) Count(query bson.M) (total int, err error) { + return svc.col.Count(query) +} + +func (svc *ModelServiceV2[T]) GetCol() (col *mongo.Col) { + return svc.col +} + +func GetCollectionNameByInstance(v any) string { + t := reflect.TypeOf(v) + field := t.Field(0) + return field.Tag.Get("collection") +} + +func getCollectionName[T any]() string { + var instance T + t := reflect.TypeOf(instance) + field := t.Field(0) + return field.Tag.Get("collection") +} + +// NewModelServiceV2 return singleton instance of ModelServiceV2 +func NewModelServiceV2[T any]() *ModelServiceV2[T] { + typeName := fmt.Sprintf("%T", *new(T)) + + mu.Lock() + defer mu.Unlock() + + if _, exists := onceMap[typeName]; !exists { + onceMap[typeName] = &sync.Once{} + } + + var instance *ModelServiceV2[T] + + onceMap[typeName].Do(func() { + collectionName := getCollectionName[T]() + collection := mongo.GetMongoCol(collectionName) + instance = &ModelServiceV2[T]{col: collection} + instanceMap[typeName] = instance + }) + + return instanceMap[typeName].(*ModelServiceV2[T]) +} + +func NewModelServiceV2WithColName[T any](colName string) *ModelServiceV2[T] { + mu.Lock() + defer mu.Unlock() + + if _, exists := onceColNameMap[colName]; !exists { + onceColNameMap[colName] = &sync.Once{} + } + + var instance *ModelServiceV2[T] + + onceColNameMap[colName].Do(func() { + collection := mongo.GetMongoCol(colName) + instance = &ModelServiceV2[T]{col: collection} + instanceMap[colName] = instance + }) + + return instanceMap[colName].(*ModelServiceV2[T]) +} diff --git a/core/models/service/base_service_v2_test.go b/core/models/service/base_service_v2_test.go new file mode 100644 index 000000000..ff5797fcd --- /dev/null +++ b/core/models/service/base_service_v2_test.go @@ -0,0 +1,259 @@ +package service_test + +import ( + "context" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "testing" + "time" +) + +type TestModel struct { + Id primitive.ObjectID `bson:"_id,omitempty" collection:"testmodels"` + models.BaseModelV2[TestModel] `bson:",inline"` + Name string `bson:"name"` +} + +func setupTestDB() { + viper.Set("mongo.db", "testdb") +} + +func teardownTestDB() { + db := mongo.GetMongoDb("testdb") + err := db.Drop(context.Background()) + if err != nil { + return + } +} + +func TestModelServiceV2_GetById(t *testing.T) { + setupTestDB() + defer teardownTestDB() + + svc := service.NewModelServiceV2[TestModel]() + testModel := TestModel{Name: "Test Name"} + + id, err := svc.InsertOne(testModel) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + result, err := svc.GetById(id) + require.Nil(t, err) + assert.Equal(t, testModel.Name, result.Name) +} + +func TestModelServiceV2_GetOne(t *testing.T) { + setupTestDB() + defer teardownTestDB() + + svc := service.NewModelServiceV2[TestModel]() + testModel := TestModel{Name: "Test Name"} + + _, err := svc.InsertOne(testModel) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + result, err := svc.GetOne(bson.M{"name": "Test Name"}, nil) + require.Nil(t, err) + assert.Equal(t, testModel.Name, result.Name) +} + +func TestModelServiceV2_GetMany(t *testing.T) { + setupTestDB() + defer teardownTestDB() + + svc := service.NewModelServiceV2[TestModel]() + testModels := []TestModel{ + {Name: "Name1"}, + {Name: "Name2"}, + } + + _, err := svc.InsertMany(testModels) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + results, err := svc.GetMany(bson.M{}, nil) + require.Nil(t, err) + assert.Equal(t, 2, len(results)) +} + +func TestModelServiceV2_InsertOne(t *testing.T) { + setupTestDB() + defer teardownTestDB() + + svc := service.NewModelServiceV2[TestModel]() + testModel := TestModel{Name: "Test Name"} + + id, err := svc.InsertOne(testModel) + require.Nil(t, err) + assert.NotEqual(t, primitive.NilObjectID, id) +} + +func TestModelServiceV2_InsertMany(t *testing.T) { + setupTestDB() + defer teardownTestDB() + + svc := service.NewModelServiceV2[TestModel]() + testModels := []TestModel{ + {Name: "Name1"}, + {Name: "Name2"}, + } + + ids, err := svc.InsertMany(testModels) + require.Nil(t, err) + assert.Equal(t, 2, len(ids)) +} + +func TestModelServiceV2_UpdateById(t *testing.T) { + setupTestDB() + defer teardownTestDB() + + svc := service.NewModelServiceV2[TestModel]() + testModel := TestModel{Name: "Old Name"} + + id, err := svc.InsertOne(testModel) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + update := bson.M{"$set": bson.M{"name": "New Name"}} + err = svc.UpdateById(id, update) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + result, err := svc.GetById(id) + require.Nil(t, err) + assert.Equal(t, "New Name", result.Name) +} + +func TestModelServiceV2_UpdateOne(t *testing.T) { + setupTestDB() + defer teardownTestDB() + + svc := service.NewModelServiceV2[TestModel]() + testModel := TestModel{Name: "Old Name"} + + _, err := svc.InsertOne(testModel) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + update := bson.M{"$set": bson.M{"name": "New Name"}} + err = svc.UpdateOne(bson.M{"name": "Old Name"}, update) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + result, err := svc.GetOne(bson.M{"name": "New Name"}, nil) + require.Nil(t, err) + assert.Equal(t, "New Name", result.Name) +} + +func TestModelServiceV2_UpdateMany(t *testing.T) { + setupTestDB() + defer teardownTestDB() + + svc := service.NewModelServiceV2[TestModel]() + testModels := []TestModel{ + {Name: "Old Name1"}, + {Name: "Old Name2"}, + } + + _, err := svc.InsertMany(testModels) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + update := bson.M{"$set": bson.M{"name": "New Name"}} + err = svc.UpdateMany(bson.M{"name": bson.M{"$regex": "^Old"}}, update) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + results, err := svc.GetMany(bson.M{"name": "New Name"}, nil) + require.Nil(t, err) + assert.Equal(t, 2, len(results)) +} + +func TestModelServiceV2_DeleteById(t *testing.T) { + setupTestDB() + defer teardownTestDB() + + svc := service.NewModelServiceV2[TestModel]() + testModel := TestModel{Name: "Test Name"} + + id, err := svc.InsertOne(testModel) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + err = svc.DeleteById(id) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + result, err := svc.GetById(id) + assert.NotNil(t, err) + assert.Nil(t, result) +} + +func TestModelServiceV2_DeleteOne(t *testing.T) { + setupTestDB() + defer teardownTestDB() + + svc := service.NewModelServiceV2[TestModel]() + testModel := TestModel{Name: "Test Name"} + + _, err := svc.InsertOne(testModel) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + err = svc.DeleteOne(bson.M{"name": "Test Name"}) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + result, err := svc.GetOne(bson.M{"name": "Test Name"}, nil) + assert.NotNil(t, err) + assert.Nil(t, result) +} + +func TestModelServiceV2_DeleteMany(t *testing.T) { + setupTestDB() + defer teardownTestDB() + + svc := service.NewModelServiceV2[TestModel]() + testModels := []TestModel{ + {Name: "Test Name1"}, + {Name: "Test Name2"}, + } + + _, err := svc.InsertMany(testModels) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + err = svc.DeleteMany(bson.M{"name": bson.M{"$regex": "^Test Name"}}) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + results, err := svc.GetMany(bson.M{"name": bson.M{"$regex": "^Test Name"}}, nil) + require.Nil(t, err) + assert.Equal(t, 0, len(results)) +} + +func TestModelServiceV2_Count(t *testing.T) { + setupTestDB() + defer teardownTestDB() + + svc := service.NewModelServiceV2[TestModel]() + testModels := []TestModel{ + {Name: "Name1"}, + {Name: "Name2"}, + } + + _, err := svc.InsertMany(testModels) + require.Nil(t, err) + time.Sleep(100 * time.Millisecond) + + total, err := svc.Count(bson.M{}) + require.Nil(t, err) + assert.Equal(t, 2, total) +} diff --git a/core/models/service/binder_basic.go b/core/models/service/binder_basic.go new file mode 100644 index 000000000..d4f331217 --- /dev/null +++ b/core/models/service/binder_basic.go @@ -0,0 +1,92 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" +) + +func NewBasicBinder(id interfaces.ModelId, fr *mongo.FindResult) (b interfaces.ModelBinder) { + return &BasicBinder{ + id: id, + fr: fr, + m: models.NewModelMap(), + } +} + +type BasicBinder struct { + id interfaces.ModelId + fr *mongo.FindResult + m *models.ModelMap +} + +func (b *BasicBinder) Bind() (res interfaces.Model, err error) { + m := b.m + + switch b.id { + case interfaces.ModelIdArtifact: + return b.Process(&m.Artifact) + case interfaces.ModelIdTag: + return b.Process(&m.Tag) + case interfaces.ModelIdNode: + return b.Process(&m.Node) + case interfaces.ModelIdProject: + return b.Process(&m.Project) + case interfaces.ModelIdSpider: + return b.Process(&m.Spider) + case interfaces.ModelIdTask: + return b.Process(&m.Task) + case interfaces.ModelIdJob: + return b.Process(&m.Job) + case interfaces.ModelIdSchedule: + return b.Process(&m.Schedule) + case interfaces.ModelIdUser: + return b.Process(&m.User) + case interfaces.ModelIdSetting: + return b.Process(&m.Setting) + case interfaces.ModelIdToken: + return b.Process(&m.Token) + case interfaces.ModelIdVariable: + return b.Process(&m.Variable) + case interfaces.ModelIdTaskQueue: + return b.Process(&m.TaskQueueItem) + case interfaces.ModelIdTaskStat: + return b.Process(&m.TaskStat) + case interfaces.ModelIdSpiderStat: + return b.Process(&m.SpiderStat) + case interfaces.ModelIdDataSource: + return b.Process(&m.DataSource) + case interfaces.ModelIdDataCollection: + return b.Process(&m.DataCollection) + case interfaces.ModelIdResult: + return b.Process(&m.Result) + case interfaces.ModelIdPassword: + return b.Process(&m.Password) + case interfaces.ModelIdExtraValue: + return b.Process(&m.ExtraValue) + case interfaces.ModelIdGit: + return b.Process(&m.Git) + case interfaces.ModelIdRole: + return b.Process(&m.Role) + case interfaces.ModelIdUserRole: + return b.Process(&m.UserRole) + case interfaces.ModelIdPermission: + return b.Process(&m.Permission) + case interfaces.ModelIdRolePermission: + return b.Process(&m.RolePermission) + case interfaces.ModelIdEnvironment: + return b.Process(&m.Environment) + case interfaces.ModelIdDependencySetting: + return b.Process(&m.DependencySetting) + default: + return nil, errors.ErrorModelInvalidModelId + } +} + +func (b *BasicBinder) Process(d interfaces.Model) (res interfaces.Model, err error) { + if err := b.fr.One(d); err != nil { + return nil, err + } + return d, nil +} diff --git a/core/models/service/binder_list.go b/core/models/service/binder_list.go new file mode 100644 index 000000000..34687c21d --- /dev/null +++ b/core/models/service/binder_list.go @@ -0,0 +1,93 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/go-trace" +) + +func NewListBinder(id interfaces.ModelId, fr *mongo.FindResult) (b interfaces.ModelListBinder) { + return &ListBinder{ + id: id, + m: models.NewModelListMap(), + fr: fr, + b: NewBasicBinder(id, fr), + } +} + +type ListBinder struct { + id interfaces.ModelId + m *models.ModelListMap + fr *mongo.FindResult + b interfaces.ModelBinder +} + +func (b *ListBinder) Bind() (l interfaces.List, err error) { + m := b.m + + switch b.id { + case interfaces.ModelIdArtifact: + return b.Process(&m.Artifacts) + case interfaces.ModelIdTag: + return b.Process(&m.Tags) + case interfaces.ModelIdNode: + return b.Process(&m.Nodes) + case interfaces.ModelIdProject: + return b.Process(&m.Projects) + case interfaces.ModelIdSpider: + return b.Process(&m.Spiders) + case interfaces.ModelIdTask: + return b.Process(&m.Tasks) + case interfaces.ModelIdSchedule: + return b.Process(&m.Schedules) + case interfaces.ModelIdUser: + return b.Process(&m.Users) + case interfaces.ModelIdSetting: + return b.Process(&m.Settings) + case interfaces.ModelIdToken: + return b.Process(&m.Tokens) + case interfaces.ModelIdVariable: + return b.Process(&m.Variables) + case interfaces.ModelIdTaskQueue: + return b.Process(&m.TaskQueueItems) + case interfaces.ModelIdTaskStat: + return b.Process(&m.TaskStats) + case interfaces.ModelIdSpiderStat: + return b.Process(&m.SpiderStats) + case interfaces.ModelIdDataSource: + return b.Process(&m.DataSources) + case interfaces.ModelIdDataCollection: + return b.Process(&m.DataCollections) + case interfaces.ModelIdResult: + return b.Process(&m.Results) + case interfaces.ModelIdPassword: + return b.Process(&m.Passwords) + case interfaces.ModelIdExtraValue: + return b.Process(&m.ExtraValues) + case interfaces.ModelIdGit: + return b.Process(&m.Gits) + case interfaces.ModelIdRole: + return b.Process(&m.Roles) + case interfaces.ModelIdUserRole: + return b.Process(&m.UserRoles) + case interfaces.ModelIdPermission: + return b.Process(&m.PermissionList) + case interfaces.ModelIdRolePermission: + return b.Process(&m.RolePermissionList) + case interfaces.ModelIdEnvironment: + return b.Process(&m.Environments) + case interfaces.ModelIdDependencySetting: + return b.Process(&m.DependencySettings) + default: + return l, errors.ErrorModelInvalidModelId + } +} + +func (b *ListBinder) Process(d interface{}) (l interfaces.List, err error) { + if err := b.fr.All(d); err != nil { + return l, trace.TraceError(err) + } + return d.(interfaces.List), nil +} diff --git a/core/models/service/data_collection_service.go b/core/models/service/data_collection_service.go new file mode 100644 index 000000000..90527954e --- /dev/null +++ b/core/models/service/data_collection_service.go @@ -0,0 +1,45 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeDataCollection(d interface{}, err error) (res *models2.DataCollection, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.DataCollection) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetDataCollectionById(id primitive.ObjectID) (res *models2.DataCollection, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdDataCollection).GetById(id) + return convertTypeDataCollection(d, err) +} + +func (svc *Service) GetDataCollection(query bson.M, opts *mongo.FindOptions) (res *models2.DataCollection, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdDataCollection).Get(query, opts) + return convertTypeDataCollection(d, err) +} + +func (svc *Service) GetDataCollectionList(query bson.M, opts *mongo.FindOptions) (res []models2.DataCollection, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdDataCollection).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.DataCollection) + res = append(res, *d) + } + return res, nil +} + +func (svc *Service) GetDataCollectionByName(name string, opts *mongo.FindOptions) (res *models2.DataCollection, err error) { + query := bson.M{"name": name} + return svc.GetDataCollection(query, opts) +} diff --git a/core/models/service/data_source_service.go b/core/models/service/data_source_service.go new file mode 100644 index 000000000..968678191 --- /dev/null +++ b/core/models/service/data_source_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeDataSource(d interface{}, err error) (res *models2.DataSource, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.DataSource) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetDataSourceById(id primitive.ObjectID) (res *models2.DataSource, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdDataSource).GetById(id) + return convertTypeDataSource(d, err) +} + +func (svc *Service) GetDataSource(query bson.M, opts *mongo.FindOptions) (res *models2.DataSource, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdDataSource).Get(query, opts) + return convertTypeDataSource(d, err) +} + +func (svc *Service) GetDataSourceList(query bson.M, opts *mongo.FindOptions) (res []models2.DataSource, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdDataSource).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.DataSource) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/dependency_setting_service.go b/core/models/service/dependency_setting_service.go new file mode 100644 index 000000000..1aba0a58f --- /dev/null +++ b/core/models/service/dependency_setting_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeDependencySetting(d interface{}, err error) (res *models2.DependencySetting, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.DependencySetting) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetDependencySettingById(id primitive.ObjectID) (res *models2.DependencySetting, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdDependencySetting).GetById(id) + return convertTypeDependencySetting(d, err) +} + +func (svc *Service) GetDependencySetting(query bson.M, opts *mongo.FindOptions) (res *models2.DependencySetting, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdDependencySetting).Get(query, opts) + return convertTypeDependencySetting(d, err) +} + +func (svc *Service) GetDependencySettingList(query bson.M, opts *mongo.FindOptions) (res []models2.DependencySetting, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdDependencySetting).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.DependencySetting) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/environment_service.go b/core/models/service/environment_service.go new file mode 100644 index 000000000..a9d74ce4c --- /dev/null +++ b/core/models/service/environment_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeEnvironment(d interface{}, err error) (res *models2.Environment, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Environment) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetEnvironmentById(id primitive.ObjectID) (res *models2.Environment, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdEnvironment).GetById(id) + return convertTypeEnvironment(d, err) +} + +func (svc *Service) GetEnvironment(query bson.M, opts *mongo.FindOptions) (res *models2.Environment, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdEnvironment).Get(query, opts) + return convertTypeEnvironment(d, err) +} + +func (svc *Service) GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []models2.Environment, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdEnvironment).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.Environment) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/extra_value_service.go b/core/models/service/extra_value_service.go new file mode 100644 index 000000000..7e223a0d0 --- /dev/null +++ b/core/models/service/extra_value_service.go @@ -0,0 +1,45 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeExtraValue(d interface{}, err error) (res *models.ExtraValue, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models.ExtraValue) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetExtraValueById(id primitive.ObjectID) (res *models.ExtraValue, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdExtraValue).GetById(id) + return convertTypeExtraValue(d, err) +} + +func (svc *Service) GetExtraValue(query bson.M, opts *mongo.FindOptions) (res *models.ExtraValue, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdExtraValue).Get(query, opts) + return convertTypeExtraValue(d, err) +} + +func (svc *Service) GetExtraValueList(query bson.M, opts *mongo.FindOptions) (res []models.ExtraValue, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdExtraValue).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models.ExtraValue) + res = append(res, *d) + } + return res, nil +} + +func (svc *Service) GetExtraValueByObjectIdModel(oid primitive.ObjectID, m string, opts *mongo.FindOptions) (res *models.ExtraValue, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdExtraValue).Get(bson.M{"oid": oid, "m": m}, opts) + return convertTypeExtraValue(d, err) +} diff --git a/core/models/service/git_service.go b/core/models/service/git_service.go new file mode 100644 index 000000000..8dbf83460 --- /dev/null +++ b/core/models/service/git_service.go @@ -0,0 +1,43 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeGit(d interface{}, err error) (res *models2.Git, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Git) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetGitById(id primitive.ObjectID) (res *models2.Git, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdGit).GetById(id) + return convertTypeGit(d, err) +} + +func (svc *Service) GetGit(query bson.M, opts *mongo.FindOptions) (res *models2.Git, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdGit).Get(query, opts) + return convertTypeGit(d, err) +} + +func (svc *Service) GetGitList(query bson.M, opts *mongo.FindOptions) (res []models2.Git, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdGit).GetList(query, opts) + if l == nil { + return nil, nil + } + for _, doc := range l.GetModels() { + d := doc.(*models2.Git) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/interface.go b/core/models/service/interface.go new file mode 100644 index 000000000..fa633a885 --- /dev/null +++ b/core/models/service/interface.go @@ -0,0 +1,105 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type ModelService interface { + interfaces.ModelService + DropAll() (err error) + GetNodeById(id primitive.ObjectID) (res *models.Node, err error) + GetNode(query bson.M, opts *mongo.FindOptions) (res *models.Node, err error) + GetNodeList(query bson.M, opts *mongo.FindOptions) (res []models.Node, err error) + GetNodeByKey(key string, opts *mongo.FindOptions) (res *models.Node, err error) + GetProjectById(id primitive.ObjectID) (res *models.Project, err error) + GetProject(query bson.M, opts *mongo.FindOptions) (res *models.Project, err error) + GetProjectList(query bson.M, opts *mongo.FindOptions) (res []models.Project, err error) + GetArtifactById(id primitive.ObjectID) (res *models.Artifact, err error) + GetArtifact(query bson.M, opts *mongo.FindOptions) (res *models.Artifact, err error) + GetArtifactList(query bson.M, opts *mongo.FindOptions) (res []models.Artifact, err error) + GetTagById(id primitive.ObjectID) (res *models.Tag, err error) + GetTag(query bson.M, opts *mongo.FindOptions) (res *models.Tag, err error) + GetTagList(query bson.M, opts *mongo.FindOptions) (res []models.Tag, err error) + GetTagIds(colName string, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error) + UpdateTagsById(colName string, id primitive.ObjectID, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error) + UpdateTags(colName string, query bson.M, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error) + GetJobById(id primitive.ObjectID) (res *models.Job, err error) + GetJob(query bson.M, opts *mongo.FindOptions) (res *models.Job, err error) + GetJobList(query bson.M, opts *mongo.FindOptions) (res []models.Job, err error) + GetScheduleById(id primitive.ObjectID) (res *models.Schedule, err error) + GetSchedule(query bson.M, opts *mongo.FindOptions) (res *models.Schedule, err error) + GetScheduleList(query bson.M, opts *mongo.FindOptions) (res []models.Schedule, err error) + GetUserById(id primitive.ObjectID) (res *models.User, err error) + GetUser(query bson.M, opts *mongo.FindOptions) (res *models.User, err error) + GetUserList(query bson.M, opts *mongo.FindOptions) (res []models.User, err error) + GetUserByUsername(username string, opts *mongo.FindOptions) (res *models.User, err error) + GetUserByUsernameWithPassword(username string, opts *mongo.FindOptions) (res *models.User, err error) + GetSettingById(id primitive.ObjectID) (res *models.Setting, err error) + GetSetting(query bson.M, opts *mongo.FindOptions) (res *models.Setting, err error) + GetSettingList(query bson.M, opts *mongo.FindOptions) (res []models.Setting, err error) + GetSettingByKey(key string, opts *mongo.FindOptions) (res *models.Setting, err error) + GetSpiderById(id primitive.ObjectID) (res *models.Spider, err error) + GetSpider(query bson.M, opts *mongo.FindOptions) (res *models.Spider, err error) + GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []models.Spider, err error) + GetTaskById(id primitive.ObjectID) (res *models.Task, err error) + GetTask(query bson.M, opts *mongo.FindOptions) (res *models.Task, err error) + GetTaskList(query bson.M, opts *mongo.FindOptions) (res []models.Task, err error) + GetTokenById(id primitive.ObjectID) (res *models.Token, err error) + GetToken(query bson.M, opts *mongo.FindOptions) (res *models.Token, err error) + GetTokenList(query bson.M, opts *mongo.FindOptions) (res []models.Token, err error) + GetVariableById(id primitive.ObjectID) (res *models.Variable, err error) + GetVariable(query bson.M, opts *mongo.FindOptions) (res *models.Variable, err error) + GetVariableList(query bson.M, opts *mongo.FindOptions) (res []models.Variable, err error) + GetVariableByKey(key string, opts *mongo.FindOptions) (res *models.Variable, err error) + GetTaskQueueItemById(id primitive.ObjectID) (res *models.TaskQueueItem, err error) + GetTaskQueueItem(query bson.M, opts *mongo.FindOptions) (res *models.TaskQueueItem, err error) + GetTaskQueueItemList(query bson.M, opts *mongo.FindOptions) (res []models.TaskQueueItem, err error) + GetTaskStatById(id primitive.ObjectID) (res *models.TaskStat, err error) + GetTaskStat(query bson.M, opts *mongo.FindOptions) (res *models.TaskStat, err error) + GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []models.TaskStat, err error) + GetSpiderStatById(id primitive.ObjectID) (res *models.SpiderStat, err error) + GetSpiderStat(query bson.M, opts *mongo.FindOptions) (res *models.SpiderStat, err error) + GetSpiderStatList(query bson.M, opts *mongo.FindOptions) (res []models.SpiderStat, err error) + GetDataSourceById(id primitive.ObjectID) (res *models.DataSource, err error) + GetDataSource(query bson.M, opts *mongo.FindOptions) (res *models.DataSource, err error) + GetDataSourceList(query bson.M, opts *mongo.FindOptions) (res []models.DataSource, err error) + GetDataCollectionById(id primitive.ObjectID) (res *models.DataCollection, err error) + GetDataCollection(query bson.M, opts *mongo.FindOptions) (res *models.DataCollection, err error) + GetDataCollectionList(query bson.M, opts *mongo.FindOptions) (res []models.DataCollection, err error) + GetDataCollectionByName(name string, opts *mongo.FindOptions) (res *models.DataCollection, err error) + GetPasswordById(id primitive.ObjectID) (res *models.Password, err error) + GetPassword(query bson.M, opts *mongo.FindOptions) (res *models.Password, err error) + GetPasswordList(query bson.M, opts *mongo.FindOptions) (res []models.Password, err error) + GetExtraValueById(id primitive.ObjectID) (res *models.ExtraValue, err error) + GetExtraValue(query bson.M, opts *mongo.FindOptions) (res *models.ExtraValue, err error) + GetExtraValueList(query bson.M, opts *mongo.FindOptions) (res []models.ExtraValue, err error) + GetExtraValueByObjectIdModel(oid primitive.ObjectID, m string, opts *mongo.FindOptions) (res *models.ExtraValue, err error) + GetGitById(id primitive.ObjectID) (res *models.Git, err error) + GetGit(query bson.M, opts *mongo.FindOptions) (res *models.Git, err error) + GetGitList(query bson.M, opts *mongo.FindOptions) (res []models.Git, err error) + GetRoleById(id primitive.ObjectID) (res *models.Role, err error) + GetRole(query bson.M, opts *mongo.FindOptions) (res *models.Role, err error) + GetRoleList(query bson.M, opts *mongo.FindOptions) (res []models.Role, err error) + GetRoleByName(name string, opts *mongo.FindOptions) (res *models.Role, err error) + GetRoleByKey(key string, opts *mongo.FindOptions) (res *models.Role, err error) + GetUserRoleById(id primitive.ObjectID) (res *models.UserRole, err error) + GetUserRole(query bson.M, opts *mongo.FindOptions) (res *models.UserRole, err error) + GetUserRoleList(query bson.M, opts *mongo.FindOptions) (res []models.UserRole, err error) + GetUserRoleListByUserId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.UserRole, err error) + GetUserRoleListByRoleId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.UserRole, err error) + GetPermissionById(id primitive.ObjectID) (res *models.Permission, err error) + GetPermission(query bson.M, opts *mongo.FindOptions) (res *models.Permission, err error) + GetPermissionList(query bson.M, opts *mongo.FindOptions) (res []models.Permission, err error) + GetPermissionByKey(key string, opts *mongo.FindOptions) (res *models.Permission, err error) + GetRolePermission(query bson.M, opts *mongo.FindOptions) (res *models.RolePermission, err error) + GetRolePermissionList(query bson.M, opts *mongo.FindOptions) (res []models.RolePermission, err error) + GetRolePermissionListByRoleId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.RolePermission, err error) + GetRolePermissionListByPermissionId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models.RolePermission, err error) + GetEnvironmentById(id primitive.ObjectID) (res *models.Environment, err error) + GetEnvironment(query bson.M, opts *mongo.FindOptions) (res *models.Environment, err error) + GetEnvironmentList(query bson.M, opts *mongo.FindOptions) (res []models.Environment, err error) +} diff --git a/core/models/service/job_service.go b/core/models/service/job_service.go new file mode 100644 index 000000000..3a3ed7118 --- /dev/null +++ b/core/models/service/job_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeJob(d interface{}, err error) (res *models2.Job, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Job) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetJobById(id primitive.ObjectID) (res *models2.Job, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdJob).GetById(id) + return convertTypeJob(d, err) +} + +func (svc *Service) GetJob(query bson.M, opts *mongo.FindOptions) (res *models2.Job, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdJob).Get(query, opts) + return convertTypeJob(d, err) +} + +func (svc *Service) GetJobList(query bson.M, opts *mongo.FindOptions) (res []models2.Job, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdJob).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.Job) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/node_service.go b/core/models/service/node_service.go new file mode 100644 index 000000000..37ac60a6f --- /dev/null +++ b/core/models/service/node_service.go @@ -0,0 +1,45 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeNode(d interface{}, err error) (res *models2.Node, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Node) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetNodeById(id primitive.ObjectID) (res *models2.Node, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdNode).GetById(id) + return convertTypeNode(d, err) +} + +func (svc *Service) GetNode(query bson.M, opts *mongo.FindOptions) (res *models2.Node, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdNode).Get(query, opts) + return convertTypeNode(d, err) +} + +func (svc *Service) GetNodeList(query bson.M, opts *mongo.FindOptions) (res []models2.Node, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdNode).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.Node) + res = append(res, *d) + } + return res, nil +} + +func (svc *Service) GetNodeByKey(key string, opts *mongo.FindOptions) (res *models2.Node, err error) { + query := bson.M{"key": key} + return svc.GetNode(query, opts) +} diff --git a/core/models/service/options.go b/core/models/service/options.go new file mode 100644 index 000000000..b8694729e --- /dev/null +++ b/core/models/service/options.go @@ -0,0 +1,25 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/interfaces" +) + +type Option func(ModelService) + +type BaseServiceOption func(svc interfaces.ModelBaseService) + +func WithBaseServiceModelId(id interfaces.ModelId) BaseServiceOption { + return func(svc interfaces.ModelBaseService) { + svc.SetModelId(id) + } +} + +func WithBaseServiceCol(col *mongo.Col) BaseServiceOption { + return func(svc interfaces.ModelBaseService) { + _svc, ok := svc.(*BaseService) + if ok { + _svc.SetCol(col) + } + } +} diff --git a/core/models/service/password_service.go b/core/models/service/password_service.go new file mode 100644 index 000000000..021310745 --- /dev/null +++ b/core/models/service/password_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypePassword(d interface{}, err error) (res *models2.Password, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Password) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetPasswordById(id primitive.ObjectID) (res *models2.Password, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdPassword).GetById(id) + return convertTypePassword(d, err) +} + +func (svc *Service) GetPassword(query bson.M, opts *mongo.FindOptions) (res *models2.Password, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdPassword).Get(query, opts) + return convertTypePassword(d, err) +} + +func (svc *Service) GetPasswordList(query bson.M, opts *mongo.FindOptions) (res []models2.Password, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdPassword).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.Password) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/permission_service.go b/core/models/service/permission_service.go new file mode 100644 index 000000000..88b41df0a --- /dev/null +++ b/core/models/service/permission_service.go @@ -0,0 +1,48 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypePermission(d interface{}, err error) (res *models2.Permission, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Permission) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetPermissionById(id primitive.ObjectID) (res *models2.Permission, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdPermission).GetById(id) + return convertTypePermission(d, err) +} + +func (svc *Service) GetPermission(query bson.M, opts *mongo.FindOptions) (res *models2.Permission, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdPermission).Get(query, opts) + return convertTypePermission(d, err) +} + +func (svc *Service) GetPermissionList(query bson.M, opts *mongo.FindOptions) (res []models2.Permission, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdPermission).GetList(query, opts) + if err != nil { + return nil, err + } + for _, doc := range l.GetModels() { + d := doc.(*models2.Permission) + res = append(res, *d) + } + return res, nil +} + +func (svc *Service) GetPermissionByKey(key string, opts *mongo.FindOptions) (res *models2.Permission, err error) { + query := bson.M{"key": key} + return svc.GetPermission(query, opts) +} diff --git a/core/models/service/project_service.go b/core/models/service/project_service.go new file mode 100644 index 000000000..39af31683 --- /dev/null +++ b/core/models/service/project_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeProject(d interface{}, err error) (res *models2.Project, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Project) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetProjectById(id primitive.ObjectID) (res *models2.Project, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdProject).GetById(id) + return convertTypeProject(d, err) +} + +func (svc *Service) GetProject(query bson.M, opts *mongo.FindOptions) (res *models2.Project, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdProject).Get(query, opts) + return convertTypeProject(d, err) +} + +func (svc *Service) GetProjectList(query bson.M, opts *mongo.FindOptions) (res []models2.Project, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdProject).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.Project) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/role_permission_service.go b/core/models/service/role_permission_service.go new file mode 100644 index 000000000..4d5f315eb --- /dev/null +++ b/core/models/service/role_permission_service.go @@ -0,0 +1,51 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeRolePermission(d interface{}, err error) (res *models2.RolePermission, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.RolePermission) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetRolePermissionById(id primitive.ObjectID) (res *models2.RolePermission, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdRolePermission).GetById(id) + return convertTypeRolePermission(d, err) +} + +func (svc *Service) GetRolePermission(query bson.M, opts *mongo.FindOptions) (res *models2.RolePermission, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdRolePermission).Get(query, opts) + return convertTypeRolePermission(d, err) +} + +func (svc *Service) GetRolePermissionList(query bson.M, opts *mongo.FindOptions) (res []models2.RolePermission, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdRolePermission).GetList(query, opts) + if err != nil { + return nil, err + } + for _, doc := range l.GetModels() { + d := doc.(*models2.RolePermission) + res = append(res, *d) + } + return res, nil +} + +func (svc *Service) GetRolePermissionListByRoleId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models2.RolePermission, err error) { + return svc.GetRolePermissionList(bson.M{"role_id": id}, opts) +} + +func (svc *Service) GetRolePermissionListByPermissionId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models2.RolePermission, err error) { + return svc.GetRolePermissionList(bson.M{"permission_id": id}, opts) +} diff --git a/core/models/service/role_service.go b/core/models/service/role_service.go new file mode 100644 index 000000000..11f960354 --- /dev/null +++ b/core/models/service/role_service.go @@ -0,0 +1,53 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeRole(d interface{}, err error) (res *models2.Role, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Role) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetRoleById(id primitive.ObjectID) (res *models2.Role, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdRole).GetById(id) + return convertTypeRole(d, err) +} + +func (svc *Service) GetRole(query bson.M, opts *mongo.FindOptions) (res *models2.Role, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdRole).Get(query, opts) + return convertTypeRole(d, err) +} + +func (svc *Service) GetRoleList(query bson.M, opts *mongo.FindOptions) (res []models2.Role, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdRole).GetList(query, opts) + if err != nil { + return nil, err + } + for _, doc := range l.GetModels() { + d := doc.(*models2.Role) + res = append(res, *d) + } + return res, nil +} + +func (svc *Service) GetRoleByName(name string, opts *mongo.FindOptions) (res *models2.Role, err error) { + query := bson.M{"name": name} + return svc.GetRole(query, opts) +} + +func (svc *Service) GetRoleByKey(key string, opts *mongo.FindOptions) (res *models2.Role, err error) { + query := bson.M{"key": key} + return svc.GetRole(query, opts) +} diff --git a/core/models/service/schedule_service.go b/core/models/service/schedule_service.go new file mode 100644 index 000000000..195fb2761 --- /dev/null +++ b/core/models/service/schedule_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeSchedule(d interface{}, err error) (res *models2.Schedule, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Schedule) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetScheduleById(id primitive.ObjectID) (res *models2.Schedule, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdSchedule).GetById(id) + return convertTypeSchedule(d, err) +} + +func (svc *Service) GetSchedule(query bson.M, opts *mongo.FindOptions) (res *models2.Schedule, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdSchedule).Get(query, opts) + return convertTypeSchedule(d, err) +} + +func (svc *Service) GetScheduleList(query bson.M, opts *mongo.FindOptions) (res []models2.Schedule, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdSchedule).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.Schedule) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/service.go b/core/models/service/service.go new file mode 100644 index 000000000..e0948039e --- /dev/null +++ b/core/models/service/service.go @@ -0,0 +1,62 @@ +package service + +import ( + "context" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/color" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" +) + +type Service struct { + env string + colorSvc interfaces.ColorService +} + +func (svc *Service) DropAll() (err error) { + db := mongo.GetMongoDb("") + colNames, err := db.ListCollectionNames(context.Background(), bson.M{}) + if err != nil { + if err == mongo2.ErrNoDocuments { + return nil + } + return err + } + for _, colName := range colNames { + col := db.Collection(colName) + if err := col.Drop(context.Background()); err != nil { + return err + } + } + return nil +} + +func (svc *Service) GetBaseService(id interfaces.ModelId) (svc2 interfaces.ModelBaseService) { + return GetBaseService(id) +} + +func NewService() (svc2 ModelService, err error) { + // service + svc := &Service{} + + svc.colorSvc, err = color.NewService() + if err != nil { + return nil, err + } + + return svc, nil +} + +var modelSvc ModelService + +func GetService() (svc ModelService, err error) { + if modelSvc != nil { + return modelSvc, nil + } + modelSvc, err = NewService() + if err != nil { + return nil, err + } + return modelSvc, nil +} diff --git a/core/models/service/setting_service.go b/core/models/service/setting_service.go new file mode 100644 index 000000000..3b37f4de6 --- /dev/null +++ b/core/models/service/setting_service.go @@ -0,0 +1,45 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeSetting(d interface{}, err error) (res *models2.Setting, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Setting) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetSettingById(id primitive.ObjectID) (res *models2.Setting, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdSetting).GetById(id) + return convertTypeSetting(d, err) +} + +func (svc *Service) GetSetting(query bson.M, opts *mongo.FindOptions) (res *models2.Setting, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdSetting).Get(query, opts) + return convertTypeSetting(d, err) +} + +func (svc *Service) GetSettingList(query bson.M, opts *mongo.FindOptions) (res []models2.Setting, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdSetting).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.Setting) + res = append(res, *d) + } + return res, nil +} + +func (svc *Service) GetSettingByKey(key string, opts *mongo.FindOptions) (res *models2.Setting, err error) { + query := bson.M{"key": key} + return svc.GetSetting(query, opts) +} diff --git a/core/models/service/spider_service.go b/core/models/service/spider_service.go new file mode 100644 index 000000000..16326d6f5 --- /dev/null +++ b/core/models/service/spider_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeSpider(d interface{}, err error) (res *models2.Spider, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Spider) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetSpiderById(id primitive.ObjectID) (res *models2.Spider, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdSpider).GetById(id) + return convertTypeSpider(d, err) +} + +func (svc *Service) GetSpider(query bson.M, opts *mongo.FindOptions) (res *models2.Spider, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdSpider).Get(query, opts) + return convertTypeSpider(d, err) +} + +func (svc *Service) GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []models2.Spider, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdSpider).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.Spider) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/spider_stat_service.go b/core/models/service/spider_stat_service.go new file mode 100644 index 000000000..64cbc45f1 --- /dev/null +++ b/core/models/service/spider_stat_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeSpiderStat(d interface{}, err error) (res *models2.SpiderStat, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.SpiderStat) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetSpiderStatById(id primitive.ObjectID) (res *models2.SpiderStat, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdSpiderStat).GetById(id) + return convertTypeSpiderStat(d, err) +} + +func (svc *Service) GetSpiderStat(query bson.M, opts *mongo.FindOptions) (res *models2.SpiderStat, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdSpiderStat).Get(query, opts) + return convertTypeSpiderStat(d, err) +} + +func (svc *Service) GetSpiderStatList(query bson.M, opts *mongo.FindOptions) (res []models2.SpiderStat, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdSpiderStat).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.SpiderStat) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/tag_service.go b/core/models/service/tag_service.go new file mode 100644 index 000000000..4a8bea506 --- /dev/null +++ b/core/models/service/tag_service.go @@ -0,0 +1,118 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" +) + +func convertTypeTag(d interface{}, err error) (res *models2.Tag, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Tag) + if !ok { + return nil, trace.TraceError(errors.ErrorModelInvalidType) + } + return res, nil +} + +func (svc *Service) GetTagById(id primitive.ObjectID) (res *models2.Tag, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdTag).GetById(id) + return convertTypeTag(d, err) +} + +func (svc *Service) GetTag(query bson.M, opts *mongo.FindOptions) (res *models2.Tag, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdTag).Get(query, opts) + return convertTypeTag(d, err) +} + +func (svc *Service) GetTagList(query bson.M, opts *mongo.FindOptions) (res []models2.Tag, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdTag).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.Tag) + res = append(res, *d) + } + return res, nil +} + +func (svc *Service) GetTagIds(colName string, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error) { + // iterate tag names + for _, tag := range tags { + // count of tags with the name + tagDb, err := svc.GetTag(bson.M{"name": tag.GetName(), "col": colName}, nil) + if err == nil { + // tag exists + tag = tagDb + } else if err == mongo2.ErrNoDocuments { + // add new tag if not exists + colorHex := tag.GetColor() + if colorHex == "" { + color, _ := svc.colorSvc.GetRandom() + colorHex = color.GetHex() + } + tag = &models2.Tag{ + Id: primitive.NewObjectID(), + Name: tag.GetName(), + Color: colorHex, + Col: colName, + } + if err := delegate.NewModelDelegate(tag).Add(); err != nil { + return tagIds, trace.TraceError(err) + } + } + + // add to tag ids + tagIds = append(tagIds, tag.GetId()) + } + + return tagIds, nil +} + +func (svc *Service) UpdateTagsById(colName string, id primitive.ObjectID, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error) { + // get tag ids to update + tagIds, err = svc.GetTagIds(colName, tags) + if err != nil { + return tagIds, trace.TraceError(err) + } + + // update in db + a, err := svc.GetArtifactById(id) + if err != nil { + return tagIds, trace.TraceError(err) + } + a.TagIds = tagIds + if err := mongo.GetMongoCol(interfaces.ModelColNameArtifact).ReplaceId(id, a); err != nil { + return tagIds, err + } + return tagIds, nil +} + +func (svc *Service) UpdateTags(colName string, query bson.M, tags []interfaces.Tag) (tagIds []primitive.ObjectID, err error) { + // tag ids to update + tagIds, err = svc.GetTagIds(colName, tags) + if err != nil { + return tagIds, trace.TraceError(err) + } + + // update + update := bson.M{ + "_tid": tagIds, + } + + // fields + fields := []string{"_tid"} + + // update in db + if err := svc.GetBaseService(interfaces.ModelIdTag).Update(query, update, fields); err != nil { + return tagIds, trace.TraceError(err) + } + + return tagIds, nil +} diff --git a/core/models/service/tag_service_legacy.go b/core/models/service/tag_service_legacy.go new file mode 100644 index 000000000..fed06deb1 --- /dev/null +++ b/core/models/service/tag_service_legacy.go @@ -0,0 +1,118 @@ +package service + +// +//import ( +// "github.com/crawlab-team/crawlab/core/interfaces" +// "github.com/crawlab-team/crawlab-db/mongo" +// "go.mongodb.org/mongo-driver/bson" +// "go.mongodb.org/mongo-driver/bson/primitive" +// mongo2 "go.mongodb.org/mongo-driver/mongo" +//) +// +//type TagServiceInterface interface { +// getTagIds(colName string, tags []Tag) (tagIds []primitive.ObjectID, err error) +// GetModelById(id primitive.ObjectID) (res Tag, err error) +// GetModel(query bson.M, opts *mongo.FindOptions) (res Tag, err error) +// GetModelList(query bson.M, opts *mongo.FindOptions) (res []Tag, err error) +// UpdateTagsById(colName string, id primitive.ObjectID, tags []Tag) (tagIds []primitive.ObjectID, err error) +// UpdateTags(colName string, query bson.M, tags []Tag) (tagIds []primitive.ObjectID, err error) +//} +// +//type tagService struct { +// *baseService +//} +// +//func (svc *tagService) getTagIds(colName string, tags []Tag) (tagIds []primitive.ObjectID, err error) { +// // iterate tag names +// for _, tag := range tags { +// // count of tags with the name +// tagDb, err := MustGetRootService().GetTag(bson.M{"name": tag.Name, "col": colName}, nil) +// if err == nil { +// // tag exists +// tag = tagDb +// } else if err == mongo2.ErrNoDocuments { +// // add new tag if not exists +// colorHex := tag.Color +// if colorHex == "" { +// color, _ := ColorService.GetRandom() +// colorHex = color.Hex +// } +// tag = Tag{ +// Name: tag.Name, +// Color: colorHex, +// Col: colName, +// } +// if err := tag.Add(); err != nil { +// return tagIds, err +// } +// } +// +// // add to tag ids +// tagIds = append(tagIds, tag.Id) +// } +// +// return tagIds, nil +//} +// +//func (svc *tagService) GetModelById(id primitive.ObjectID) (res Tag, err error) { +// err = svc.findId(id).One(&res) +// return res, err +//} +// +//func (svc *tagService) GetModel(query bson.M, opts *mongo.FindOptions) (res Tag, err error) { +// err = svc.find(query, opts).One(&res) +// return res, err +//} +// +//func (svc *tagService) GetModelList(query bson.M, opts *mongo.FindOptions) (res []Tag, err error) { +// err = svc.find(query, opts).All(&res) +// return res, err +//} +// +//func (svc *tagService) UpdateTagsById(colName string, id primitive.ObjectID, tags []Tag) (tagIds []primitive.ObjectID, err error) { +// // get tag ids to update +// tagIds, err = svc.getTagIds(colName, tags) +// if err != nil { +// return tagIds, err +// } +// +// // update in db +// a, err := MustGetRootService().GetArtifactById(id) +// if err != nil { +// return tagIds, err +// } +// a.TagIds = tagIds +// if err := mongo.GetMongoCol(interfaces.ModelColNameArtifact).ReplaceId(id, a); err != nil { +// return tagIds, err +// } +// return tagIds, nil +//} +// +//func (svc *tagService) UpdateTags(colName string, query bson.M, tags []Tag) (tagIds []primitive.ObjectID, err error) { +// // tag ids to update +// tagIds, err = svc.getTagIds(colName, tags) +// if err != nil { +// return tagIds, err +// } +// +// // update +// update := bson.M{ +// "_tid": tagIds, +// } +// +// // fields +// fields := []string{"_tid"} +// +// // update in db +// if err := ArtifactService.Update(query, update, fields); err != nil { +// return tagIds, err +// } +// +// return tagIds, nil +//} +// +//func NewTagService() (svc *tagService) { +// return &tagService{svc.GetBaseService(interfaces.ModelIdTag)} +//} +// +//var TagService *tagService diff --git a/core/models/service/task_queue_service.go b/core/models/service/task_queue_service.go new file mode 100644 index 000000000..fcbc2362f --- /dev/null +++ b/core/models/service/task_queue_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeTaskQueueItem(d interface{}, err error) (res *models2.TaskQueueItem, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.TaskQueueItem) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetTaskQueueItemById(id primitive.ObjectID) (res *models2.TaskQueueItem, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdTaskQueue).GetById(id) + return convertTypeTaskQueueItem(d, err) +} + +func (svc *Service) GetTaskQueueItem(query bson.M, opts *mongo.FindOptions) (res *models2.TaskQueueItem, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdTaskQueue).Get(query, opts) + return convertTypeTaskQueueItem(d, err) +} + +func (svc *Service) GetTaskQueueItemList(query bson.M, opts *mongo.FindOptions) (res []models2.TaskQueueItem, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdTaskQueue).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.TaskQueueItem) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/task_service.go b/core/models/service/task_service.go new file mode 100644 index 000000000..99a9ae838 --- /dev/null +++ b/core/models/service/task_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeTask(d interface{}, err error) (res *models2.Task, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Task) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetTaskById(id primitive.ObjectID) (res *models2.Task, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdTask).GetById(id) + return convertTypeTask(d, err) +} + +func (svc *Service) GetTask(query bson.M, opts *mongo.FindOptions) (res *models2.Task, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdTask).Get(query, opts) + return convertTypeTask(d, err) +} + +func (svc *Service) GetTaskList(query bson.M, opts *mongo.FindOptions) (res []models2.Task, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdTask).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.Task) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/task_stat_service.go b/core/models/service/task_stat_service.go new file mode 100644 index 000000000..1306569e0 --- /dev/null +++ b/core/models/service/task_stat_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeTaskStat(d interface{}, err error) (res *models2.TaskStat, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.TaskStat) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetTaskStatById(id primitive.ObjectID) (res *models2.TaskStat, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdTaskStat).GetById(id) + return convertTypeTaskStat(d, err) +} + +func (svc *Service) GetTaskStat(query bson.M, opts *mongo.FindOptions) (res *models2.TaskStat, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdTaskStat).Get(query, opts) + return convertTypeTaskStat(d, err) +} + +func (svc *Service) GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []models2.TaskStat, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdTaskStat).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.TaskStat) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/token_service.go b/core/models/service/token_service.go new file mode 100644 index 000000000..c802f9aa5 --- /dev/null +++ b/core/models/service/token_service.go @@ -0,0 +1,40 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeToken(d interface{}, err error) (res *models2.Token, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Token) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetTokenById(id primitive.ObjectID) (res *models2.Token, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdToken).GetById(id) + return convertTypeToken(d, err) +} + +func (svc *Service) GetToken(query bson.M, opts *mongo.FindOptions) (res *models2.Token, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdToken).Get(query, opts) + return convertTypeToken(d, err) +} + +func (svc *Service) GetTokenList(query bson.M, opts *mongo.FindOptions) (res []models2.Token, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdToken).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.Token) + res = append(res, *d) + } + return res, nil +} diff --git a/core/models/service/user_role_service.go b/core/models/service/user_role_service.go new file mode 100644 index 000000000..80a021fa7 --- /dev/null +++ b/core/models/service/user_role_service.go @@ -0,0 +1,51 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeUserRole(d interface{}, err error) (res *models2.UserRole, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.UserRole) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetUserRoleById(id primitive.ObjectID) (res *models2.UserRole, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdUserRole).GetById(id) + return convertTypeUserRole(d, err) +} + +func (svc *Service) GetUserRole(query bson.M, opts *mongo.FindOptions) (res *models2.UserRole, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdUserRole).Get(query, opts) + return convertTypeUserRole(d, err) +} + +func (svc *Service) GetUserRoleList(query bson.M, opts *mongo.FindOptions) (res []models2.UserRole, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdUserRole).GetList(query, opts) + if err != nil { + return nil, err + } + for _, doc := range l.GetModels() { + d := doc.(*models2.UserRole) + res = append(res, *d) + } + return res, nil +} + +func (svc *Service) GetUserRoleListByUserId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models2.UserRole, err error) { + return svc.GetUserRoleList(bson.M{"user_id": id}, opts) +} + +func (svc *Service) GetUserRoleListByRoleId(id primitive.ObjectID, opts *mongo.FindOptions) (res []models2.UserRole, err error) { + return svc.GetUserRoleList(bson.M{"role_id": id}, opts) +} diff --git a/core/models/service/user_service.go b/core/models/service/user_service.go new file mode 100644 index 000000000..a64f46bcc --- /dev/null +++ b/core/models/service/user_service.go @@ -0,0 +1,58 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeUser(d interface{}, err error) (res *models2.User, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.User) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetUserById(id primitive.ObjectID) (res *models2.User, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdUser).GetById(id) + return convertTypeUser(d, err) +} + +func (svc *Service) GetUser(query bson.M, opts *mongo.FindOptions) (res *models2.User, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdUser).Get(query, opts) + return convertTypeUser(d, err) +} + +func (svc *Service) GetUserList(query bson.M, opts *mongo.FindOptions) (res []models2.User, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdUser).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.User) + res = append(res, *d) + } + return res, nil +} + +func (svc *Service) GetUserByUsername(username string, opts *mongo.FindOptions) (res *models2.User, err error) { + query := bson.M{"username": username} + return svc.GetUser(query, opts) +} + +func (svc *Service) GetUserByUsernameWithPassword(username string, opts *mongo.FindOptions) (res *models2.User, err error) { + u, err := svc.GetUserByUsername(username, opts) + if err != nil { + return nil, err + } + p, err := svc.GetPasswordById(u.Id) + if err != nil { + return nil, err + } + u.Password = p.Password + return u, nil +} diff --git a/core/models/service/variable_service.go b/core/models/service/variable_service.go new file mode 100644 index 000000000..ce63d2e68 --- /dev/null +++ b/core/models/service/variable_service.go @@ -0,0 +1,45 @@ +package service + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + models2 "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func convertTypeVariable(d interface{}, err error) (res *models2.Variable, err2 error) { + if err != nil { + return nil, err + } + res, ok := d.(*models2.Variable) + if !ok { + return nil, errors.ErrorModelInvalidType + } + return res, nil +} + +func (svc *Service) GetVariableById(id primitive.ObjectID) (res *models2.Variable, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdVariable).GetById(id) + return convertTypeVariable(d, err) +} + +func (svc *Service) GetVariable(query bson.M, opts *mongo.FindOptions) (res *models2.Variable, err error) { + d, err := svc.GetBaseService(interfaces.ModelIdVariable).Get(query, opts) + return convertTypeVariable(d, err) +} + +func (svc *Service) GetVariableList(query bson.M, opts *mongo.FindOptions) (res []models2.Variable, err error) { + l, err := svc.GetBaseService(interfaces.ModelIdVariable).GetList(query, opts) + for _, doc := range l.GetModels() { + d := doc.(*models2.Variable) + res = append(res, *d) + } + return res, nil +} + +func (svc *Service) GetVariableByKey(key string, opts *mongo.FindOptions) (res *models2.Variable, err error) { + query := bson.M{"key": key} + return svc.GetVariable(query, opts) +} diff --git a/core/node/config/config.go b/core/node/config/config.go new file mode 100644 index 000000000..a869e7e6f --- /dev/null +++ b/core/node/config/config.go @@ -0,0 +1,68 @@ +package config + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/spf13/viper" +) + +type Config entity.NodeInfo + +type Options struct { + Key string + Name string + IsMaster bool + AuthKey string + MaxRunners int +} + +var DefaultMaxRunner = 8 + +var DefaultConfigOptions = &Options{ + Key: utils.NewUUIDString(), + IsMaster: utils.IsMaster(), + AuthKey: constants.DefaultGrpcAuthKey, + MaxRunners: 0, +} + +func NewConfig(opts *Options) (cfg *Config) { + if opts == nil { + opts = DefaultConfigOptions + } + if opts.Key == "" { + if viper.GetString("node.key") != "" { + opts.Key = viper.GetString("node.key") + } else { + opts.Key = utils.NewUUIDString() + } + } + if opts.Name == "" { + if viper.GetString("node.name") != "" { + opts.Name = viper.GetString("node.name") + } else { + opts.Name = opts.Key + } + } + if opts.AuthKey == "" { + if viper.GetString("grpc.authKey") != "" { + opts.AuthKey = viper.GetString("grpc.authKey") + } else { + opts.AuthKey = constants.DefaultGrpcAuthKey + } + } + if opts.MaxRunners == 0 { + if viper.GetInt("task.handler.maxRunners") != 0 { + opts.MaxRunners = viper.GetInt("task.handler.maxRunners") + } else { + opts.MaxRunners = DefaultMaxRunner + } + } + return &Config{ + Key: opts.Key, + Name: opts.Name, + IsMaster: opts.IsMaster, + AuthKey: opts.AuthKey, + MaxRunners: opts.MaxRunners, + } +} diff --git a/core/node/config/config_service.go b/core/node/config/config_service.go new file mode 100644 index 000000000..03b47ba9e --- /dev/null +++ b/core/node/config/config_service.go @@ -0,0 +1,130 @@ +package config + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "os" + "path" +) + +type Service struct { + cfg *Config + path string +} + +func (svc *Service) Init() (err error) { + // check config directory path + configDirPath := path.Dir(svc.path) + if !utils.Exists(configDirPath) { + if err := os.MkdirAll(configDirPath, os.FileMode(0766)); err != nil { + return trace.TraceError(err) + } + } + + if !utils.Exists(svc.path) { + // not exists, set to default config + // and create a config file for persistence + svc.cfg = NewConfig(nil) + data, err := json.Marshal(svc.cfg) + if err != nil { + return trace.TraceError(err) + } + if err := os.WriteFile(svc.path, data, os.FileMode(0766)); err != nil { + return trace.TraceError(err) + } + } else { + // exists, read and set to config + data, err := os.ReadFile(svc.path) + if err != nil { + return trace.TraceError(err) + } + if err := json.Unmarshal(data, svc.cfg); err != nil { + return trace.TraceError(err) + } + } + + return nil +} + +func (svc *Service) Reload() (err error) { + return svc.Init() +} + +func (svc *Service) GetBasicNodeInfo() (res interfaces.Entity) { + return &entity.NodeInfo{ + Key: svc.GetNodeKey(), + Name: svc.GetNodeName(), + IsMaster: svc.IsMaster(), + AuthKey: svc.GetAuthKey(), + MaxRunners: svc.GetMaxRunners(), + } +} + +func (svc *Service) GetNodeKey() (res string) { + return svc.cfg.Key +} + +func (svc *Service) GetNodeName() (res string) { + return svc.cfg.Name +} + +func (svc *Service) IsMaster() (res bool) { + return svc.cfg.IsMaster +} + +func (svc *Service) GetAuthKey() (res string) { + return svc.cfg.AuthKey +} + +func (svc *Service) GetMaxRunners() (res int) { + return svc.cfg.MaxRunners +} + +func (svc *Service) GetConfigPath() (path string) { + return svc.path +} + +func (svc *Service) SetConfigPath(path string) { + svc.path = path +} + +func NewNodeConfigService() (svc2 interfaces.NodeConfigService, err error) { + // cfg + cfg := NewConfig(nil) + + // config service + svc := &Service{ + cfg: cfg, + } + + // normalize config path + cfgPath := config.GetConfigPath() + svc.SetConfigPath(cfgPath) + + // init + if err := svc.Init(); err != nil { + return nil, err + } + + return svc, nil +} + +var _service interfaces.NodeConfigService + +func GetNodeConfigService() interfaces.NodeConfigService { + if _service != nil { + return _service + } + + var err error + _service, err = NewNodeConfigService() + if err != nil { + panic(err) + } + + return _service +} diff --git a/core/node/config/options.go b/core/node/config/options.go new file mode 100644 index 000000000..3bc297191 --- /dev/null +++ b/core/node/config/options.go @@ -0,0 +1,13 @@ +package config + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" +) + +type Option func(svc interfaces.NodeConfigService) + +func WithConfigPath(path string) Option { + return func(svc interfaces.NodeConfigService) { + svc.SetConfigPath(path) + } +} diff --git a/core/node/service/master_service.go b/core/node/service/master_service.go new file mode 100644 index 000000000..495223ba2 --- /dev/null +++ b/core/node/service/master_service.go @@ -0,0 +1,368 @@ +package service + +import ( + "github.com/apex/log" + "github.com/crawlab-team/crawlab-db/mongo" + config2 "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/grpc/server" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/common" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/core/notification" + "github.com/crawlab-team/crawlab/core/system" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "time" +) + +type MasterService struct { + // dependencies + modelSvc service.ModelService + cfgSvc interfaces.NodeConfigService + server interfaces.GrpcServer + schedulerSvc interfaces.TaskSchedulerService + handlerSvc interfaces.TaskHandlerService + scheduleSvc interfaces.ScheduleService + notificationSvc *notification.Service + spiderAdminSvc interfaces.SpiderAdminService + systemSvc *system.Service + + // settings + cfgPath string + address interfaces.Address + monitorInterval time.Duration + stopOnError bool +} + +func (svc *MasterService) Init() (err error) { + // do nothing + return nil +} + +func (svc *MasterService) Start() { + // create indexes + common.CreateIndexes() + + // start grpc server + if err := svc.server.Start(); err != nil { + panic(err) + } + + // register to db + if err := svc.Register(); err != nil { + panic(err) + } + + // start monitoring worker nodes + go svc.Monitor() + + // start task handler + go svc.handlerSvc.Start() + + // start task scheduler + go svc.schedulerSvc.Start() + + // start schedule service + go svc.scheduleSvc.Start() + + // start notification service + go svc.notificationSvc.Start() + + // start spider admin service + go svc.spiderAdminSvc.Start() + + // wait for quit signal + svc.Wait() + + // stop + svc.Stop() +} + +func (svc *MasterService) Wait() { + utils.DefaultWait() +} + +func (svc *MasterService) Stop() { + _ = svc.server.Stop() + log.Infof("master[%s] service has stopped", svc.GetConfigService().GetNodeKey()) +} + +func (svc *MasterService) Monitor() { + log.Infof("master[%s] monitoring started", svc.GetConfigService().GetNodeKey()) + for { + if err := svc.monitor(); err != nil { + trace.PrintError(err) + if svc.stopOnError { + log.Errorf("master[%s] monitor error, now stopping...", svc.GetConfigService().GetNodeKey()) + svc.Stop() + return + } + } + + time.Sleep(svc.monitorInterval) + } +} + +func (svc *MasterService) GetConfigService() (cfgSvc interfaces.NodeConfigService) { + return svc.cfgSvc +} + +func (svc *MasterService) GetConfigPath() (path string) { + return svc.cfgPath +} + +func (svc *MasterService) SetConfigPath(path string) { + svc.cfgPath = path +} + +func (svc *MasterService) GetAddress() (address interfaces.Address) { + return svc.address +} + +func (svc *MasterService) SetAddress(address interfaces.Address) { + svc.address = address +} + +func (svc *MasterService) SetMonitorInterval(duration time.Duration) { + svc.monitorInterval = duration +} + +func (svc *MasterService) Register() (err error) { + nodeKey := svc.GetConfigService().GetNodeKey() + nodeName := svc.GetConfigService().GetNodeName() + node, err := svc.modelSvc.GetNodeByKey(nodeKey, nil) + if err != nil && err.Error() == mongo2.ErrNoDocuments.Error() { + // not exists + log.Infof("master[%s] does not exist in db", nodeKey) + node := &models.Node{ + Key: nodeKey, + Name: nodeName, + MaxRunners: config.DefaultConfigOptions.MaxRunners, + IsMaster: true, + Status: constants.NodeStatusOnline, + Enabled: true, + Active: true, + ActiveTs: time.Now(), + } + if viper.GetInt("task.handler.maxRunners") > 0 { + node.MaxRunners = viper.GetInt("task.handler.maxRunners") + } + nodeD := delegate.NewModelNodeDelegate(node) + if err := nodeD.Add(); err != nil { + return err + } + log.Infof("added master[%s] in db. id: %s", nodeKey, nodeD.GetModel().GetId().Hex()) + return nil + } else if err == nil { + // exists + log.Infof("master[%s] exists in db", nodeKey) + nodeD := delegate.NewModelNodeDelegate(node) + if err := nodeD.UpdateStatusOnline(); err != nil { + return err + } + log.Infof("updated master[%s] in db. id: %s", nodeKey, nodeD.GetModel().GetId().Hex()) + return nil + } else { + // error + return err + } +} + +func (svc *MasterService) StopOnError() { + svc.stopOnError = true +} + +func (svc *MasterService) GetServer() (svr interfaces.GrpcServer) { + return svc.server +} + +func (svc *MasterService) monitor() (err error) { + // update master node status in db + if err := svc.updateMasterNodeStatus(); err != nil { + if err.Error() == mongo2.ErrNoDocuments.Error() { + return nil + } + return err + } + + // all worker nodes + nodes, err := svc.getAllWorkerNodes() + if err != nil { + return err + } + + // error flag + isErr := false + + // iterate all nodes + for _, n := range nodes { + // subscribe + if err := svc.subscribeNode(&n); err != nil { + isErr = true + continue + } + + // ping client + if err := svc.pingNodeClient(&n); err != nil { + isErr = true + continue + } + + // update node available runners + if err := svc.updateNodeAvailableRunners(&n); err != nil { + isErr = true + continue + } + } + + if isErr { + return trace.TraceError(errors.ErrorNodeMonitorError) + } + + return nil +} + +func (svc *MasterService) getAllWorkerNodes() (nodes []models.Node, err error) { + query := bson.M{ + "key": bson.M{"$ne": svc.cfgSvc.GetNodeKey()}, // not self + "active": true, // active + } + nodes, err = svc.modelSvc.GetNodeList(query, nil) + if err != nil { + if err == mongo2.ErrNoDocuments { + return nil, nil + } + return nil, trace.TraceError(err) + } + return nodes, nil +} + +func (svc *MasterService) updateMasterNodeStatus() (err error) { + nodeKey := svc.GetConfigService().GetNodeKey() + node, err := svc.modelSvc.GetNodeByKey(nodeKey, nil) + if err != nil { + return err + } + nodeD := delegate.NewModelNodeDelegate(node) + return nodeD.UpdateStatusOnline() +} + +func (svc *MasterService) setWorkerNodeOffline(n interfaces.Node) (err error) { + return delegate.NewModelNodeDelegate(n).UpdateStatusOffline() +} + +func (svc *MasterService) subscribeNode(n interfaces.Node) (err error) { + _, err = svc.server.GetSubscribe("node:" + n.GetKey()) + if err != nil { + log.Errorf("cannot subscribe worker node[%s]: %v", n.GetKey(), err) + if err := svc.setWorkerNodeOffline(n); err != nil { + return trace.TraceError(err) + } + return trace.TraceError(err) + } + return nil +} + +func (svc *MasterService) pingNodeClient(n interfaces.Node) (err error) { + if err := svc.server.SendStreamMessage("node:"+n.GetKey(), grpc.StreamMessageCode_PING); err != nil { + log.Errorf("cannot ping worker node client[%s]: %v", n.GetKey(), err) + if err := svc.setWorkerNodeOffline(n); err != nil { + return trace.TraceError(err) + } + return trace.TraceError(err) + } + return nil +} + +func (svc *MasterService) updateNodeAvailableRunners(n interfaces.Node) (err error) { + query := bson.M{ + "node_id": n.GetId(), + "status": constants.TaskStatusRunning, + } + runningTasksCount, err := mongo.GetMongoCol(interfaces.ModelColNameTask).Count(query) + if err != nil { + return trace.TraceError(err) + } + n.SetAvailableRunners(n.GetMaxRunners() - runningTasksCount) + return delegate.NewModelDelegate(n).Save() +} + +func NewMasterService(opts ...Option) (res interfaces.NodeMasterService, err error) { + // master service + svc := &MasterService{ + cfgPath: config2.GetConfigPath(), + monitorInterval: 15 * time.Second, + stopOnError: false, + } + + // apply options + for _, opt := range opts { + opt(svc) + } + + // server options + var serverOpts []server.Option + if svc.address != nil { + serverOpts = append(serverOpts, server.WithAddress(svc.address)) + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + cfgSvc interfaces.NodeConfigService, + modelSvc service.ModelService, + server interfaces.GrpcServer, + schedulerSvc interfaces.TaskSchedulerService, + handlerSvc interfaces.TaskHandlerService, + scheduleSvc interfaces.ScheduleService, + spiderAdminSvc interfaces.SpiderAdminService, + ) { + svc.cfgSvc = cfgSvc + svc.modelSvc = modelSvc + svc.server = server + svc.schedulerSvc = schedulerSvc + svc.handlerSvc = handlerSvc + svc.scheduleSvc = scheduleSvc + svc.spiderAdminSvc = spiderAdminSvc + }); err != nil { + return nil, err + } + + // notification service + svc.notificationSvc = notification.GetService() + + // system service + svc.systemSvc = system.GetService() + + // init + if err := svc.Init(); err != nil { + return nil, err + } + + return svc, nil +} + +func ProvideMasterService(path string, opts ...Option) func() (interfaces.NodeMasterService, error) { + // path + if path == "" || path == config2.GetConfigPath() { + if viper.GetString("config.path") != "" { + path = viper.GetString("config.path") + } else { + path = config2.GetConfigPath() + } + } + opts = append(opts, WithConfigPath(path)) + + return func() (interfaces.NodeMasterService, error) { + return NewMasterService(opts...) + } +} diff --git a/core/node/service/master_service_v2.go b/core/node/service/master_service_v2.go new file mode 100644 index 000000000..f6da2f533 --- /dev/null +++ b/core/node/service/master_service_v2.go @@ -0,0 +1,388 @@ +package service + +import ( + "errors" + "github.com/apex/log" + "github.com/cenkalti/backoff/v4" + config2 "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/grpc/server" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/common" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/core/notification" + "github.com/crawlab-team/crawlab/core/schedule" + "github.com/crawlab-team/crawlab/core/spider/admin" + "github.com/crawlab-team/crawlab/core/system" + "github.com/crawlab-team/crawlab/core/task/handler" + "github.com/crawlab-team/crawlab/core/task/scheduler" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "sync" + "time" +) + +type MasterServiceV2 struct { + // dependencies + cfgSvc interfaces.NodeConfigService + server *server.GrpcServerV2 + schedulerSvc *scheduler.ServiceV2 + handlerSvc *handler.ServiceV2 + scheduleSvc *schedule.ServiceV2 + notificationSvc *notification.Service + spiderAdminSvc *admin.ServiceV2 + systemSvc *system.Service + + // settings + cfgPath string + address interfaces.Address + monitorInterval time.Duration + stopOnError bool +} + +func (svc *MasterServiceV2) Init() (err error) { + // do nothing + return nil +} + +func (svc *MasterServiceV2) Start() { + // create indexes + common.CreateIndexes() + + // start grpc server + if err := svc.server.Start(); err != nil { + panic(err) + } + + // register to db + if err := svc.Register(); err != nil { + panic(err) + } + + // start monitoring worker nodes + go svc.Monitor() + + // start task handler + go svc.handlerSvc.Start() + + // start task scheduler + go svc.schedulerSvc.Start() + + // start schedule service + go svc.scheduleSvc.Start() + + // start notification service + go svc.notificationSvc.Start() + + // start spider admin service + go svc.spiderAdminSvc.Start() + + // wait for quit signal + svc.Wait() + + // stop + svc.Stop() +} + +func (svc *MasterServiceV2) Wait() { + utils.DefaultWait() +} + +func (svc *MasterServiceV2) Stop() { + _ = svc.server.Stop() + log.Infof("master[%s] service has stopped", svc.GetConfigService().GetNodeKey()) +} + +func (svc *MasterServiceV2) Monitor() { + log.Infof("master[%s] monitoring started", svc.GetConfigService().GetNodeKey()) + for { + if err := svc.monitor(); err != nil { + trace.PrintError(err) + if svc.stopOnError { + log.Errorf("master[%s] monitor error, now stopping...", svc.GetConfigService().GetNodeKey()) + svc.Stop() + return + } + } + + time.Sleep(svc.monitorInterval) + } +} + +func (svc *MasterServiceV2) GetConfigService() (cfgSvc interfaces.NodeConfigService) { + return svc.cfgSvc +} + +func (svc *MasterServiceV2) GetConfigPath() (path string) { + return svc.cfgPath +} + +func (svc *MasterServiceV2) SetConfigPath(path string) { + svc.cfgPath = path +} + +func (svc *MasterServiceV2) GetAddress() (address interfaces.Address) { + return svc.address +} + +func (svc *MasterServiceV2) SetAddress(address interfaces.Address) { + svc.address = address +} + +func (svc *MasterServiceV2) SetMonitorInterval(duration time.Duration) { + svc.monitorInterval = duration +} + +func (svc *MasterServiceV2) Register() (err error) { + nodeKey := svc.GetConfigService().GetNodeKey() + nodeName := svc.GetConfigService().GetNodeName() + node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + if err != nil && err.Error() == mongo2.ErrNoDocuments.Error() { + // not exists + log.Infof("master[%s] does not exist in db", nodeKey) + node := models.NodeV2{ + Key: nodeKey, + Name: nodeName, + MaxRunners: config.DefaultConfigOptions.MaxRunners, + IsMaster: true, + Status: constants.NodeStatusOnline, + Enabled: true, + Active: true, + ActiveAt: time.Now(), + } + node.SetCreated(primitive.NilObjectID) + node.SetUpdated(primitive.NilObjectID) + id, err := service.NewModelServiceV2[models.NodeV2]().InsertOne(node) + if err != nil { + return err + } + log.Infof("added master[%s] in db. id: %s", nodeKey, id.Hex()) + return nil + } else if err == nil { + // exists + log.Infof("master[%s] exists in db", nodeKey) + node.Status = constants.NodeStatusOnline + node.Active = true + node.ActiveAt = time.Now() + err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + if err != nil { + return err + } + log.Infof("updated master[%s] in db. id: %s", nodeKey, node.Id.Hex()) + return nil + } else { + // error + return err + } +} + +func (svc *MasterServiceV2) StopOnError() { + svc.stopOnError = true +} + +func (svc *MasterServiceV2) GetServer() (svr interfaces.GrpcServer) { + return svc.server +} + +func (svc *MasterServiceV2) monitor() (err error) { + // update master node status in db + if err := svc.updateMasterNodeStatus(); err != nil { + if err.Error() == mongo2.ErrNoDocuments.Error() { + return nil + } + return err + } + + // all worker nodes + workerNodes, err := svc.getAllWorkerNodes() + if err != nil { + return err + } + + // iterate all worker nodes + wg := sync.WaitGroup{} + wg.Add(len(workerNodes)) + for _, n := range workerNodes { + go func(n *models.NodeV2) { + // subscribe + ok := svc.subscribeNode(n) + if !ok { + go svc.setWorkerNodeOffline(n) + wg.Done() + return + } + + // ping client + ok = svc.pingNodeClient(n) + if !ok { + go svc.setWorkerNodeOffline(n) + wg.Done() + return + } + + // update node available runners + if err := svc.updateNodeAvailableRunners(n); err != nil { + trace.PrintError(err) + wg.Done() + return + } + + // done + wg.Done() + }(&n) + } + + wg.Wait() + + return nil +} + +func (svc *MasterServiceV2) getAllWorkerNodes() (nodes []models.NodeV2, err error) { + query := bson.M{ + "key": bson.M{"$ne": svc.cfgSvc.GetNodeKey()}, // not self + "active": true, // active + } + nodes, err = service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + if err != nil { + if errors.Is(err, mongo2.ErrNoDocuments) { + return nil, nil + } + return nil, trace.TraceError(err) + } + return nodes, nil +} + +func (svc *MasterServiceV2) updateMasterNodeStatus() (err error) { + nodeKey := svc.GetConfigService().GetNodeKey() + node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + if err != nil { + return err + } + node.Status = constants.NodeStatusOnline + node.Active = true + node.ActiveAt = time.Now() + err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + if err != nil { + return err + } + return nil +} + +func (svc *MasterServiceV2) setWorkerNodeOffline(node *models.NodeV2) { + node.Status = constants.NodeStatusOffline + node.Active = false + err := backoff.Retry(func() error { + return service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + }, backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 3)) + if err != nil { + trace.PrintError(err) + } +} + +func (svc *MasterServiceV2) subscribeNode(n *models.NodeV2) (ok bool) { + _, err := svc.server.GetSubscribe("node:" + n.Key) + if err != nil { + log.Errorf("cannot subscribe worker node[%s]: %v", n.Key, err) + return false + } + return true +} + +func (svc *MasterServiceV2) pingNodeClient(n *models.NodeV2) (ok bool) { + if err := svc.server.SendStreamMessage("node:"+n.Key, grpc.StreamMessageCode_PING); err != nil { + log.Errorf("cannot ping worker node client[%s]: %v", n.Key, err) + return false + } + return true +} + +func (svc *MasterServiceV2) updateNodeAvailableRunners(node *models.NodeV2) (err error) { + query := bson.M{ + "node_id": node.Id, + "status": constants.TaskStatusRunning, + } + runningTasksCount, err := service.NewModelServiceV2[models.TaskV2]().Count(query) + if err != nil { + return trace.TraceError(err) + } + node.AvailableRunners = node.MaxRunners - runningTasksCount + err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + if err != nil { + return err + } + return nil +} + +func NewMasterServiceV2() (res interfaces.NodeMasterService, err error) { + // master service + svc := &MasterServiceV2{ + cfgPath: config2.GetConfigPath(), + monitorInterval: 15 * time.Second, + stopOnError: false, + } + + // server options + var serverOpts []server.Option + if svc.address != nil { + serverOpts = append(serverOpts, server.WithAddress(svc.address)) + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + cfgSvc interfaces.NodeConfigService, + ) { + svc.cfgSvc = cfgSvc + }); err != nil { + return nil, err + } + + // grpc server + svc.server, err = server.GetGrpcServerV2() + if err != nil { + return nil, err + } + + // scheduler service + svc.schedulerSvc, err = scheduler.GetTaskSchedulerServiceV2() + if err != nil { + return nil, err + } + + // handler service + svc.handlerSvc, err = handler.GetTaskHandlerServiceV2() + if err != nil { + return nil, err + } + + // schedule service + svc.scheduleSvc, err = schedule.GetScheduleServiceV2() + if err != nil { + return nil, err + } + + // notification service + svc.notificationSvc = notification.GetService() + + // spider admin service + svc.spiderAdminSvc, err = admin.GetSpiderAdminServiceV2() + if err != nil { + return nil, err + } + + // system service + svc.systemSvc = system.GetService() + + // init + if err := svc.Init(); err != nil { + return nil, err + } + + return svc, nil +} diff --git a/core/node/service/options.go b/core/node/service/options.go new file mode 100644 index 000000000..941879f51 --- /dev/null +++ b/core/node/service/options.go @@ -0,0 +1,47 @@ +package service + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type Option func(svc interfaces.NodeService) + +func WithConfigPath(path string) Option { + return func(svc interfaces.NodeService) { + svc.SetConfigPath(path) + } +} + +func WithAddress(address interfaces.Address) Option { + return func(svc interfaces.NodeService) { + svc.SetAddress(address) + } +} + +func WithMonitorInterval(duration time.Duration) Option { + return func(svc interfaces.NodeService) { + svc2, ok := svc.(interfaces.NodeMasterService) + if ok { + svc2.SetMonitorInterval(duration) + } + } +} + +func WithStopOnError() Option { + return func(svc interfaces.NodeService) { + svc2, ok := svc.(interfaces.NodeMasterService) + if ok { + svc2.StopOnError() + } + } +} + +func WithHeartbeatInterval(duration time.Duration) Option { + return func(svc interfaces.NodeService) { + svc2, ok := svc.(interfaces.NodeWorkerService) + if ok { + svc2.SetHeartbeatInterval(duration) + } + } +} diff --git a/core/node/service/worker_service.go b/core/node/service/worker_service.go new file mode 100644 index 000000000..3df9895ab --- /dev/null +++ b/core/node/service/worker_service.go @@ -0,0 +1,238 @@ +package service + +import ( + "context" + "encoding/json" + "github.com/apex/log" + config2 "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/grpc/client" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "github.com/spf13/viper" + "time" +) + +type WorkerService struct { + // dependencies + cfgSvc interfaces.NodeConfigService + client interfaces.GrpcClient + handlerSvc interfaces.TaskHandlerService + + // settings + cfgPath string + address interfaces.Address + heartbeatInterval time.Duration + + // internals + n interfaces.Node + s grpc.NodeService_SubscribeClient +} + +func (svc *WorkerService) Init() (err error) { + // do nothing + return nil +} + +func (svc *WorkerService) Start() { + // start grpc client + if err := svc.client.Start(); err != nil { + panic(err) + } + + // register to master + svc.Register() + + // start receiving stream messages + go svc.Recv() + + // start sending heartbeat to master + go svc.ReportStatus() + + // start handler + go svc.handlerSvc.Start() + + // wait for quit signal + svc.Wait() + + // stop + svc.Stop() +} + +func (svc *WorkerService) Wait() { + utils.DefaultWait() +} + +func (svc *WorkerService) Stop() { + _ = svc.client.Stop() + log.Infof("worker[%s] service has stopped", svc.cfgSvc.GetNodeKey()) +} + +func (svc *WorkerService) Register() { + ctx, cancel := svc.client.Context() + defer cancel() + req := svc.client.NewRequest(svc.GetConfigService().GetBasicNodeInfo()) + res, err := svc.client.GetNodeClient().Register(ctx, req) + if err != nil { + panic(err) + } + if err := json.Unmarshal(res.Data, svc.n); err != nil { + panic(err) + } + log.Infof("worker[%s] registered to master. id: %s", svc.GetConfigService().GetNodeKey(), svc.n.GetId().Hex()) + return +} + +func (svc *WorkerService) Recv() { + msgCh := svc.client.GetMessageChannel() + for { + // return if client is closed + if svc.client.IsClosed() { + return + } + + // receive message from channel + msg := <-msgCh + + // handle message + if err := svc.handleStreamMessage(msg); err != nil { + continue + } + } +} + +func (svc *WorkerService) handleStreamMessage(msg *grpc.StreamMessage) (err error) { + log.Debugf("[WorkerService] handle msg: %v", msg) + switch msg.Code { + case grpc.StreamMessageCode_PING: + if _, err := svc.client.GetNodeClient().SendHeartbeat(context.Background(), svc.client.NewRequest(svc.cfgSvc.GetBasicNodeInfo())); err != nil { + return trace.TraceError(err) + } + case grpc.StreamMessageCode_RUN_TASK: + var t models.Task + if err := json.Unmarshal(msg.Data, &t); err != nil { + return trace.TraceError(err) + } + if err := svc.handlerSvc.Run(t.Id); err != nil { + return trace.TraceError(err) + } + case grpc.StreamMessageCode_CANCEL_TASK: + var t models.Task + if err := json.Unmarshal(msg.Data, &t); err != nil { + return trace.TraceError(err) + } + if err := svc.handlerSvc.Cancel(t.Id); err != nil { + return trace.TraceError(err) + } + } + + return nil +} + +func (svc *WorkerService) ReportStatus() { + for { + // return if client is closed + if svc.client.IsClosed() { + return + } + + // report status + svc.reportStatus() + + // sleep + time.Sleep(svc.heartbeatInterval) + } +} + +func (svc *WorkerService) GetConfigService() (cfgSvc interfaces.NodeConfigService) { + return svc.cfgSvc +} + +func (svc *WorkerService) GetConfigPath() (path string) { + return svc.cfgPath +} + +func (svc *WorkerService) SetConfigPath(path string) { + svc.cfgPath = path +} + +func (svc *WorkerService) GetAddress() (address interfaces.Address) { + return svc.address +} + +func (svc *WorkerService) SetAddress(address interfaces.Address) { + svc.address = address +} + +func (svc *WorkerService) SetHeartbeatInterval(duration time.Duration) { + svc.heartbeatInterval = duration +} + +func (svc *WorkerService) reportStatus() { + ctx, cancel := context.WithTimeout(context.Background(), svc.heartbeatInterval) + defer cancel() + _, err := svc.client.GetNodeClient().SendHeartbeat(ctx, &grpc.Request{ + NodeKey: svc.cfgSvc.GetNodeKey(), + }) + if err != nil { + trace.PrintError(err) + } +} + +func NewWorkerService(opts ...Option) (res *WorkerService, err error) { + svc := &WorkerService{ + cfgPath: config2.GetConfigPath(), + heartbeatInterval: 15 * time.Second, + n: &models.Node{}, + } + + // apply options + for _, opt := range opts { + opt(svc) + } + + // dependency options + var clientOpts []client.Option + if svc.address != nil { + clientOpts = append(clientOpts, client.WithAddress(svc.address)) + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + cfgSvc interfaces.NodeConfigService, + client interfaces.GrpcClient, + taskHandlerSvc interfaces.TaskHandlerService, + ) { + svc.cfgSvc = cfgSvc + svc.client = client + svc.handlerSvc = taskHandlerSvc + }); err != nil { + return nil, err + } + + // init + if err := svc.Init(); err != nil { + return nil, err + } + + return svc, nil +} + +func ProvideWorkerService(path string, opts ...Option) func() (interfaces.NodeWorkerService, error) { + // path + if path == "" || path == config2.GetConfigPath() { + if viper.GetString("config.path") != "" { + path = viper.GetString("config.path") + } else { + path = config2.GetConfigPath() + } + } + opts = append(opts, WithConfigPath(path)) + + return func() (interfaces.NodeWorkerService, error) { + return NewWorkerService(opts...) + } +} diff --git a/core/node/service/worker_service_v2.go b/core/node/service/worker_service_v2.go new file mode 100644 index 000000000..155c35dd4 --- /dev/null +++ b/core/node/service/worker_service_v2.go @@ -0,0 +1,225 @@ +package service + +import ( + "context" + "encoding/json" + "github.com/apex/log" + config2 "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/grpc/client" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/task/handler" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "time" +) + +type WorkerServiceV2 struct { + // dependencies + cfgSvc interfaces.NodeConfigService + client *client.GrpcClientV2 + handlerSvc *handler.ServiceV2 + + // settings + cfgPath string + address interfaces.Address + heartbeatInterval time.Duration + + // internals + n interfaces.Node + s grpc.NodeService_SubscribeClient +} + +func (svc *WorkerServiceV2) Init() (err error) { + // do nothing + return nil +} + +func (svc *WorkerServiceV2) Start() { + // start grpc client + if err := svc.client.Start(); err != nil { + panic(err) + } + + // register to master + svc.Register() + + // start receiving stream messages + go svc.Recv() + + // start sending heartbeat to master + go svc.ReportStatus() + + // start handler + go svc.handlerSvc.Start() + + // wait for quit signal + svc.Wait() + + // stop + svc.Stop() +} + +func (svc *WorkerServiceV2) Wait() { + utils.DefaultWait() +} + +func (svc *WorkerServiceV2) Stop() { + _ = svc.client.Stop() + log.Infof("worker[%s] service has stopped", svc.cfgSvc.GetNodeKey()) +} + +func (svc *WorkerServiceV2) Register() { + ctx, cancel := svc.client.Context() + defer cancel() + req := svc.client.NewRequest(svc.GetConfigService().GetBasicNodeInfo()) + res, err := svc.client.NodeClient.Register(ctx, req) + if err != nil { + panic(err) + } + if err := json.Unmarshal(res.Data, svc.n); err != nil { + panic(err) + } + log.Infof("worker[%s] registered to master. id: %s", svc.GetConfigService().GetNodeKey(), svc.n.GetId().Hex()) + return +} + +func (svc *WorkerServiceV2) Recv() { + msgCh := svc.client.GetMessageChannel() + for { + // return if client is closed + if svc.client.IsClosed() { + return + } + + // receive message from channel + msg := <-msgCh + + // handle message + if err := svc.handleStreamMessage(msg); err != nil { + continue + } + } +} + +func (svc *WorkerServiceV2) handleStreamMessage(msg *grpc.StreamMessage) (err error) { + log.Debugf("[WorkerServiceV2] handle msg: %v", msg) + switch msg.Code { + case grpc.StreamMessageCode_PING: + if _, err := svc.client.NodeClient.SendHeartbeat(context.Background(), svc.client.NewRequest(svc.cfgSvc.GetBasicNodeInfo())); err != nil { + return trace.TraceError(err) + } + case grpc.StreamMessageCode_RUN_TASK: + var t models.Task + if err := json.Unmarshal(msg.Data, &t); err != nil { + return trace.TraceError(err) + } + if err := svc.handlerSvc.Run(t.Id); err != nil { + return trace.TraceError(err) + } + case grpc.StreamMessageCode_CANCEL_TASK: + var t models.Task + if err := json.Unmarshal(msg.Data, &t); err != nil { + return trace.TraceError(err) + } + if err := svc.handlerSvc.Cancel(t.Id); err != nil { + return trace.TraceError(err) + } + } + + return nil +} + +func (svc *WorkerServiceV2) ReportStatus() { + for { + // return if client is closed + if svc.client.IsClosed() { + return + } + + // report status + svc.reportStatus() + + // sleep + time.Sleep(svc.heartbeatInterval) + } +} + +func (svc *WorkerServiceV2) GetConfigService() (cfgSvc interfaces.NodeConfigService) { + return svc.cfgSvc +} + +func (svc *WorkerServiceV2) GetConfigPath() (path string) { + return svc.cfgPath +} + +func (svc *WorkerServiceV2) SetConfigPath(path string) { + svc.cfgPath = path +} + +func (svc *WorkerServiceV2) GetAddress() (address interfaces.Address) { + return svc.address +} + +func (svc *WorkerServiceV2) SetAddress(address interfaces.Address) { + svc.address = address +} + +func (svc *WorkerServiceV2) SetHeartbeatInterval(duration time.Duration) { + svc.heartbeatInterval = duration +} + +func (svc *WorkerServiceV2) reportStatus() { + ctx, cancel := context.WithTimeout(context.Background(), svc.heartbeatInterval) + defer cancel() + _, err := svc.client.NodeClient.SendHeartbeat(ctx, &grpc.Request{ + NodeKey: svc.cfgSvc.GetNodeKey(), + }) + if err != nil { + trace.PrintError(err) + } +} + +func NewWorkerServiceV2() (res *WorkerServiceV2, err error) { + svc := &WorkerServiceV2{ + cfgPath: config2.GetConfigPath(), + heartbeatInterval: 15 * time.Second, + n: &models.Node{}, + } + + // dependency options + var clientOpts []client.Option + if svc.address != nil { + clientOpts = append(clientOpts, client.WithAddress(svc.address)) + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + cfgSvc interfaces.NodeConfigService, + ) { + svc.cfgSvc = cfgSvc + }); err != nil { + return nil, err + } + + // grpc client + svc.client, err = client.NewGrpcClientV2() + if err != nil { + return nil, err + } + + // handler service + svc.handlerSvc, err = handler.GetTaskHandlerServiceV2() + if err != nil { + return nil, err + } + + // init + if err := svc.Init(); err != nil { + return nil, err + } + + return svc, nil +} diff --git a/core/node/test/base.go b/core/node/test/base.go new file mode 100644 index 000000000..223c6a926 --- /dev/null +++ b/core/node/test/base.go @@ -0,0 +1,206 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + service2 "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/node/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/spf13/viper" + "go.uber.org/dig" + "io/ioutil" + "os" + "path" + "testing" + "time" +) + +func init() { + var err error + T, err = NewTest() + if err != nil { + panic(err) + } +} + +var T *Test + +type Test struct { + DefaultSvc interfaces.NodeMasterService + MasterSvc interfaces.NodeMasterService + WorkerSvc interfaces.NodeWorkerService + MasterSvcMonitor interfaces.NodeMasterService + WorkerSvcMonitor interfaces.NodeWorkerService + ModelSvc service2.ModelService +} + +func NewTest() (res *Test, err error) { + // test + t := &Test{} + + // recreate config directory path + _ = os.RemoveAll(viper.GetString("metadata")) + _ = os.MkdirAll(viper.GetString("metadata"), os.FileMode(0766)) + + // master config and settings + masterNodeConfigName := "config-master.json" + masterNodeConfigPath := path.Join(viper.GetString("metadata"), masterNodeConfigName) + if err := ioutil.WriteFile(masterNodeConfigPath, []byte("{\"key\":\"master\",\"is_master\":true}"), os.FileMode(0766)); err != nil { + return nil, err + } + masterHost := "0.0.0.0" + masterPort := "9667" + + // worker config and settings + workerNodeConfigName := "config-worker.json" + workerNodeConfigPath := path.Join(viper.GetString("metadata"), workerNodeConfigName) + if err = ioutil.WriteFile(workerNodeConfigPath, []byte("{\"key\":\"worker\",\"is_master\":false}"), os.FileMode(0766)); err != nil { + return nil, err + } + workerHost := "localhost" + workerPort := masterPort + + // master for monitor config and settings + masterNodeMonitorConfigName := "config-master-monitor.json" + masterNodeMonitorConfigPath := path.Join(viper.GetString("metadata"), masterNodeMonitorConfigName) + if err := ioutil.WriteFile(masterNodeMonitorConfigPath, []byte("{\"key\":\"master-monitor\",\"is_master\":true}"), os.FileMode(0766)); err != nil { + return nil, err + } + masterMonitorHost := masterHost + masterMonitorPort := "9668" + + // worker for monitor config and settings + workerNodeMonitorConfigName := "config-worker-monitor.json" + workerNodeMonitorConfigPath := path.Join(viper.GetString("metadata"), workerNodeMonitorConfigName) + if err := ioutil.WriteFile(workerNodeMonitorConfigPath, []byte("{\"key\":\"worker-monitor\",\"is_master\":false}"), os.FileMode(0766)); err != nil { + return nil, err + } + workerMonitorHost := workerHost + workerMonitorPort := masterMonitorPort + + // dependency injection + c := dig.New() + if err := c.Provide(service.ProvideMasterService( + masterNodeConfigPath, + service.WithMonitorInterval(3*time.Second), + service.WithAddress(entity.NewAddress(&entity.AddressOptions{ + Host: masterHost, + Port: masterPort, + })), + )); err != nil { + return nil, err + } + if err := c.Provide(service.ProvideWorkerService( + workerNodeConfigPath, + service.WithHeartbeatInterval(1*time.Second), + service.WithAddress(entity.NewAddress(&entity.AddressOptions{ + Host: workerHost, + Port: workerPort, + })), + )); err != nil { + return nil, err + } + if err := c.Provide(service2.NewService); err != nil { + return nil, err + } + if err := c.Invoke(func(masterSvc interfaces.NodeMasterService, workerSvc interfaces.NodeWorkerService, modelSvc service2.ModelService) { + t.MasterSvc = masterSvc + t.WorkerSvc = workerSvc + t.ModelSvc = modelSvc + }); err != nil { + return nil, err + } + + // default service + t.DefaultSvc, err = service.NewMasterService() + if err != nil { + return nil, err + } + + // master and worker for monitor + t.MasterSvcMonitor, err = service.NewMasterService( + service.WithConfigPath(masterNodeMonitorConfigPath), + service.WithAddress(entity.NewAddress(&entity.AddressOptions{ + Host: masterMonitorHost, + Port: masterMonitorPort, + })), + service.WithMonitorInterval(3*time.Second), + service.WithStopOnError(), + ) + if err != nil { + return nil, err + } + t.WorkerSvcMonitor, err = service.NewWorkerService( + service.WithConfigPath(workerNodeMonitorConfigPath), + service.WithAddress(entity.NewAddress(&entity.AddressOptions{ + Host: workerMonitorHost, + Port: workerMonitorPort, + })), + service.WithHeartbeatInterval(1*time.Second), + service.WithStopOnError(), + ) + if err != nil { + return nil, err + } + + // removed all data in db + _ = t.ModelSvc.DropAll() + + // visualize dependencies + if err := utils.VisualizeContainer(c); err != nil { + return nil, err + } + + return t, nil +} + +func (t *Test) Setup(t2 *testing.T) { + if err := t.ModelSvc.DropAll(); err != nil { + panic(err) + } + _ = os.RemoveAll(viper.GetString("metadata")) + t2.Cleanup(t.Cleanup) +} + +func (t *Test) Cleanup() { + if err := t.ModelSvc.DropAll(); err != nil { + panic(err) + } + _ = os.RemoveAll(viper.GetString("metadata")) +} + +func (t *Test) StartMasterWorker() { + startMasterWorker() +} + +func (t *Test) StopMasterWorker() { + stopMasterWorker() +} + +func startMasterWorker() { + go T.MasterSvc.Start() + time.Sleep(1 * time.Second) + go T.WorkerSvc.Start() + time.Sleep(1 * time.Second) +} + +func stopMasterWorker() { + go T.MasterSvc.Stop() + time.Sleep(1 * time.Second) + go T.WorkerSvc.Stop() + time.Sleep(1 * time.Second) +} + +func startMasterWorkerMonitor() { + go T.MasterSvcMonitor.Start() + time.Sleep(1 * time.Second) + go T.WorkerSvcMonitor.Start() + time.Sleep(1 * time.Second) +} + +func stopMasterWorkerMonitor() { + go T.MasterSvcMonitor.Stop() + time.Sleep(1 * time.Second) + go T.WorkerSvcMonitor.Stop() + time.Sleep(1 * time.Second) +} diff --git a/core/node/test/base_test.go b/core/node/test/base_test.go new file mode 100644 index 000000000..70832586c --- /dev/null +++ b/core/node/test/base_test.go @@ -0,0 +1,67 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestNodeServices_Master_Worker(t *testing.T) { + T, _ = NewTest() + T.Setup(t) + startMasterWorker() + + // validate master + masterNodeKey := T.MasterSvc.GetConfigService().GetNodeKey() + masterNode, err := T.ModelSvc.GetNodeByKey(masterNodeKey, nil) + require.Nil(t, err) + require.Equal(t, constants.NodeStatusOnline, masterNode.Status) + require.Equal(t, masterNodeKey, masterNode.Key) + require.True(t, masterNode.IsMaster) + + // validate worker + workerNodeKey := T.WorkerSvc.GetConfigService().GetNodeKey() + workerNode, err := T.ModelSvc.GetNodeByKey(workerNodeKey, nil) + require.Nil(t, err) + require.Equal(t, constants.NodeStatusOnline, workerNode.Status) + require.Equal(t, workerNodeKey, workerNode.Key) + require.False(t, workerNode.IsMaster) + + stopMasterWorker() +} + +func TestNodeServices_Default(t *testing.T) { + T, _ = NewTest() + T.Setup(t) + + go T.DefaultSvc.Start() + time.Sleep(1 * time.Second) + + // validate default + defaultNodeKey := T.DefaultSvc.GetConfigService().GetNodeKey() + defaultNode, err := T.ModelSvc.GetNodeByKey(defaultNodeKey, nil) + require.Nil(t, err) + require.Equal(t, constants.NodeStatusOnline, defaultNode.Status) + require.Equal(t, defaultNodeKey, defaultNode.Key) + require.True(t, defaultNode.IsMaster) + + T.DefaultSvc.Stop() + time.Sleep(1 * time.Second) +} + +func TestNodeServices_Monitor(t *testing.T) { + T, _ = NewTest() + T.Setup(t) + startMasterWorkerMonitor() + time.Sleep(3 * time.Second) + + // stop worker + T.WorkerSvcMonitor.Stop() + time.Sleep(5 * time.Second) + + // validate + require.True(t, T.MasterSvcMonitor.GetServer().IsStopped()) + + stopMasterWorkerMonitor() +} diff --git a/core/notification/base_test.go b/core/notification/base_test.go new file mode 100644 index 000000000..5b29810d3 --- /dev/null +++ b/core/notification/base_test.go @@ -0,0 +1,73 @@ +package notification + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/gavv/httpexpect/v2" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson/primitive" + "net/http/httptest" + "testing" +) + +func init() { + viper.Set("mongo.db", "crawlab_test") + var err error + T, err = NewTest() + if err != nil { + panic(err) + } +} + +type Test struct { + svc *Service + svr *httptest.Server + + // test data + TestNode interfaces.Node + TestSpider interfaces.Spider + TestTask interfaces.Task + TestTaskStat interfaces.TaskStat +} + +func (t *Test) Setup(t2 *testing.T) { + _ = t.svc.Start() + t2.Cleanup(t.Cleanup) +} + +func (t *Test) Cleanup() { + _ = t.svc.Stop() +} + +func (t *Test) NewExpect(t2 *testing.T) (e *httpexpect.Expect) { + e = httpexpect.New(t2, t.svr.URL) + return e +} + +var T *Test + +func NewTest() (res *Test, err error) { + // test + t := &Test{ + svc: NewService(), + } + + // test node + t.TestNode = &models.Node{Id: primitive.NewObjectID(), Name: "test-node"} + _ = delegate.NewModelDelegate(t.TestNode).Add() + + // test spider + t.TestSpider = &models.Spider{Id: primitive.NewObjectID(), Name: "test-spider"} + _ = delegate.NewModelDelegate(t.TestSpider).Add() + + // test task + t.TestTask = &models.Task{Id: primitive.NewObjectID(), SpiderId: t.TestSpider.GetId(), NodeId: t.TestNode.GetId()} + _ = delegate.NewModelDelegate(t.TestTask).Add() + + // test task stat + t.TestTaskStat = &models.TaskStat{Id: t.TestTask.GetId()} + _ = delegate.NewModelDelegate(t.TestTaskStat).Add() + + return t, nil +} diff --git a/core/notification/constants.go b/core/notification/constants.go new file mode 100644 index 000000000..d3f56b721 --- /dev/null +++ b/core/notification/constants.go @@ -0,0 +1,10 @@ +package notification + +const ( + TypeMail = "mail" + TypeMobile = "mobile" +) + +const ( + SettingsColName = "notification_settings" +) diff --git a/core/notification/mail.go b/core/notification/mail.go new file mode 100644 index 000000000..905a5446a --- /dev/null +++ b/core/notification/mail.go @@ -0,0 +1,178 @@ +package notification + +import ( + "errors" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/matcornic/hermes/v2" + "gopkg.in/gomail.v2" + "net/mail" + "runtime/debug" + "strconv" + "strings" +) + +func SendMail(m *models.NotificationSettingMail, to, cc, title, content string) error { + // theme + theme := new(MailThemeFlat) + + // hermes instance + h := hermes.Hermes{ + Theme: theme, + Product: hermes.Product{ + Logo: "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMzAwIiBoZWlnaHQ9IjMwMCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KICAgIDxnIGZpbGw9Im5vbmUiPgogICAgICAgIDxjaXJjbGUgY3g9IjE1MCIgY3k9IjE1MCIgcj0iMTMwIiBmaWxsPSJub25lIiBzdHJva2Utd2lkdGg9IjQwIiBzdHJva2U9IiM0MDllZmYiPgogICAgICAgIDwvY2lyY2xlPgogICAgICAgIDxjaXJjbGUgY3g9IjE1MCIgY3k9IjE1MCIgcj0iMTEwIiBmaWxsPSJ3aGl0ZSI+CiAgICAgICAgPC9jaXJjbGU+CiAgICAgICAgPGNpcmNsZSBjeD0iMTUwIiBjeT0iMTUwIiByPSI3MCIgZmlsbD0iIzQwOWVmZiI+CiAgICAgICAgPC9jaXJjbGU+CiAgICAgICAgPHBhdGggZD0iCiAgICAgICAgICAgIE0gMTUwLDE1MAogICAgICAgICAgICBMIDI4MCwyMjUKICAgICAgICAgICAgQSAxNTAsMTUwIDkwIDAgMCAyODAsNzUKICAgICAgICAgICAgIiBmaWxsPSIjNDA5ZWZmIj4KICAgICAgICA8L3BhdGg+CiAgICA8L2c+Cjwvc3ZnPgo=", + Name: "Crawlab", + Copyright: "© 2024 Crawlab-Team", + }, + } + + // config + port, _ := strconv.Atoi(m.Port) + password := m.Password // test password: ALWVDPRHBEXOENXD + SMTPUser := m.User + smtpConfig := smtpAuthentication{ + Server: m.Server, + Port: port, + SenderEmail: m.SenderEmail, + SenderIdentity: m.SenderIdentity, + SMTPPassword: password, + SMTPUser: SMTPUser, + } + options := sendOptions{ + To: to, + Cc: cc, + Subject: title, + } + + // add style + content += theme.GetStyle() + + // markdown + markdown := hermes.Markdown(content + GetFooter()) + + // email instance + email := hermes.Email{ + Body: hermes.Body{ + Signature: "Happy Crawling ☺", + FreeMarkdown: markdown, + }, + } + + // generate html + html, err := h.GenerateHTML(email) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // generate text + text, err := h.GeneratePlainText(email) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + // send the email + if err := send(smtpConfig, options, html, text); err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return err + } + + return nil +} + +type smtpAuthentication struct { + Server string + Port int + SenderEmail string + SenderIdentity string + SMTPUser string + SMTPPassword string +} + +// sendOptions are options for sending an email +type sendOptions struct { + To string + Subject string + Cc string +} + +// send email +func send(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, txtBody string) error { + if smtpConfig.Server == "" { + return errors.New("SMTP server config is empty") + } + + if smtpConfig.Port == 0 { + return errors.New("SMTP port config is empty") + } + + if smtpConfig.SMTPUser == "" { + return errors.New("SMTP user is empty") + } + + if smtpConfig.SenderIdentity == "" { + return errors.New("SMTP sender identity is empty") + } + + if smtpConfig.SenderEmail == "" { + return errors.New("SMTP sender email is empty") + } + + if options.To == "" { + return errors.New("no receiver emails configured") + } + + from := mail.Address{ + Name: smtpConfig.SenderIdentity, + Address: smtpConfig.SenderEmail, + } + + var toList []string + if strings.Contains(options.To, ";") { + toList = strings.Split(options.To, ";") + // trim space + for i, to := range toList { + toList[i] = strings.TrimSpace(to) + } + } else { + toList = []string{options.To} + } + + m := gomail.NewMessage() + m.SetHeader("From", from.String()) + m.SetHeader("To", getRecipientList(options.To)...) + m.SetHeader("Subject", options.Subject) + if options.Cc != "" { + m.SetHeader("Cc", getRecipientList(options.Cc)...) + } + + m.SetBody("text/plain", txtBody) + m.AddAlternative("text/html", htmlBody) + + d := gomail.NewDialer(smtpConfig.Server, smtpConfig.Port, smtpConfig.SMTPUser, smtpConfig.SMTPPassword) + + return d.DialAndSend(m) +} + +func getRecipientList(value string) (values []string) { + if strings.Contains(value, ";") { + values = strings.Split(value, ";") + // trim space + for i, v := range values { + values[i] = strings.TrimSpace(v) + } + } else { + values = []string{value} + } + return values +} + +func GetFooter() string { + return ` +[Github](https://github.com/crawlab-team/crawlab) | [Documentation](http://docs.crawlab.cn) | [Docker](https://hub.docker.com/r/tikazyq/crawlab) +` +} diff --git a/core/notification/mail_theme.go b/core/notification/mail_theme.go new file mode 100644 index 000000000..9266ddd8c --- /dev/null +++ b/core/notification/mail_theme.go @@ -0,0 +1,8 @@ +package notification + +import "github.com/matcornic/hermes/v2" + +type MailTheme interface { + hermes.Theme + GetStyle() string +} diff --git a/core/notification/mail_theme_flat.go b/core/notification/mail_theme_flat.go new file mode 100644 index 000000000..6edb480f2 --- /dev/null +++ b/core/notification/mail_theme_flat.go @@ -0,0 +1,287 @@ +package notification + +// MailThemeFlat is a theme +type MailThemeFlat struct{} + +// Name returns the name of the flat theme +func (dt *MailThemeFlat) Name() string { + return "flat" +} + +// HTMLTemplate returns a Golang template that will generate an HTML email. +func (dt *MailThemeFlat) HTMLTemplate() string { + return ` + + + + + + + + + + + + + + + +` +} + +// PlainTextTemplate returns a Golang template that will generate an plain text email. +func (dt *MailThemeFlat) PlainTextTemplate() string { + return `{{ with .Email.Body.Intros }} + {{ range $line := . }} +

{{ $line }}

+ {{ end }} +{{ end }} +{{ if (ne .Email.Body.FreeMarkdown "") }} + {{ .Email.Body.FreeMarkdown.ToHTML }} +{{ else }} + {{ with .Email.Body.Dictionary }} +
    + {{ range $entry := . }} +
  • {{ $entry.Key }}: {{ $entry.Value }}
    • + {{ end }} +
+ {{ end }} + {{ with .Email.Body.Table }} + {{ $data := .Data }} + {{ $columns := .Columns }} + {{ if gt (len $data) 0 }} + + + {{ $col := index $data 0 }} + {{ range $entry := $col }} + + {{ end }} + + {{ range $row := $data }} + + {{ range $cell := $row }} + + {{ end }} + + {{ end }} +
{{ $entry.Key }}
+ {{ $cell.Value }} +
+ {{ end }} + {{ end }} + {{ with .Email.Body.Actions }} + {{ range $action := . }} +

{{ $action.Instructions }} {{ $action.Button.Link }}

+ {{ end }} + {{ end }} +{{ end }} +{{ with .Email.Body.Outros }} + {{ range $line := . }} +

{{ $line }}

+ {{ end }} +{{ end }} +

{{.Email.Body.Signature}},
{{.Hermes.Product.Name}} - {{.Hermes.Product.Link}}

+ +

{{.Hermes.Product.Copyright}}

+` +} + +func (dt *MailThemeFlat) GetStyle() string { + return ` + +` +} diff --git a/core/notification/mobile.go b/core/notification/mobile.go new file mode 100644 index 000000000..cd36641b0 --- /dev/null +++ b/core/notification/mobile.go @@ -0,0 +1,62 @@ +package notification + +import ( + "errors" + "github.com/crawlab-team/go-trace" + "github.com/imroc/req" + "strings" +) + +type ResBody struct { + ErrCode int `json:"errcode"` + ErrMsg string `json:"errmsg"` +} + +func SendMobileNotification(webhook string, title string, content string) error { + // request header + header := req.Header{ + "Content-Type": "application/json; charset=utf-8", + } + + // request data + data := req.Param{ + "msgtype": "markdown", + "markdown": req.Param{ + "title": title, + "text": content, + "content": content, + }, + "at": req.Param{ + "atMobiles": []string{}, + "isAtAll": false, + }, + "text": content, + } + if strings.Contains(strings.ToLower(webhook), "feishu") { + data = req.Param{ + "msg_type": "text", + "content": req.Param{ + "text": content, + }, + } + } + + // perform request + res, err := req.Post(webhook, header, req.BodyJSON(&data)) + if err != nil { + return trace.TraceError(err) + } + + // parse response + var resBody ResBody + if err := res.ToJSON(&resBody); err != nil { + return trace.TraceError(err) + } + + // validate response code + if resBody.ErrCode != 0 { + return errors.New(resBody.ErrMsg) + } + + return nil +} diff --git a/core/notification/models.go b/core/notification/models.go new file mode 100644 index 000000000..efb554486 --- /dev/null +++ b/core/notification/models.go @@ -0,0 +1,32 @@ +package notification + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type NotificationSetting struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Type string `json:"type" bson:"type"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + Global bool `json:"global" bson:"global"` + Title string `json:"title,omitempty" bson:"title,omitempty"` + Template string `json:"template,omitempty" bson:"template,omitempty"` + TaskTrigger string `json:"task_trigger" bson:"task_trigger"` + Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` + Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` +} + +type NotificationSettingMail struct { + Server string `json:"server" bson:"server"` + Port string `json:"port,omitempty" bson:"port,omitempty"` + User string `json:"user,omitempty" bson:"user,omitempty"` + Password string `json:"password,omitempty" bson:"password,omitempty"` + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` + SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"` + To string `json:"to,omitempty" bson:"to,omitempty"` + Cc string `json:"cc,omitempty" bson:"cc,omitempty"` +} + +type NotificationSettingMobile struct { + Webhook string `json:"webhook" bson:"webhook"` +} diff --git a/core/notification/payload.go b/core/notification/payload.go new file mode 100644 index 000000000..7337ebe0f --- /dev/null +++ b/core/notification/payload.go @@ -0,0 +1,8 @@ +package notification + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type SendPayload struct { + TaskId primitive.ObjectID `json:"task_id"` + Data string `json:"data"` +} diff --git a/core/notification/service.go b/core/notification/service.go new file mode 100644 index 000000000..b2fd8a270 --- /dev/null +++ b/core/notification/service.go @@ -0,0 +1,395 @@ +package notification + +import ( + "github.com/apex/log" + mongo2 "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + parser "github.com/crawlab-team/template-parser" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" +) + +type Service struct { + col *mongo2.Col // notification settings + modelSvc service.ModelService +} + +func (svc *Service) Init() (err error) { + if !utils.IsPro() { + return nil + } + + return nil +} + +func (svc *Service) Start() (err error) { + // initialize data + if err := svc.initData(); err != nil { + return err + } + + return nil +} + +func (svc *Service) Stop() (err error) { + return nil +} + +func (svc *Service) initData() (err error) { + total, err := svc.col.Count(nil) + if err != nil { + return err + } + if total > 0 { + return nil + } + + // data to initialize + settings := []NotificationSetting{ + { + Id: primitive.NewObjectID(), + Type: TypeMail, + Enabled: true, + Name: "任务通知（邮件）", + Description: "这是默认的邮件通知。您可以使用您自己的设置进行编辑。", + TaskTrigger: constants.NotificationTriggerTaskFinish, + Title: "[Crawlab] 爬虫任务更新: {{$.status}}", + Template: `尊敬的 {{$.user.username}}, + +请查看下面的任务数据。 + +|键|值| +|:-:|:--| +|任务状态|{{$.status}}| +|任务优先级|{{$.priority}}| +|任务模式|{{$.mode}}| +|执行命令|{{$.cmd}}| +|执行参数|{{$.param}}| +|错误信息|{{$.error}}| +|节点|{{$.node.name}}| +|爬虫|{{$.spider.name}}| +|项目|{{$.spider.project.name}}| +|定时任务|{{$.schedule.name}}| +|结果数|{{$.:task_stat.result_count}}| +|等待时间（秒）|{#{{$.:task_stat.wait_duration}}/1000#}| +|运行时间（秒）|{#{{$.:task_stat.runtime_duration}}/1000#}| +|总时间（秒）|{#{{$.:task_stat.total_duration}}/1000#}| +|平均结果数/秒|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| +`, + Mail: NotificationSettingMail{ + Server: "smtp.163.com", + Port: "465", + To: "{{$.user[create].email}}", + }, + }, + { + Id: primitive.NewObjectID(), + Type: TypeMail, + Enabled: true, + Name: "Task Change (Mail)", + Description: "This is the default mail notification. You can edit it with your own settings", + TaskTrigger: constants.NotificationTriggerTaskFinish, + Title: "[Crawlab] Task Update: {{$.status}}", + Template: `Dear {{$.user.username}}, + +Please find the task data as below. + +|Key|Value| +|:-:|:--| +|Task Status|{{$.status}}| +|Task Priority|{{$.priority}}| +|Task Mode|{{$.mode}}| +|Task Command|{{$.cmd}}| +|Task Params|{{$.param}}| +|Error Message|{{$.error}}| +|Node|{{$.node.name}}| +|Spider|{{$.spider.name}}| +|Project|{{$.spider.project.name}}| +|Schedule|{{$.schedule.name}}| +|Result Count|{{$.:task_stat.result_count}}| +|Wait Duration (sec)|{#{{$.:task_stat.wait_duration}}/1000#}| +|Runtime Duration (sec)|{#{{$.:task_stat.runtime_duration}}/1000#}| +|Total Duration (sec)|{#{{$.:task_stat.total_duration}}/1000#}| +|Avg Results / Sec|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| +`, + Mail: NotificationSettingMail{ + Server: "smtp.163.com", + Port: "465", + To: "{{$.user[create].email}}", + }, + }, + { + Id: primitive.NewObjectID(), + Type: TypeMobile, + Enabled: true, + Name: "任务通知（移动端）", + Description: "这是默认的手机通知。您可以使用您自己的设置进行编辑。", + TaskTrigger: constants.NotificationTriggerTaskFinish, + Title: "[Crawlab] 任务更新: {{$.status}}", + Template: `尊敬的 {{$.user.username}}, + +请查看下面的任务数据。 + +- **任务状态**: {{$.status}} +- **任务优先级**: {{$.priority}} +- **任务模式**: {{$.mode}} +- **执行命令**: {{$.cmd}} +- **执行参数**: {{$.param}} +- **错误信息**: {{$.error}} +- **节点**: {{$.node.name}} +- **爬虫**: {{$.spider.name}} +- **项目**: {{$.spider.project.name}} +- **定时任务**: {{$.schedule.name}} +- **结果数**: {{$.:task_stat.result_count}} +- **等待时间（秒）**: {#{{$.:task_stat.wait_duration}}/1000#} +- **运行时间（秒）**: {#{{$.:task_stat.runtime_duration}}/1000#} +- **总时间（秒）**: {#{{$.:task_stat.total_duration}}/1000#} +- **平均结果数/秒**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, + Mobile: NotificationSettingMobile{}, + }, + { + Id: primitive.NewObjectID(), + Type: TypeMobile, + Enabled: true, + Name: "Task Change (Mobile)", + Description: "This is the default mobile notification. You can edit it with your own settings", + TaskTrigger: constants.NotificationTriggerTaskError, + Title: "[Crawlab] Task Update: {{$.status}}", + Template: `Dear {{$.user.username}}, + +Please find the task data as below. + +- **Task Status**: {{$.status}} +- **Task Priority**: {{$.priority}} +- **Task Mode**: {{$.mode}} +- **Task Command**: {{$.cmd}} +- **Task Params**: {{$.param}} +- **Error Message**: {{$.error}} +- **Node**: {{$.node.name}} +- **Spider**: {{$.spider.name}} +- **Project**: {{$.spider.project.name}} +- **Schedule**: {{$.schedule.name}} +- **Result Count**: {{$.:task_stat.result_count}} +- **Wait Duration (sec)**: {#{{$.:task_stat.wait_duration}}/1000#} +- **Runtime Duration (sec)**: {#{{$.:task_stat.runtime_duration}}/1000#} +- **Total Duration (sec)**: {#{{$.:task_stat.total_duration}}/1000#} +- **Avg Results / Sec**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, + Mobile: NotificationSettingMobile{}, + }, + } + var data []interface{} + for _, s := range settings { + data = append(data, s) + } + _, err = svc.col.InsertMany(data) + if err != nil { + return err + } + return nil +} + +func (svc *Service) Send(s NotificationSetting, entity bson.M) (err error) { + switch s.Type { + case TypeMail: + return svc.SendMail(s, entity) + case TypeMobile: + return svc.SendMobile(s, entity) + } + return nil +} + +func (svc *Service) SendMail(s NotificationSetting, entity bson.M) (err error) { + // to + to, err := parser.Parse(s.Mail.To, entity) + if err != nil { + log.Warnf("parsing 'to' error: %v", err) + } + if to == "" { + return nil + } + + // cc + cc, err := parser.Parse(s.Mail.Cc, entity) + if err != nil { + log.Warnf("parsing 'cc' error: %v", err) + } + + // title + title, err := parser.Parse(s.Title, entity) + if err != nil { + log.Warnf("parsing 'title' error: %v", err) + } + + // content + content, err := parser.Parse(s.Template, entity) + if err != nil { + log.Warnf("parsing 'content' error: %v", err) + } + + // send mail + if err := SendMail(&models.NotificationSettingMail{ + Server: s.Mail.Server, + Port: s.Mail.Port, + User: s.Mail.User, + Password: s.Mail.Password, + SenderEmail: s.Mail.SenderEmail, + SenderIdentity: s.Mail.SenderIdentity, + To: s.Mail.To, + Cc: s.Mail.Cc, + }, to, cc, title, content); err != nil { + return err + } + + return nil +} + +func (svc *Service) SendMobile(s NotificationSetting, entity bson.M) (err error) { + // webhook + webhook, err := parser.Parse(s.Mobile.Webhook, entity) + if err != nil { + log.Warnf("parsing 'webhook' error: %v", err) + } + if webhook == "" { + return nil + } + + // title + title, err := parser.Parse(s.Title, entity) + if err != nil { + log.Warnf("parsing 'title' error: %v", err) + } + + // content + content, err := parser.Parse(s.Template, entity) + if err != nil { + log.Warnf("parsing 'content' error: %v", err) + } + + // send + if err := SendMobileNotification(webhook, title, content); err != nil { + return err + } + + return nil +} + +func (svc *Service) GetSettingList(query bson.M, pagination *entity.Pagination, sort bson.D) (res []NotificationSetting, total int, err error) { + // options + var options *mongo2.FindOptions + if pagination != nil || sort != nil { + options = new(mongo2.FindOptions) + if pagination != nil { + options.Skip = pagination.Size * (pagination.Page - 1) + options.Limit = pagination.Size + } + if sort != nil { + options.Sort = sort + } + } + + // get list + var list []NotificationSetting + if err := svc.col.Find(query, options).All(&list); err != nil { + if err.Error() == mongo.ErrNoDocuments.Error() { + return nil, 0, nil + } else { + return nil, 0, err + } + } + + // total count + total, err = svc.col.Count(query) + if err != nil { + return nil, 0, err + } + + return list, total, nil +} + +func (svc *Service) GetSetting(id primitive.ObjectID) (res *NotificationSetting, err error) { + var s NotificationSetting + if err := svc.col.FindId(id).One(&s); err != nil { + return nil, err + } + return &s, nil +} + +func (svc *Service) PosSetting(s *NotificationSetting) (err error) { + s.Id = primitive.NewObjectID() + if _, err := svc.col.Insert(s); err != nil { + return err + } + return nil +} + +func (svc *Service) PutSetting(id primitive.ObjectID, s NotificationSetting) (err error) { + if err := svc.col.ReplaceId(id, s); err != nil { + return err + } + + return nil +} + +func (svc *Service) DeleteSetting(id primitive.ObjectID) (err error) { + if err := svc.col.DeleteId(id); err != nil { + return err + } + + return nil +} + +func (svc *Service) EnableSetting(id primitive.ObjectID) (err error) { + return svc._toggleSettingFunc(true)(id) +} + +func (svc *Service) DisableSetting(id primitive.ObjectID) (err error) { + return svc._toggleSettingFunc(false)(id) +} + +func (svc *Service) _toggleSettingFunc(value bool) func(id primitive.ObjectID) error { + return func(id primitive.ObjectID) (err error) { + var s NotificationSetting + if err := svc.col.FindId(id).One(&s); err != nil { + return err + } + s.Enabled = value + if err := svc.col.ReplaceId(id, s); err != nil { + return err + } + return nil + } +} + +func NewService() *Service { + // service + svc := &Service{ + col: mongo2.GetMongoCol(SettingsColName), + } + + // model service + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + svc.modelSvc = modelSvc + + if err := svc.Init(); err != nil { + panic(err) + } + + return svc +} + +var _service *Service + +func GetService() *Service { + if _service == nil { + _service = NewService() + } + return _service +} diff --git a/core/notification/service_test.go b/core/notification/service_test.go new file mode 100644 index 000000000..20e094784 --- /dev/null +++ b/core/notification/service_test.go @@ -0,0 +1,19 @@ +package notification + +import ( + "net/http" + "testing" + "time" +) + +func TestService_sendMobile(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + time.Sleep(1 * time.Second) + + data := map[string]interface{}{ + "task_id": T.TestTask.GetId().Hex(), + } + e.POST("/send/mobile").WithJSON(data). + Expect().Status(http.StatusOK) +} diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go new file mode 100644 index 000000000..23ade59bd --- /dev/null +++ b/core/notification/service_v2.go @@ -0,0 +1,362 @@ +package notification + +import ( + "github.com/apex/log" + mongo2 "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + parser "github.com/crawlab-team/template-parser" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" +) + +type ServiceV2 struct { +} + +func (svc *ServiceV2) Start() (err error) { + // initialize data + if err := svc.initData(); err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) Stop() (err error) { + return nil +} + +func (svc *ServiceV2) initData() (err error) { + total, err := service.NewModelServiceV2[models.NotificationSettingV2]().Count(nil) + if err != nil { + return err + } + if total > 0 { + return nil + } + + // data to initialize + settings := []models.NotificationSettingV2{ + { + Id: primitive.NewObjectID(), + Type: TypeMail, + Enabled: true, + Name: "任务通知（邮件）", + Description: "这是默认的邮件通知。您可以使用您自己的设置进行编辑。", + TaskTrigger: constants.NotificationTriggerTaskFinish, + Title: "[Crawlab] 爬虫任务更新: {{$.status}}", + Template: `尊敬的 {{$.user.username}}, + +请查看下面的任务数据。 + +|键|值| +|:-:|:--| +|任务状态|{{$.status}}| +|任务优先级|{{$.priority}}| +|任务模式|{{$.mode}}| +|执行命令|{{$.cmd}}| +|执行参数|{{$.param}}| +|错误信息|{{$.error}}| +|节点|{{$.node.name}}| +|爬虫|{{$.spider.name}}| +|项目|{{$.spider.project.name}}| +|定时任务|{{$.schedule.name}}| +|结果数|{{$.:task_stat.result_count}}| +|等待时间（秒）|{#{{$.:task_stat.wait_duration}}/1000#}| +|运行时间（秒）|{#{{$.:task_stat.runtime_duration}}/1000#}| +|总时间（秒）|{#{{$.:task_stat.total_duration}}/1000#}| +|平均结果数/秒|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| +`, + Mail: models.NotificationSettingMail{ + Server: "smtp.163.com", + Port: "465", + To: "{{$.user[create].email}}", + }, + }, + { + Id: primitive.NewObjectID(), + Type: TypeMail, + Enabled: true, + Name: "Task Change (Mail)", + Description: "This is the default mail notification. You can edit it with your own settings", + TaskTrigger: constants.NotificationTriggerTaskFinish, + Title: "[Crawlab] Task Update: {{$.status}}", + Template: `Dear {{$.user.username}}, + +Please find the task data as below. + +|Key|Value| +|:-:|:--| +|Task Status|{{$.status}}| +|Task Priority|{{$.priority}}| +|Task Mode|{{$.mode}}| +|Task Command|{{$.cmd}}| +|Task Params|{{$.param}}| +|Error Message|{{$.error}}| +|Node|{{$.node.name}}| +|Spider|{{$.spider.name}}| +|Project|{{$.spider.project.name}}| +|Schedule|{{$.schedule.name}}| +|Result Count|{{$.:task_stat.result_count}}| +|Wait Duration (sec)|{#{{$.:task_stat.wait_duration}}/1000#}| +|Runtime Duration (sec)|{#{{$.:task_stat.runtime_duration}}/1000#}| +|Total Duration (sec)|{#{{$.:task_stat.total_duration}}/1000#}| +|Avg Results / Sec|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| +`, + Mail: models.NotificationSettingMail{ + Server: "smtp.163.com", + Port: "465", + To: "{{$.user[create].email}}", + }, + }, + { + Id: primitive.NewObjectID(), + Type: TypeMobile, + Enabled: true, + Name: "任务通知（移动端）", + Description: "这是默认的手机通知。您可以使用您自己的设置进行编辑。", + TaskTrigger: constants.NotificationTriggerTaskFinish, + Title: "[Crawlab] 任务更新: {{$.status}}", + Template: `尊敬的 {{$.user.username}}, + +请查看下面的任务数据。 + +- **任务状态**: {{$.status}} +- **任务优先级**: {{$.priority}} +- **任务模式**: {{$.mode}} +- **执行命令**: {{$.cmd}} +- **执行参数**: {{$.param}} +- **错误信息**: {{$.error}} +- **节点**: {{$.node.name}} +- **爬虫**: {{$.spider.name}} +- **项目**: {{$.spider.project.name}} +- **定时任务**: {{$.schedule.name}} +- **结果数**: {{$.:task_stat.result_count}} +- **等待时间（秒）**: {#{{$.:task_stat.wait_duration}}/1000#} +- **运行时间（秒）**: {#{{$.:task_stat.runtime_duration}}/1000#} +- **总时间（秒）**: {#{{$.:task_stat.total_duration}}/1000#} +- **平均结果数/秒**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, + Mobile: models.NotificationSettingMobile{}, + }, + { + Id: primitive.NewObjectID(), + Type: TypeMobile, + Enabled: true, + Name: "Task Change (Mobile)", + Description: "This is the default mobile notification. You can edit it with your own settings", + TaskTrigger: constants.NotificationTriggerTaskError, + Title: "[Crawlab] Task Update: {{$.status}}", + Template: `Dear {{$.user.username}}, + +Please find the task data as below. + +- **Task Status**: {{$.status}} +- **Task Priority**: {{$.priority}} +- **Task Mode**: {{$.mode}} +- **Task Command**: {{$.cmd}} +- **Task Params**: {{$.param}} +- **Error Message**: {{$.error}} +- **Node**: {{$.node.name}} +- **Spider**: {{$.spider.name}} +- **Project**: {{$.spider.project.name}} +- **Schedule**: {{$.schedule.name}} +- **Result Count**: {{$.:task_stat.result_count}} +- **Wait Duration (sec)**: {#{{$.:task_stat.wait_duration}}/1000#} +- **Runtime Duration (sec)**: {#{{$.:task_stat.runtime_duration}}/1000#} +- **Total Duration (sec)**: {#{{$.:task_stat.total_duration}}/1000#} +- **Avg Results / Sec**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, + Mobile: models.NotificationSettingMobile{}, + }, + } + _, err = service.NewModelServiceV2[models.NotificationSettingV2]().InsertMany(settings) + if err != nil { + return err + } + return nil +} + +func (svc *ServiceV2) Send(s *models.NotificationSettingV2, entity bson.M) (err error) { + switch s.Type { + case TypeMail: + return svc.SendMail(s, entity) + case TypeMobile: + return svc.SendMobile(s, entity) + } + return nil +} + +func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, entity bson.M) (err error) { + // to + to, err := parser.Parse(s.Mail.To, entity) + if err != nil { + log.Warnf("parsing 'to' error: %v", err) + } + if to == "" { + return nil + } + + // cc + cc, err := parser.Parse(s.Mail.Cc, entity) + if err != nil { + log.Warnf("parsing 'cc' error: %v", err) + } + + // title + title, err := parser.Parse(s.Title, entity) + if err != nil { + log.Warnf("parsing 'title' error: %v", err) + } + + // content + content, err := parser.Parse(s.Template, entity) + if err != nil { + log.Warnf("parsing 'content' error: %v", err) + } + + // send mail + if err := SendMail(&s.Mail, to, cc, title, content); err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) SendMobile(s *models.NotificationSettingV2, entity bson.M) (err error) { + // webhook + webhook, err := parser.Parse(s.Mobile.Webhook, entity) + if err != nil { + log.Warnf("parsing 'webhook' error: %v", err) + } + if webhook == "" { + return nil + } + + // title + title, err := parser.Parse(s.Title, entity) + if err != nil { + log.Warnf("parsing 'title' error: %v", err) + } + + // content + content, err := parser.Parse(s.Template, entity) + if err != nil { + log.Warnf("parsing 'content' error: %v", err) + } + + // send + if err := SendMobileNotification(webhook, title, content); err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) GetSettingList(query bson.M, pagination *entity.Pagination, sort bson.D) (res []models.NotificationSettingV2, total int, err error) { + // options + var options *mongo2.FindOptions + if pagination != nil || sort != nil { + options = new(mongo2.FindOptions) + if pagination != nil { + options.Skip = pagination.Size * (pagination.Page - 1) + options.Limit = pagination.Size + } + if sort != nil { + options.Sort = sort + } + } + + // get list + list, err := service.NewModelServiceV2[models.NotificationSettingV2]().GetMany(query, options) + if err != nil { + if err.Error() == mongo.ErrNoDocuments.Error() { + return nil, 0, nil + } else { + return nil, 0, err + } + } + + // total count + total, err = service.NewModelServiceV2[models.NotificationSettingV2]().Count(query) + if err != nil { + return nil, 0, err + } + + return list, total, nil +} + +func (svc *ServiceV2) GetSetting(id primitive.ObjectID) (res *models.NotificationSettingV2, err error) { + s, err := service.NewModelServiceV2[models.NotificationSettingV2]().GetById(id) + if err != nil { + return nil, err + } + return s, nil +} + +func (svc *ServiceV2) PosSetting(s *models.NotificationSettingV2) (err error) { + s.Id = primitive.NewObjectID() + _, err = service.NewModelServiceV2[models.NotificationSettingV2]().InsertOne(*s) + if err != nil { + return err + } + return nil +} + +func (svc *ServiceV2) PutSetting(id primitive.ObjectID, s models.NotificationSettingV2) (err error) { + err = service.NewModelServiceV2[models.NotificationSettingV2]().ReplaceById(id, s) + if err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) DeleteSetting(id primitive.ObjectID) (err error) { + err = service.NewModelServiceV2[models.NotificationSettingV2]().DeleteById(id) + if err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) EnableSetting(id primitive.ObjectID) (err error) { + return svc._toggleSettingFunc(true)(id) +} + +func (svc *ServiceV2) DisableSetting(id primitive.ObjectID) (err error) { + return svc._toggleSettingFunc(false)(id) +} + +func (svc *ServiceV2) _toggleSettingFunc(value bool) func(id primitive.ObjectID) error { + return func(id primitive.ObjectID) (err error) { + s, err := service.NewModelServiceV2[models.NotificationSettingV2]().GetById(id) + if err != nil { + return err + } + s.Enabled = value + err = service.NewModelServiceV2[models.NotificationSettingV2]().ReplaceById(id, *s) + if err != nil { + return err + } + return nil + } +} + +func NewServiceV2() *ServiceV2 { + // service + svc := &ServiceV2{} + + return svc +} + +var _serviceV2 *ServiceV2 + +func GetServiceV2() *ServiceV2 { + if _serviceV2 == nil { + _serviceV2 = NewServiceV2() + } + return _serviceV2 +} diff --git a/core/process/daemon.go b/core/process/daemon.go new file mode 100644 index 000000000..0a5395c42 --- /dev/null +++ b/core/process/daemon.go @@ -0,0 +1,170 @@ +package process + +import ( + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/sys_exec" + "github.com/crawlab-team/go-trace" + "math/rand" + "os/exec" + "time" +) + +const ( + SignalCreate = iota + SignalStart + SignalStopped + SignalError + SignalExited + SignalReachedMaxErrors +) + +type Daemon struct { + // settings + maxErrors int + exitTimeout time.Duration + + // internals + errors int + errMsg string + exitCode int + newCmdFn func() *exec.Cmd + cmd *exec.Cmd + stopped bool + ch chan int +} + +func (d *Daemon) Start() (err error) { + go d.handleSignal() + + for { + // command + d.cmd = d.newCmdFn() + d.ch <- SignalCreate + + // attempt to run + _ = d.cmd.Start() + d.ch <- SignalStart + + if err := d.cmd.Wait(); err != nil { + // stopped + d.ch <- SignalStopped + if d.stopped { + log.Infof("daemon stopped") + return nil + } + + // error + d.ch <- SignalError + d.errMsg = err.Error() + trace.PrintError(err) + } + + // exited + d.ch <- SignalExited + + // exit code + d.exitCode = d.cmd.ProcessState.ExitCode() + + // check exit code + if d.exitCode == 0 { + log.Infof("process exited with code 0") + return + } + + // error message + d.errMsg = errors.ErrorProcessDaemonProcessExited.Error() + + // increment errors + d.errors++ + + // validate if error count exceeds max errors + if d.errors >= d.maxErrors { + log.Infof("reached max errors: %d", d.maxErrors) + d.ch <- SignalReachedMaxErrors + return errors.ErrorProcessReachedMaxErrors + } + + // re-attempt + waitSec := rand.Intn(5) + log.Infof("re-attempt to start process in %d seconds...", waitSec) + time.Sleep(time.Duration(waitSec) * time.Second) + } +} + +func (d *Daemon) Stop() { + d.stopped = true + opts := &sys_exec.KillProcessOptions{ + Timeout: d.exitTimeout, + Force: false, + } + _ = sys_exec.KillProcess(d.cmd, opts) +} + +func (d *Daemon) GetMaxErrors() (maxErrors int) { + return d.maxErrors +} + +func (d *Daemon) SetMaxErrors(maxErrors int) { + d.maxErrors = maxErrors +} + +func (d *Daemon) GetExitTimeout() (timeout time.Duration) { + return d.exitTimeout +} + +func (d *Daemon) SetExitTimeout(timeout time.Duration) { + d.exitTimeout = timeout +} + +func (d *Daemon) GetCmd() (cmd *exec.Cmd) { + return d.cmd +} + +func (d *Daemon) GetCh() (ch chan int) { + return d.ch +} + +func (d *Daemon) handleSignal() { + for { + select { + case signal := <-d.ch: + switch signal { + case SignalCreate: + log.Infof("process created") + case SignalStart: + log.Infof("process started") + case SignalStopped: + log.Infof("process stopped") + case SignalError: + trace.PrintError(errors.NewProcessError(d.errMsg)) + case SignalExited: + log.Infof("process exited") + case SignalReachedMaxErrors: + log.Infof("reached max errors") + return + } + } + } +} + +func NewProcessDaemon(newCmdFn func() *exec.Cmd, opts ...DaemonOption) (d interfaces.ProcessDaemon) { + // daemon + d = &Daemon{ + maxErrors: 5, + exitTimeout: 15 * time.Second, + errors: 0, + errMsg: "", + newCmdFn: newCmdFn, + stopped: false, + ch: make(chan int), + } + + // apply options + for _, opt := range opts { + opt(d) + } + + return d +} diff --git a/core/process/daemon_test.go b/core/process/daemon_test.go new file mode 100644 index 000000000..7734eb598 --- /dev/null +++ b/core/process/daemon_test.go @@ -0,0 +1,21 @@ +package process + +import ( + "github.com/stretchr/testify/require" + "os/exec" + "testing" +) + +func TestDaemon(t *testing.T) { + d := NewProcessDaemon(func() *exec.Cmd { + return exec.Command("echo", "hello") + }) + err := d.Start() + require.Nil(t, err) + + d = NewProcessDaemon(func() *exec.Cmd { + return exec.Command("return", "1") + }) + err = d.Start() + require.NotNil(t, err) +} diff --git a/core/process/manage.go b/core/process/manage.go new file mode 100644 index 000000000..d2b050136 --- /dev/null +++ b/core/process/manage.go @@ -0,0 +1,60 @@ +package process + +import ( + "github.com/crawlab-team/go-trace" + "os/exec" + "regexp" + "runtime" + "strings" +) + +var pidRegexp, _ = regexp.Compile("(?:^|\\s+)\\d+(?:$|\\s+)") + +func ProcessIdExists(id int) (ok bool) { + lines, err := ListProcess(string(rune(id))) + if err != nil { + return false + } + for _, line := range lines { + matched := pidRegexp.MatchString(line) + if matched { + return true + } + } + return false +} + +func ListProcess(text string) (lines []string, err error) { + if runtime.GOOS == "windows" { + return listProcessWindow(text) + } else { + return listProcessLinuxMac(text) + } +} + +func listProcessWindow(text string) (lines []string, err error) { + cmd := exec.Command("tasklist", "/fi", text) + out, err := cmd.CombinedOutput() + _, ok := err.(*exec.ExitError) + if !ok { + return nil, trace.TraceError(err) + } + lines = strings.Split(string(out), "\n") + return lines, nil +} + +func listProcessLinuxMac(text string) (lines []string, err error) { + cmd := exec.Command("ps", "aux") + out, err := cmd.CombinedOutput() + _, ok := err.(*exec.ExitError) + if !ok { + return nil, trace.TraceError(err) + } + _lines := strings.Split(string(out), "\n") + for _, l := range _lines { + if strings.Contains(l, text) { + lines = append(lines, l) + } + } + return lines, nil +} diff --git a/core/process/options.go b/core/process/options.go new file mode 100644 index 000000000..f0e046c91 --- /dev/null +++ b/core/process/options.go @@ -0,0 +1,20 @@ +package process + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type DaemonOption func(d interfaces.ProcessDaemon) + +func WithDaemonMaxErrors(maxErrors int) DaemonOption { + return func(d interfaces.ProcessDaemon) { + d.SetMaxErrors(maxErrors) + } +} + +func WithExitTimeout(timeout time.Duration) DaemonOption { + return func(d interfaces.ProcessDaemon) { + + } +} diff --git a/core/result/options.go b/core/result/options.go new file mode 100644 index 000000000..59c633015 --- /dev/null +++ b/core/result/options.go @@ -0,0 +1,16 @@ +package result + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type Option func(opts *Options) + +type Options struct { + registryKey string // registry key + SpiderId primitive.ObjectID // data source id +} + +func WithRegistryKey(key string) Option { + return func(opts *Options) { + opts.registryKey = key + } +} diff --git a/core/result/service.go b/core/result/service.go new file mode 100644 index 000000000..3acd63219 --- /dev/null +++ b/core/result/service.go @@ -0,0 +1,89 @@ +package result + +import ( + "fmt" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" + "sync" +) + +func NewResultService(registryKey string, s *models.Spider) (svc2 interfaces.ResultService, err error) { + // result service function + var fn interfaces.ResultServiceRegistryFn + + if registryKey == "" { + // default + fn = NewResultServiceMongo + } else { + // from registry + reg := GetResultServiceRegistry() + fn = reg.Get(registryKey) + if fn == nil { + return nil, errors.NewResultError(fmt.Sprintf("%s is not implemented", registryKey)) + } + } + + // generate result service + svc, err := fn(s.ColId, s.DataSourceId) + if err != nil { + return nil, trace.TraceError(err) + } + + return svc, nil +} + +var store = sync.Map{} + +func GetResultService(spiderId primitive.ObjectID, opts ...Option) (svc2 interfaces.ResultService, err error) { + // model service + modelSvc, err := service.GetService() + if err != nil { + return nil, trace.TraceError(err) + } + + // spider + s, err := modelSvc.GetSpiderById(spiderId) + if err != nil { + return nil, trace.TraceError(err) + } + + // apply options + _opts := &Options{} + for _, opt := range opts { + opt(_opts) + } + + // store key + storeKey := s.ColId.Hex() + ":" + s.DataSourceId.Hex() + + // attempt to load result service from store + res, _ := store.Load(storeKey) + if res != nil { + svc, ok := res.(interfaces.ResultService) + if ok { + return svc, nil + } + } + + // registry key + var registryKey string + ds, _ := modelSvc.GetDataSourceById(s.DataSourceId) + if ds != nil { + registryKey = ds.Type + } + + // create a new result service if not exists + svc, err := NewResultService(registryKey, s) + if err != nil { + return nil, err + } + + // save into store + store.Store(storeKey, svc) + + return svc, nil +} diff --git a/core/result/service_mongo.go b/core/result/service_mongo.go new file mode 100644 index 000000000..cdc6ff3fa --- /dev/null +++ b/core/result/service_mongo.go @@ -0,0 +1,146 @@ +package result + +import ( + "time" + + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type ServiceMongo struct { + // dependencies + modelSvc service.ModelService + modelColSvc interfaces.ModelBaseService + + // internals + colId primitive.ObjectID // _id of models.DataCollection + dc *models.DataCollection // models.DataCollection + t time.Time +} + +func (svc *ServiceMongo) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { + _query := svc.getQuery(query) + _opts := svc.getOpts(opts) + return svc.getList(_query, _opts) +} + +func (svc *ServiceMongo) Count(query generic.ListQuery) (n int, err error) { + _query := svc.getQuery(query) + return svc.modelColSvc.Count(_query) +} + +func (svc *ServiceMongo) Insert(docs ...interface{}) (err error) { + if svc.dc.Dedup.Enabled && len(svc.dc.Dedup.Keys) > 0 { + for _, doc := range docs { + hash, err := utils.GetResultHash(doc, svc.dc.Dedup.Keys) + if err != nil { + return err + } + doc.(interfaces.Result).SetValue(constants.HashKey, hash) + query := bson.M{constants.HashKey: hash} + switch svc.dc.Dedup.Type { + case constants.DedupTypeOverwrite: + err = mongo.GetMongoCol(svc.dc.Name).ReplaceWithOptions(query, doc, &options.ReplaceOptions{Upsert: &[]bool{true}[0]}) + if err != nil { + return trace.TraceError(err) + } + default: + var o bson.M + err := mongo.GetMongoCol(svc.dc.Name).Find(query, &mongo.FindOptions{Limit: 1}).One(&o) + if err == nil { + // exists, ignore + continue + } + if err != mongo2.ErrNoDocuments { + // error + return trace.TraceError(err) + } + // not exists, insert + _, err = mongo.GetMongoCol(svc.dc.Name).Insert(doc) + if err != nil { + return trace.TraceError(err) + } + } + } + } else { + _, err = mongo.GetMongoCol(svc.dc.Name).InsertMany(docs) + if err != nil { + return trace.TraceError(err) + } + } + return nil +} + +func (svc *ServiceMongo) Index(fields []string) { + for _, field := range fields { + _ = mongo.GetMongoCol(svc.dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{field: 1}}) + } +} + +func (svc *ServiceMongo) SetTime(t time.Time) { + svc.t = t +} + +func (svc *ServiceMongo) GetTime() (t time.Time) { + return svc.t +} + +func (svc *ServiceMongo) getList(query bson.M, opts *mongo.FindOptions) (results []interface{}, err error) { + list, err := svc.modelColSvc.GetList(query, opts) + if err != nil { + return nil, err + } + for _, d := range list.GetModels() { + r, ok := d.(interfaces.Result) + if ok { + results = append(results, r) + } + } + return results, nil +} + +func (svc *ServiceMongo) getQuery(query generic.ListQuery) (res bson.M) { + return utils.GetMongoQuery(query) +} + +func (svc *ServiceMongo) getOpts(opts *generic.ListOptions) (res *mongo.FindOptions) { + return utils.GetMongoOpts(opts) +} + +func NewResultServiceMongo(colId primitive.ObjectID, _ primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &ServiceMongo{ + colId: colId, + t: time.Now(), + } + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, err + } + + // data collection + svc.dc, _ = svc.modelSvc.GetDataCollectionById(colId) + go func() { + for { + time.Sleep(1 * time.Second) + svc.dc, _ = svc.modelSvc.GetDataCollectionById(colId) + } + }() + + // data collection model service + svc.modelColSvc = service.GetBaseServiceByColName(interfaces.ModelIdResult, svc.dc.Name) + + return svc, nil +} diff --git a/core/result/service_registry.go b/core/result/service_registry.go new file mode 100644 index 000000000..2c635397a --- /dev/null +++ b/core/result/service_registry.go @@ -0,0 +1,48 @@ +package result + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "sync" +) + +type ServiceRegistry struct { + // internals + services sync.Map +} + +func (r *ServiceRegistry) Register(key string, fn interfaces.ResultServiceRegistryFn) { + r.services.Store(key, fn) +} + +func (r *ServiceRegistry) Unregister(key string) { + r.services.Delete(key) +} + +func (r *ServiceRegistry) Get(key string) (fn interfaces.ResultServiceRegistryFn) { + res, ok := r.services.Load(key) + if ok { + fn, ok = res.(interfaces.ResultServiceRegistryFn) + if !ok { + return nil + } + return fn + } + return nil +} + +func NewResultServiceRegistry() (r interfaces.ResultServiceRegistry) { + r = &ServiceRegistry{ + services: sync.Map{}, + } + return r +} + +var _svc interfaces.ResultServiceRegistry + +func GetResultServiceRegistry() (r interfaces.ResultServiceRegistry) { + if _svc != nil { + return _svc + } + _svc = NewResultServiceRegistry() + return _svc +} diff --git a/core/result/test/base.go b/core/result/test/base.go new file mode 100644 index 000000000..14331b208 --- /dev/null +++ b/core/result/test/base.go @@ -0,0 +1,76 @@ +package test + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/result" + "go.uber.org/dig" + "testing" +) + +func init() { + T = NewTest() +} + +var T *Test + +type Test struct { + // dependencies + modelSvc service.ModelService + resultSvc interfaces.ResultService + + // test data + TestColName string + TestCol *mongo.Col + TestDc *models.DataCollection +} + +func (t *Test) Setup(t2 *testing.T) { + t2.Cleanup(t.Cleanup) +} + +func (t *Test) Cleanup() { + _ = t.modelSvc.DropAll() +} + +func NewTest() *Test { + var err error + + // test + t := &Test{ + TestColName: "test_results", + } + + // dependency injection + c := dig.New() + if err := c.Provide(service.NewService); err != nil { + panic(err) + } + if err := c.Invoke(func( + modelSvc service.ModelService, + ) { + t.modelSvc = modelSvc + }); err != nil { + panic(err) + } + + // data collection + t.TestDc = &models.DataCollection{ + Name: t.TestColName, + } + if err := delegate.NewModelDelegate(t.TestDc).Add(); err != nil { + panic(err) + } + t.TestCol = mongo.GetMongoCol(t.TestColName) + + // result service + t.resultSvc, err = result.GetResultService(t.TestDc.GetId()) + if err != nil { + panic(err) + } + + return t +} diff --git a/core/result/test/service_test.go b/core/result/test/service_test.go new file mode 100644 index 000000000..b44faf4e8 --- /dev/null +++ b/core/result/test/service_test.go @@ -0,0 +1,67 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/stretchr/testify/require" + "testing" +) + +func TestResultService_GetList(t *testing.T) { + var err error + T.Setup(t) + + n := 1000 + var docs []interface{} + for i := 0; i < n; i++ { + d := &models.Result{ + "i": i, + } + docs = append(docs, d) + } + _, err = T.TestCol.InsertMany(docs) + require.Nil(t, err) + + // get all + results, err := T.resultSvc.List(nil, nil) + require.Nil(t, err) + require.Equal(t, n, len(results)) + + //query := bson.M{ + // "i": bson.M{ + // "$lt": n / 2, + // }, + //} + //results, err = T.resultSvc.List(query, nil) + //require.Nil(t, err) + //require.Equal(t, n/2, len(results)) +} + +func TestResultService_Count(t *testing.T) { + var err error + T.Setup(t) + + n := 1000 + var docs []interface{} + for i := 0; i < n; i++ { + d := &models.Result{ + "i": i, + } + docs = append(docs, d) + } + _, err = T.TestCol.InsertMany(docs) + require.Nil(t, err) + + // get all + total, err := T.resultSvc.Count(nil) + require.Nil(t, err) + require.Equal(t, n, total) + + //query := bson.M{ + // "i": bson.M{ + // "$lt": n / 2, + // }, + //} + //total, err = T.resultSvc.Count(query) + //require.Nil(t, err) + //require.Equal(t, n/2, total) +} diff --git a/core/routes/group.go b/core/routes/group.go new file mode 100644 index 000000000..cade1af40 --- /dev/null +++ b/core/routes/group.go @@ -0,0 +1,20 @@ +package routes + +import ( + "github.com/crawlab-team/crawlab/core/middlewares" + "github.com/gin-gonic/gin" +) + +type RouterGroups struct { + AuthGroup *gin.RouterGroup + AnonymousGroup *gin.RouterGroup + FilerGroup *gin.RouterGroup +} + +func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) { + return &RouterGroups{ + AuthGroup: app.Group("/", middlewares.AuthorizationMiddleware()), + AnonymousGroup: app.Group("/"), + FilerGroup: app.Group("/filer", middlewares.FilerAuthorizationMiddleware()), + } +} diff --git a/core/routes/router.go b/core/routes/router.go new file mode 100644 index 000000000..c82603bcd --- /dev/null +++ b/core/routes/router.go @@ -0,0 +1,178 @@ +package routes + +import ( + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/controllers" + "github.com/gin-gonic/gin" + "net/http" + "path" +) + +type RouterServiceInterface interface { + RegisterControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListController) + RegisterHandlerToGroup(group *gin.RouterGroup, path string, method string, handler gin.HandlerFunc) +} + +type RouterService struct { + app *gin.Engine +} + +func NewRouterService(app *gin.Engine) (svc *RouterService) { + return &RouterService{ + app: app, + } +} + +func (svc *RouterService) RegisterControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.BasicController) { + group.GET(basePath, ctr.Get) + group.POST(basePath, ctr.Post) + group.PUT(basePath, ctr.Put) + group.DELETE(basePath, ctr.Delete) +} + +func (svc *RouterService) RegisterListControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListController) { + group.GET(basePath+"/:id", ctr.Get) + group.GET(basePath, ctr.GetList) + group.POST(basePath, ctr.Post) + group.POST(basePath+"/batch", ctr.PostList) + group.PUT(basePath+"/:id", ctr.Put) + group.PUT(basePath, ctr.PutList) + group.DELETE(basePath+"/:id", ctr.Delete) + group.DELETE(basePath, ctr.DeleteList) +} + +func (svc *RouterService) RegisterActionControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ActionController) { + for _, action := range ctr.Actions() { + routerPath := path.Join(basePath, action.Path) + switch action.Method { + case http.MethodGet: + group.GET(routerPath, action.HandlerFunc) + case http.MethodPost: + group.POST(routerPath, action.HandlerFunc) + case http.MethodPut: + group.PUT(routerPath, action.HandlerFunc) + case http.MethodDelete: + group.DELETE(routerPath, action.HandlerFunc) + } + } +} + +func (svc *RouterService) RegisterListActionControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListActionController) { + svc.RegisterListControllerToGroup(group, basePath, ctr) + svc.RegisterActionControllerToGroup(group, basePath, ctr) +} + +func (svc *RouterService) RegisterHandlerToGroup(group *gin.RouterGroup, path string, method string, handler gin.HandlerFunc) { + switch method { + case http.MethodGet: + group.GET(path, handler) + case http.MethodPost: + group.POST(path, handler) + case http.MethodPut: + group.PUT(path, handler) + case http.MethodDelete: + group.DELETE(path, handler) + default: + log.Warn(fmt.Sprintf("%s is not a valid http method", method)) + } +} + +func InitRoutes(app *gin.Engine) (err error) { + // routes groups + groups := NewRouterGroups(app) + + // router service + svc := NewRouterService(app) + + // register routes + registerRoutesAnonymousGroup(svc, groups) + registerRoutesAuthGroup(svc, groups) + registerRoutesFilterGroup(svc, groups) + + return nil +} + +func registerRoutesAnonymousGroup(svc *RouterService, groups *RouterGroups) { + // login + svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/", controllers.LoginController) + + // version + svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/version", controllers.VersionController) + + // system info + svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/system-info", controllers.SystemInfoController) + + // demo + svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/demo", controllers.DemoController) + + // sync + svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/sync", controllers.SyncController) +} + +func registerRoutesAuthGroup(svc *RouterService, groups *RouterGroups) { + // node + svc.RegisterListControllerToGroup(groups.AuthGroup, "/nodes", controllers.NodeController) + + // project + svc.RegisterListControllerToGroup(groups.AuthGroup, "/projects", controllers.ProjectController) + + // user + svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/users", controllers.UserController) + + // spider + svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/spiders", controllers.SpiderController) + + // task + svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/tasks", controllers.TaskController) + + // tag + svc.RegisterListControllerToGroup(groups.AuthGroup, "/tags", controllers.TagController) + + // setting + svc.RegisterListControllerToGroup(groups.AuthGroup, "/settings", controllers.SettingController) + + // data collection + svc.RegisterListControllerToGroup(groups.AuthGroup, "/data/collections", controllers.DataCollectionController) + + // result + svc.RegisterActionControllerToGroup(groups.AuthGroup, "/results", controllers.ResultController) + + // schedule + svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/schedules", controllers.ScheduleController) + + // stats + svc.RegisterActionControllerToGroup(groups.AuthGroup, "/stats", controllers.StatsController) + + // token + svc.RegisterListControllerToGroup(groups.AuthGroup, "/tokens", controllers.TokenController) + + // git + svc.RegisterListControllerToGroup(groups.AuthGroup, "/gits", controllers.GitController) + + // role + svc.RegisterListControllerToGroup(groups.AuthGroup, "/roles", controllers.RoleController) + + // permission + svc.RegisterListControllerToGroup(groups.AuthGroup, "/permissions", controllers.PermissionController) + + // export + svc.RegisterActionControllerToGroup(groups.AuthGroup, "/export", controllers.ExportController) + + // notification + svc.RegisterActionControllerToGroup(groups.AuthGroup, "/notifications", controllers.NotificationController) + + // filter + svc.RegisterActionControllerToGroup(groups.AuthGroup, "/filters", controllers.FilterController) + + // data sources + svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/data-sources", controllers.DataSourceController) + + // environments + svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/environments", controllers.EnvironmentController) +} + +func registerRoutesFilterGroup(svc *RouterService, groups *RouterGroups) { + // filer + svc.RegisterActionControllerToGroup(groups.FilerGroup, "", controllers.FilerController) +} diff --git a/core/routes/router_test.go b/core/routes/router_test.go new file mode 100644 index 000000000..7b72c8524 --- /dev/null +++ b/core/routes/router_test.go @@ -0,0 +1,26 @@ +package routes + +import ( + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "net/http" + "testing" + "time" +) + +func TestInitRoutes(t *testing.T) { + app := gin.New() + err := InitRoutes(app) + require.Nil(t, err) + + srv := &http.Server{ + Handler: app, + Addr: "localhost:8000", + } + go func() { + err = srv.ListenAndServe() + require.Nil(t, err) + }() + + time.Sleep(5 * time.Second) +} diff --git a/core/schedule/logger.go b/core/schedule/logger.go new file mode 100644 index 000000000..fa6bf8340 --- /dev/null +++ b/core/schedule/logger.go @@ -0,0 +1,35 @@ +package schedule + +import ( + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/go-trace" + "github.com/robfig/cron/v3" + "strings" +) + +type Logger struct { +} + +func (l *Logger) Info(msg string, keysAndValues ...interface{}) { + p := l.getPlaceholder(len(keysAndValues)) + log.Infof(fmt.Sprintf("cron: %s %s", msg, p), keysAndValues...) +} + +func (l *Logger) Error(err error, msg string, keysAndValues ...interface{}) { + p := l.getPlaceholder(len(keysAndValues)) + log.Errorf(fmt.Sprintf("cron: %s %s", msg, p), keysAndValues...) + trace.PrintError(err) +} + +func (l *Logger) getPlaceholder(n int) (s string) { + var arr []string + for i := 0; i < n; i++ { + arr = append(arr, "%v") + } + return strings.Join(arr, " ") +} + +func NewLogger() cron.Logger { + return &Logger{} +} diff --git a/core/schedule/options.go b/core/schedule/options.go new file mode 100644 index 000000000..6ad22f99c --- /dev/null +++ b/core/schedule/options.go @@ -0,0 +1,37 @@ +package schedule + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type Option func(svc interfaces.ScheduleService) + +func WithConfigPath(path string) Option { + return func(svc interfaces.ScheduleService) { + svc.SetConfigPath(path) + } +} + +func WithLocation(loc *time.Location) Option { + return func(svc interfaces.ScheduleService) { + svc.SetLocation(loc) + } +} + +func WithDelayIfStillRunning() Option { + return func(svc interfaces.ScheduleService) { + svc.SetDelay(true) + } +} + +func WithSkipIfStillRunning() Option { + return func(svc interfaces.ScheduleService) { + svc.SetSkip(true) + } +} + +func WithUpdateInterval(interval time.Duration) Option { + return func(svc interfaces.ScheduleService) { + } +} diff --git a/core/schedule/service.go b/core/schedule/service.go new file mode 100644 index 000000000..f4b978da5 --- /dev/null +++ b/core/schedule/service.go @@ -0,0 +1,287 @@ +package schedule + +import ( + "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/robfig/cron/v3" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "sync" + "time" +) + +type Service struct { + // dependencies + interfaces.WithConfigPath + modelSvc service.ModelService + adminSvc interfaces.SpiderAdminService + + // settings variables + loc *time.Location + delay bool + skip bool + updateInterval time.Duration + + // internals + cron *cron.Cron + logger cron.Logger + schedules []models.Schedule + stopped bool + mu sync.Mutex +} + +func (svc *Service) GetLocation() (loc *time.Location) { + return svc.loc +} + +func (svc *Service) SetLocation(loc *time.Location) { + svc.loc = loc +} + +func (svc *Service) GetDelay() (delay bool) { + return svc.delay +} + +func (svc *Service) SetDelay(delay bool) { + svc.delay = delay +} + +func (svc *Service) GetSkip() (skip bool) { + return svc.skip +} + +func (svc *Service) SetSkip(skip bool) { + svc.skip = skip +} + +func (svc *Service) GetUpdateInterval() (interval time.Duration) { + return svc.updateInterval +} + +func (svc *Service) SetUpdateInterval(interval time.Duration) { + svc.updateInterval = interval +} + +func (svc *Service) Init() (err error) { + return svc.fetch() +} + +func (svc *Service) Start() { + svc.cron.Start() + go svc.Update() +} + +func (svc *Service) Wait() { + utils.DefaultWait() + svc.Stop() +} + +func (svc *Service) Stop() { + svc.stopped = true + svc.cron.Stop() +} + +func (svc *Service) Enable(s interfaces.Schedule, args ...interface{}) (err error) { + svc.mu.Lock() + defer svc.mu.Unlock() + + id, err := svc.cron.AddFunc(s.GetCron(), svc.schedule(s.GetId())) + if err != nil { + return trace.TraceError(err) + } + s.SetEnabled(true) + s.SetEntryId(id) + u := utils.GetUserFromArgs(args...) + return delegate.NewModelDelegate(s, u).Save() +} + +func (svc *Service) Disable(s interfaces.Schedule, args ...interface{}) (err error) { + svc.mu.Lock() + defer svc.mu.Unlock() + + svc.cron.Remove(s.GetEntryId()) + s.SetEnabled(false) + s.SetEntryId(-1) + u := utils.GetUserFromArgs(args...) + return delegate.NewModelDelegate(s, u).Save() +} + +func (svc *Service) Update() { + for { + if svc.stopped { + return + } + + svc.update() + + time.Sleep(svc.updateInterval) + } +} + +func (svc *Service) GetCron() (c *cron.Cron) { + return svc.cron +} + +func (svc *Service) update() { + // fetch enabled schedules + if err := svc.fetch(); err != nil { + trace.PrintError(err) + return + } + + // entry id map + entryIdsMap := svc.getEntryIdsMap() + + // iterate enabled schedules + for _, s := range svc.schedules { + _, ok := entryIdsMap[s.EntryId] + if ok { + entryIdsMap[s.EntryId] = true + } else { + if err := svc.Enable(&s); err != nil { + trace.PrintError(err) + continue + } + } + } + + // remove non-existent entries + for id, ok := range entryIdsMap { + if !ok { + svc.cron.Remove(id) + } + } +} + +func (svc *Service) getEntryIdsMap() (res map[cron.EntryID]bool) { + res = map[cron.EntryID]bool{} + for _, e := range svc.cron.Entries() { + res[e.ID] = false + } + return res +} + +func (svc *Service) fetch() (err error) { + query := bson.M{ + "enabled": true, + } + svc.schedules, err = svc.modelSvc.GetScheduleList(query, nil) + if err != nil { + return err + } + return nil +} + +func (svc *Service) schedule(id primitive.ObjectID) (fn func()) { + return func() { + // schedule + s, err := svc.modelSvc.GetScheduleById(id) + if err != nil { + trace.PrintError(err) + return + } + + // spider + spider, err := svc.modelSvc.GetSpiderById(s.GetSpiderId()) + if err != nil { + trace.PrintError(err) + return + } + + // options + opts := &interfaces.SpiderRunOptions{ + Mode: s.GetMode(), + NodeIds: s.GetNodeIds(), + Cmd: s.GetCmd(), + Param: s.GetParam(), + Priority: s.GetPriority(), + ScheduleId: s.GetId(), + UserId: s.UserId, + } + + // normalize options + if opts.Mode == "" { + opts.Mode = spider.Mode + } + if len(opts.NodeIds) == 0 { + opts.NodeIds = spider.NodeIds + } + if opts.Cmd == "" { + opts.Cmd = spider.Cmd + } + if opts.Param == "" { + opts.Param = spider.Param + } + if opts.Priority == 0 { + if spider.Priority > 0 { + opts.Priority = spider.Priority + } else { + opts.Priority = 5 + } + } + + // schedule or assign a task in the task queue + if _, err := svc.adminSvc.Schedule(s.GetSpiderId(), opts); err != nil { + trace.PrintError(err) + } + } +} + +func NewScheduleService() (svc2 interfaces.ScheduleService, err error) { + // service + svc := &Service{ + WithConfigPath: config.NewConfigPathService(), + loc: time.Local, + // TODO: implement delay and skip + delay: false, + skip: false, + updateInterval: 1 * time.Minute, + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + modelSvc service.ModelService, + adminSvc interfaces.SpiderAdminService, + ) { + svc.modelSvc = modelSvc + svc.adminSvc = adminSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + // logger + svc.logger = NewLogger() + + // cron + svc.cron = cron.New( + cron.WithLogger(svc.logger), + cron.WithLocation(svc.loc), + cron.WithChain(cron.Recover(svc.logger)), + ) + + // initialize + if err := svc.Init(); err != nil { + return nil, err + } + + return svc, nil +} + +var svc interfaces.ScheduleService + +func GetScheduleService() (res interfaces.ScheduleService, err error) { + if svc != nil { + return svc, nil + } + svc, err = NewScheduleService() + if err != nil { + return nil, err + } + return svc, nil +} diff --git a/core/schedule/service_v2.go b/core/schedule/service_v2.go new file mode 100644 index 000000000..c1cffbefc --- /dev/null +++ b/core/schedule/service_v2.go @@ -0,0 +1,283 @@ +package schedule + +import ( + "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/spider/admin" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/robfig/cron/v3" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "sync" + "time" +) + +type ServiceV2 struct { + // dependencies + interfaces.WithConfigPath + modelSvc *service.ModelServiceV2[models.ScheduleV2] + adminSvc *admin.ServiceV2 + + // settings variables + loc *time.Location + delay bool + skip bool + updateInterval time.Duration + + // internals + cron *cron.Cron + logger cron.Logger + schedules []models.ScheduleV2 + stopped bool + mu sync.Mutex +} + +func (svc *ServiceV2) GetLocation() (loc *time.Location) { + return svc.loc +} + +func (svc *ServiceV2) SetLocation(loc *time.Location) { + svc.loc = loc +} + +func (svc *ServiceV2) GetDelay() (delay bool) { + return svc.delay +} + +func (svc *ServiceV2) SetDelay(delay bool) { + svc.delay = delay +} + +func (svc *ServiceV2) GetSkip() (skip bool) { + return svc.skip +} + +func (svc *ServiceV2) SetSkip(skip bool) { + svc.skip = skip +} + +func (svc *ServiceV2) GetUpdateInterval() (interval time.Duration) { + return svc.updateInterval +} + +func (svc *ServiceV2) SetUpdateInterval(interval time.Duration) { + svc.updateInterval = interval +} + +func (svc *ServiceV2) Init() (err error) { + return svc.fetch() +} + +func (svc *ServiceV2) Start() { + svc.cron.Start() + go svc.Update() +} + +func (svc *ServiceV2) Wait() { + utils.DefaultWait() + svc.Stop() +} + +func (svc *ServiceV2) Stop() { + svc.stopped = true + svc.cron.Stop() +} + +func (svc *ServiceV2) Enable(s models.ScheduleV2, by primitive.ObjectID) (err error) { + svc.mu.Lock() + defer svc.mu.Unlock() + + id, err := svc.cron.AddFunc(s.Cron, svc.schedule(s.Id)) + if err != nil { + return trace.TraceError(err) + } + s.Enabled = true + s.EntryId = id + s.SetUpdated(by) + return svc.modelSvc.ReplaceById(s.Id, s) +} + +func (svc *ServiceV2) Disable(s models.ScheduleV2, by primitive.ObjectID) (err error) { + svc.mu.Lock() + defer svc.mu.Unlock() + + svc.cron.Remove(s.EntryId) + s.Enabled = false + s.EntryId = -1 + s.SetUpdated(by) + return svc.modelSvc.ReplaceById(s.Id, s) +} + +func (svc *ServiceV2) Update() { + for { + if svc.stopped { + return + } + + svc.update() + + time.Sleep(svc.updateInterval) + } +} + +func (svc *ServiceV2) GetCron() (c *cron.Cron) { + return svc.cron +} + +func (svc *ServiceV2) update() { + // fetch enabled schedules + if err := svc.fetch(); err != nil { + trace.PrintError(err) + return + } + + // entry id map + entryIdsMap := svc.getEntryIdsMap() + + // iterate enabled schedules + for _, s := range svc.schedules { + _, ok := entryIdsMap[s.EntryId] + if ok { + entryIdsMap[s.EntryId] = true + } else { + if !s.Enabled { + err := svc.Enable(s, s.GetCreatedBy()) + if err != nil { + trace.PrintError(err) + continue + } + } + } + } + + // remove non-existent entries + for id, ok := range entryIdsMap { + if !ok { + svc.cron.Remove(id) + } + } +} + +func (svc *ServiceV2) getEntryIdsMap() (res map[cron.EntryID]bool) { + res = map[cron.EntryID]bool{} + for _, e := range svc.cron.Entries() { + res[e.ID] = false + } + return res +} + +func (svc *ServiceV2) fetch() (err error) { + query := bson.M{ + "enabled": true, + } + svc.schedules, err = svc.modelSvc.GetMany(query, nil) + if err != nil { + return err + } + return nil +} + +func (svc *ServiceV2) schedule(id primitive.ObjectID) (fn func()) { + return func() { + // schedule + s, err := svc.modelSvc.GetById(id) + if err != nil { + trace.PrintError(err) + return + } + + // spider + spider, err := service.NewModelServiceV2[models.SpiderV2]().GetById(s.SpiderId) + if err != nil { + trace.PrintError(err) + return + } + + // options + opts := &interfaces.SpiderRunOptions{ + Mode: s.Mode, + NodeIds: s.NodeIds, + Cmd: s.Cmd, + Param: s.Param, + Priority: s.Priority, + ScheduleId: s.Id, + UserId: s.GetCreatedBy(), + } + + // normalize options + if opts.Mode == "" { + opts.Mode = spider.Mode + } + if len(opts.NodeIds) == 0 { + opts.NodeIds = spider.NodeIds + } + if opts.Cmd == "" { + opts.Cmd = spider.Cmd + } + if opts.Param == "" { + opts.Param = spider.Param + } + if opts.Priority == 0 { + if spider.Priority > 0 { + opts.Priority = spider.Priority + } else { + opts.Priority = 5 + } + } + + // schedule or assign a task in the task queue + if _, err := svc.adminSvc.Schedule(s.SpiderId, opts); err != nil { + trace.PrintError(err) + } + } +} + +func NewScheduleServiceV2() (svc2 *ServiceV2, err error) { + // service + svc := &ServiceV2{ + WithConfigPath: config.NewConfigPathService(), + loc: time.Local, + // TODO: implement delay and skip + delay: false, + skip: false, + updateInterval: 1 * time.Minute, + } + svc.adminSvc, err = admin.GetSpiderAdminServiceV2() + if err != nil { + return nil, err + } + svc.modelSvc = service.NewModelServiceV2[models.ScheduleV2]() + + // logger + svc.logger = NewLogger() + + // cron + svc.cron = cron.New( + cron.WithLogger(svc.logger), + cron.WithLocation(svc.loc), + cron.WithChain(cron.Recover(svc.logger)), + ) + + // initialize + if err := svc.Init(); err != nil { + return nil, err + } + + return svc, nil +} + +var svcV2 *ServiceV2 + +func GetScheduleServiceV2() (res *ServiceV2, err error) { + if svcV2 != nil { + return svcV2, nil + } + svcV2, err = NewScheduleServiceV2() + if err != nil { + return nil, err + } + return svcV2, nil +} diff --git a/core/schedule/test/base.go b/core/schedule/test/base.go new file mode 100644 index 000000000..cd7c885ed --- /dev/null +++ b/core/schedule/test/base.go @@ -0,0 +1,91 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/schedule" + "go.uber.org/dig" + "testing" +) + +func init() { + var err error + T, err = NewTest() + if err != nil { + panic(err) + } +} + +var T *Test + +type Test struct { + // dependencies + modelSvc service.ModelService + scheduleSvc interfaces.ScheduleService + + // test data + TestSchedule interfaces.Schedule + TestSpider interfaces.Spider + ScriptName string + Script string +} + +func (t *Test) Setup(t2 *testing.T) { + t.scheduleSvc.Start() + t2.Cleanup(t.Cleanup) +} + +func (t *Test) Cleanup() { + t.scheduleSvc.Stop() + _ = t.modelSvc.GetBaseService(interfaces.ModelIdTask).Delete(nil) +} + +func NewTest() (t *Test, err error) { + // test + t = &Test{ + TestSpider: &models.Spider{ + Name: "test_spider", + Cmd: "go run main.go", + }, + ScriptName: "main.go", + Script: `package main +import "fmt" +func main() { + fmt.Println("it works") +}`, + } + + // dependency injection + c := dig.New() + if err := c.Provide(service.GetService); err != nil { + return nil, err + } + if err := c.Provide(schedule.NewScheduleService); err != nil { + return nil, err + } + if err := c.Invoke(func(modelSvc service.ModelService, scheduleSvc interfaces.ScheduleService) { + t.modelSvc = modelSvc + t.scheduleSvc = scheduleSvc + }); err != nil { + return nil, err + } + + // add spider to db + if err := delegate.NewModelDelegate(t.TestSpider).Add(); err != nil { + return nil, err + } + + // test schedule + t.TestSchedule = &models.Schedule{ + Name: "test_schedule", + SpiderId: t.TestSpider.GetId(), + Cron: "* * * * *", + } + if err := delegate.NewModelDelegate(t.TestSchedule).Add(); err != nil { + return nil, err + } + + return t, nil +} diff --git a/core/schedule/test/schedule_service_test.go b/core/schedule/test/schedule_service_test.go new file mode 100644 index 000000000..7a815da1b --- /dev/null +++ b/core/schedule/test/schedule_service_test.go @@ -0,0 +1,44 @@ +package test + +import ( + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestScheduleService_Enable_Disable(t *testing.T) { + var err error + T.Setup(t) + + time.Sleep(1 * time.Second) + err = T.scheduleSvc.Enable(T.TestSchedule) + require.Nil(t, err) + time.Sleep(1 * time.Second) + + require.True(t, T.TestSchedule.GetEnabled()) + require.Greater(t, int(T.TestSchedule.GetEntryId()), -1) + e := T.scheduleSvc.GetCron().Entry(T.TestSchedule.GetEntryId()) + require.Equal(t, T.TestSchedule.GetEntryId(), e.ID) + time.Sleep(1 * time.Second) + + err = T.scheduleSvc.Disable(T.TestSchedule) + require.False(t, T.TestSchedule.GetEnabled()) + require.Equal(t, 0, len(T.scheduleSvc.GetCron().Entries())) +} + +func TestScheduleService_Run(t *testing.T) { + var err error + T.Setup(t) + + time.Sleep(1 * time.Second) + err = T.scheduleSvc.Enable(T.TestSchedule) + require.Nil(t, err) + time.Sleep(1 * time.Minute) + + tasks, err := T.modelSvc.GetTaskList(nil, nil) + require.Nil(t, err) + require.Greater(t, len(tasks), 0) + for _, task := range tasks { + require.False(t, task.ScheduleId.IsZero()) + } +} diff --git a/core/spider/admin/options.go b/core/spider/admin/options.go new file mode 100644 index 000000000..32e08e76e --- /dev/null +++ b/core/spider/admin/options.go @@ -0,0 +1,11 @@ +package admin + +import "github.com/crawlab-team/crawlab/core/interfaces" + +type Option func(svc interfaces.SpiderAdminService) + +func WithConfigPath(path string) Option { + return func(svc interfaces.SpiderAdminService) { + svc.SetConfigPath(path) + } +} diff --git a/core/spider/admin/service.go b/core/spider/admin/service.go new file mode 100644 index 000000000..83d6ac9e8 --- /dev/null +++ b/core/spider/admin/service.go @@ -0,0 +1,352 @@ +package admin + +import ( + "context" + "github.com/apex/log" + vcs "github.com/crawlab-team/crawlab-vcs" + config2 "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/google/uuid" + "github.com/robfig/cron/v3" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "os" + "path" + "path/filepath" + "sync" + "time" +) + +type Service struct { + // dependencies + nodeCfgSvc interfaces.NodeConfigService + modelSvc service.ModelService + schedulerSvc interfaces.TaskSchedulerService + cron *cron.Cron + syncLock bool + + // settings + cfgPath string +} + +func (svc *Service) GetConfigPath() (path string) { + return svc.cfgPath +} + +func (svc *Service) SetConfigPath(path string) { + svc.cfgPath = path +} + +func (svc *Service) Start() (err error) { + return svc.SyncGit() +} + +func (svc *Service) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { + // spider + s, err := svc.modelSvc.GetSpiderById(id) + if err != nil { + return nil, err + } + + // assign tasks + return svc.scheduleTasks(s, opts) +} + +func (svc *Service) Clone(id primitive.ObjectID, opts *interfaces.SpiderCloneOptions) (err error) { + // TODO: implement + return nil +} + +func (svc *Service) Delete(id primitive.ObjectID) (err error) { + panic("implement me") +} + +func (svc *Service) SyncGit() (err error) { + if _, err = svc.cron.AddFunc("* * * * *", svc.syncGit); err != nil { + return trace.TraceError(err) + } + svc.cron.Start() + return nil +} + +func (svc *Service) SyncGitOne(g interfaces.Git) (err error) { + svc.syncGitOne(g) + return nil +} + +func (svc *Service) Export(id primitive.ObjectID) (filePath string, err error) { + // spider fs + workspacePath := viper.GetString("workspace") + spiderFolderPath := filepath.Join(workspacePath, id.Hex()) + + // zip files in workspace + dirPath := spiderFolderPath + zipFilePath := path.Join(os.TempDir(), uuid.New().String()+".zip") + if err := utils.ZipDirectory(dirPath, zipFilePath); err != nil { + return "", trace.TraceError(err) + } + + return zipFilePath, nil +} + +func (svc *Service) scheduleTasks(s *models.Spider, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { + // main task + mainTask := &models.Task{ + SpiderId: s.Id, + Mode: opts.Mode, + NodeIds: opts.NodeIds, + Cmd: opts.Cmd, + Param: opts.Param, + ScheduleId: opts.ScheduleId, + Priority: opts.Priority, + UserId: opts.UserId, + CreateTs: time.Now(), + } + + // normalize + if mainTask.Mode == "" { + mainTask.Mode = s.Mode + } + if mainTask.NodeIds == nil { + mainTask.NodeIds = s.NodeIds + } + if mainTask.Cmd == "" { + mainTask.Cmd = s.Cmd + } + if mainTask.Param == "" { + mainTask.Param = s.Param + } + if mainTask.Priority == 0 { + mainTask.Priority = s.Priority + } + + if svc.isMultiTask(opts) { + // multi tasks + nodeIds, err := svc.getNodeIds(opts) + if err != nil { + return nil, err + } + for _, nodeId := range nodeIds { + t := &models.Task{ + SpiderId: s.Id, + Mode: opts.Mode, + Cmd: opts.Cmd, + Param: opts.Param, + NodeId: nodeId, + ScheduleId: opts.ScheduleId, + Priority: opts.Priority, + UserId: opts.UserId, + CreateTs: time.Now(), + } + t2, err := svc.schedulerSvc.Enqueue(t) + if err != nil { + return nil, err + } + taskIds = append(taskIds, t2.GetId()) + } + } else { + // single task + nodeIds, err := svc.getNodeIds(opts) + if err != nil { + return nil, err + } + if len(nodeIds) > 0 { + mainTask.NodeId = nodeIds[0] + } + t2, err := svc.schedulerSvc.Enqueue(mainTask) + if err != nil { + return nil, err + } + taskIds = append(taskIds, t2.GetId()) + } + + return taskIds, nil +} + +func (svc *Service) getNodeIds(opts *interfaces.SpiderRunOptions) (nodeIds []primitive.ObjectID, err error) { + if opts.Mode == constants.RunTypeAllNodes { + query := bson.M{ + "active": true, + "enabled": true, + "status": constants.NodeStatusOnline, + } + nodes, err := svc.modelSvc.GetNodeList(query, nil) + if err != nil { + return nil, err + } + for _, node := range nodes { + nodeIds = append(nodeIds, node.GetId()) + } + } else if opts.Mode == constants.RunTypeSelectedNodes { + nodeIds = opts.NodeIds + } + return nodeIds, nil +} + +func (svc *Service) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool) { + if opts.Mode == constants.RunTypeAllNodes { + query := bson.M{ + "active": true, + "enabled": true, + "status": constants.NodeStatusOnline, + } + nodes, err := svc.modelSvc.GetNodeList(query, nil) + if err != nil { + trace.PrintError(err) + return false + } + return len(nodes) > 1 + } else if opts.Mode == constants.RunTypeRandom { + return false + } else if opts.Mode == constants.RunTypeSelectedNodes { + return len(opts.NodeIds) > 1 + } else { + return false + } +} + +func (svc *Service) syncGit() { + if svc.syncLock { + log.Infof("[SpiderAdminService] sync git is locked, skip") + return + } + log.Infof("[SpiderAdminService] start to sync git") + + svc.syncLock = true + defer func() { + svc.syncLock = false + }() + + // spiders + spiders, err := svc.modelSvc.GetSpiderList(nil, nil) + if err != nil { + trace.PrintError(err) + return + } + + // spider ids + var spiderIds []primitive.ObjectID + for _, s := range spiders { + spiderIds = append(spiderIds, s.Id) + } + + if len(spiderIds) > 0 { + // gits + gits, err := svc.modelSvc.GetGitList(bson.M{ + "_id": bson.M{ + "$in": spiderIds, + }, + "auto_pull": true, + }, nil) + if err != nil { + trace.PrintError(err) + return + } + + wg := sync.WaitGroup{} + wg.Add(len(gits)) + for _, g := range gits { + go func(g models.Git) { + svc.syncGitOne(&g) + wg.Done() + }(g) + } + wg.Wait() + } + + log.Infof("[SpiderAdminService] finished sync git") +} + +func (svc *Service) syncGitOne(g interfaces.Git) { + log.Infof("[SpiderAdminService] sync git %s", g.GetId()) + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + // git client + workspacePath := viper.GetString("workspace") + gitClient, err := vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, g.GetId().Hex()))) + if err != nil { + return + } + + // set auth + utils.InitGitClientAuth(g, gitClient) + + // check if remote has changes + ok, err := gitClient.IsRemoteChanged() + if err != nil { + trace.PrintError(err) + return + } + if !ok { + // no change + return + } + + // pull and sync to workspace + if err := gitClient.Reset(); err != nil { + trace.PrintError(err) + return + } + if err := gitClient.Pull(); err != nil { + trace.PrintError(err) + return + } + + // wait for context to end + <-ctx.Done() +} + +func NewSpiderAdminService(opts ...Option) (svc2 interfaces.SpiderAdminService, err error) { + svc := &Service{ + cfgPath: config2.GetConfigPath(), + } + + // apply options + for _, opt := range opts { + opt(svc) + } + + // dependency injection + if err := container.GetContainer().Invoke(func(nodeCfgSvc interfaces.NodeConfigService, modelSvc service.ModelService, schedulerSvc interfaces.TaskSchedulerService) { + svc.nodeCfgSvc = nodeCfgSvc + svc.modelSvc = modelSvc + svc.schedulerSvc = schedulerSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + // cron + svc.cron = cron.New() + + // validate node type + if !svc.nodeCfgSvc.IsMaster() { + return nil, trace.TraceError(errors.ErrorSpiderForbidden) + } + + return svc, nil +} + +var _service interfaces.SpiderAdminService + +func GetSpiderAdminService() (svc2 interfaces.SpiderAdminService, err error) { + if _service != nil { + return _service, nil + } + + _service, err = NewSpiderAdminService() + if err != nil { + return nil, err + } + + return _service, nil +} diff --git a/core/spider/admin/service_v2.go b/core/spider/admin/service_v2.go new file mode 100644 index 000000000..73a845965 --- /dev/null +++ b/core/spider/admin/service_v2.go @@ -0,0 +1,328 @@ +package admin + +import ( + "context" + "github.com/apex/log" + vcs "github.com/crawlab-team/crawlab-vcs" + config2 "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/core/task/scheduler" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/google/uuid" + "github.com/robfig/cron/v3" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "os" + "path" + "path/filepath" + "sync" + "time" +) + +type ServiceV2 struct { + // dependencies + nodeCfgSvc interfaces.NodeConfigService + schedulerSvc *scheduler.ServiceV2 + cron *cron.Cron + syncLock bool + + // settings + cfgPath string +} + +func (svc *ServiceV2) Start() (err error) { + return svc.SyncGit() +} + +func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { + // spider + s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + if err != nil { + return nil, err + } + + // assign tasks + return svc.scheduleTasks(s, opts) +} + +func (svc *ServiceV2) SyncGit() (err error) { + if _, err = svc.cron.AddFunc("* * * * *", svc.syncGit); err != nil { + return trace.TraceError(err) + } + svc.cron.Start() + return nil +} + +func (svc *ServiceV2) SyncGitOne(g *models.GitV2) (err error) { + svc.syncGitOne(g) + return nil +} + +func (svc *ServiceV2) Export(id primitive.ObjectID) (filePath string, err error) { + // spider fs + workspacePath := viper.GetString("workspace") + spiderFolderPath := filepath.Join(workspacePath, id.Hex()) + + // zip files in workspace + dirPath := spiderFolderPath + zipFilePath := path.Join(os.TempDir(), uuid.New().String()+".zip") + if err := utils.ZipDirectory(dirPath, zipFilePath); err != nil { + return "", trace.TraceError(err) + } + + return zipFilePath, nil +} + +func (svc *ServiceV2) scheduleTasks(s *models.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { + // main task + mainTask := &models.TaskV2{ + SpiderId: s.Id, + Mode: opts.Mode, + NodeIds: opts.NodeIds, + Cmd: opts.Cmd, + Param: opts.Param, + ScheduleId: opts.ScheduleId, + Priority: opts.Priority, + UserId: opts.UserId, + CreateTs: time.Now(), + } + mainTask.SetId(primitive.NewObjectID()) + + // normalize + if mainTask.Mode == "" { + mainTask.Mode = s.Mode + } + if mainTask.NodeIds == nil { + mainTask.NodeIds = s.NodeIds + } + if mainTask.Cmd == "" { + mainTask.Cmd = s.Cmd + } + if mainTask.Param == "" { + mainTask.Param = s.Param + } + if mainTask.Priority == 0 { + mainTask.Priority = s.Priority + } + + if svc.isMultiTask(opts) { + // multi tasks + nodeIds, err := svc.getNodeIds(opts) + if err != nil { + return nil, err + } + for _, nodeId := range nodeIds { + t := &models.TaskV2{ + SpiderId: s.Id, + Mode: opts.Mode, + Cmd: opts.Cmd, + Param: opts.Param, + NodeId: nodeId, + ScheduleId: opts.ScheduleId, + Priority: opts.Priority, + UserId: opts.UserId, + CreateTs: time.Now(), + } + t.SetId(primitive.NewObjectID()) + t2, err := svc.schedulerSvc.Enqueue(t, opts.UserId) + if err != nil { + return nil, err + } + taskIds = append(taskIds, t2.Id) + } + } else { + // single task + nodeIds, err := svc.getNodeIds(opts) + if err != nil { + return nil, err + } + if len(nodeIds) > 0 { + mainTask.NodeId = nodeIds[0] + } + t2, err := svc.schedulerSvc.Enqueue(mainTask, opts.UserId) + if err != nil { + return nil, err + } + taskIds = append(taskIds, t2.Id) + } + + return taskIds, nil +} + +func (svc *ServiceV2) getNodeIds(opts *interfaces.SpiderRunOptions) (nodeIds []primitive.ObjectID, err error) { + if opts.Mode == constants.RunTypeAllNodes { + query := bson.M{ + "active": true, + "enabled": true, + "status": constants.NodeStatusOnline, + } + nodes, err := service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + if err != nil { + return nil, err + } + for _, node := range nodes { + nodeIds = append(nodeIds, node.Id) + } + } else if opts.Mode == constants.RunTypeSelectedNodes { + nodeIds = opts.NodeIds + } + return nodeIds, nil +} + +func (svc *ServiceV2) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool) { + if opts.Mode == constants.RunTypeAllNodes { + query := bson.M{ + "active": true, + "enabled": true, + "status": constants.NodeStatusOnline, + } + nodes, err := service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + if err != nil { + trace.PrintError(err) + return false + } + return len(nodes) > 1 + } else if opts.Mode == constants.RunTypeRandom { + return false + } else if opts.Mode == constants.RunTypeSelectedNodes { + return len(opts.NodeIds) > 1 + } else { + return false + } +} + +func (svc *ServiceV2) syncGit() { + if svc.syncLock { + log.Infof("[SpiderAdminService] sync git is locked, skip") + return + } + log.Infof("[SpiderAdminService] start to sync git") + + svc.syncLock = true + defer func() { + svc.syncLock = false + }() + + // spiders + spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(nil, nil) + if err != nil { + trace.PrintError(err) + return + } + + // spider ids + var spiderIds []primitive.ObjectID + for _, s := range spiders { + spiderIds = append(spiderIds, s.Id) + } + + if len(spiderIds) > 0 { + // gits + gits, err := service.NewModelServiceV2[models.GitV2]().GetMany(bson.M{ + "_id": bson.M{ + "$in": spiderIds, + }, + "auto_pull": true, + }, nil) + if err != nil { + trace.PrintError(err) + return + } + + wg := sync.WaitGroup{} + wg.Add(len(gits)) + for _, g := range gits { + go func(g models.GitV2) { + svc.syncGitOne(&g) + wg.Done() + }(g) + } + wg.Wait() + } + + log.Infof("[SpiderAdminService] finished sync git") +} + +func (svc *ServiceV2) syncGitOne(g *models.GitV2) { + log.Infof("[SpiderAdminService] sync git %s", g.Id) + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + // git client + workspacePath := viper.GetString("workspace") + gitClient, err := vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, g.Id.Hex()))) + if err != nil { + return + } + + // set auth + utils.InitGitClientAuthV2(g, gitClient) + + // check if remote has changes + ok, err := gitClient.IsRemoteChanged() + if err != nil { + trace.PrintError(err) + return + } + if !ok { + // no change + return + } + + // pull and sync to workspace + if err := gitClient.Reset(); err != nil { + trace.PrintError(err) + return + } + if err := gitClient.Pull(); err != nil { + trace.PrintError(err) + return + } + + // wait for context to end + <-ctx.Done() +} + +func NewSpiderAdminServiceV2() (svc2 *ServiceV2, err error) { + svc := &ServiceV2{ + nodeCfgSvc: config.GetNodeConfigService(), + cfgPath: config2.GetConfigPath(), + } + svc.schedulerSvc, err = scheduler.GetTaskSchedulerServiceV2() + if err != nil { + return nil, err + } + + // cron + svc.cron = cron.New() + + // validate node type + if !svc.nodeCfgSvc.IsMaster() { + return nil, trace.TraceError(errors.ErrorSpiderForbidden) + } + + return svc, nil +} + +var svcV2 *ServiceV2 + +func GetSpiderAdminServiceV2() (svc2 *ServiceV2, err error) { + if svcV2 != nil { + return svcV2, nil + } + + svcV2, err = NewSpiderAdminServiceV2() + if err != nil { + return nil, err + } + + return svcV2, nil +} diff --git a/core/stats/options.go b/core/stats/options.go new file mode 100644 index 000000000..b40e16c30 --- /dev/null +++ b/core/stats/options.go @@ -0,0 +1,5 @@ +package stats + +import "github.com/crawlab-team/crawlab/core/interfaces" + +type Option func(svc interfaces.StatsService) diff --git a/core/stats/service.go b/core/stats/service.go new file mode 100644 index 000000000..17df1be9a --- /dev/null +++ b/core/stats/service.go @@ -0,0 +1,323 @@ +package stats + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" +) + +type Service struct { +} + +func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) { + stats := bson.M{} + + // nodes + stats["nodes"], err = mongo.GetMongoCol(interfaces.ModelColNameNode).Count(bson.M{"active": true}) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["nodes"] = 0 + } + + // projects + stats["projects"], err = mongo.GetMongoCol(interfaces.ModelColNameProject).Count(nil) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["projects"] = 0 + } + + // spiders + stats["spiders"], err = mongo.GetMongoCol(interfaces.ModelColNameSpider).Count(nil) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["spiders"] = 0 + } + + // schedules + stats["schedules"], err = mongo.GetMongoCol(interfaces.ModelColNameSchedule).Count(nil) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["schedules"] = 0 + } + + // tasks + stats["tasks"], err = mongo.GetMongoCol(interfaces.ModelColNameTask).Count(nil) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["tasks"] = 0 + } + + // error tasks + stats["error_tasks"], err = mongo.GetMongoCol(interfaces.ModelColNameTask).Count(bson.M{"status": constants.TaskStatusError}) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["error_tasks"] = 0 + } + + // results + stats["results"], err = svc.getOverviewResults(query) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["results"] = 0 + } + + // users + stats["users"], err = mongo.GetMongoCol(interfaces.ModelColNameUser).Count(nil) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["users"] = 0 + } + + return stats, nil +} + +func (svc *Service) GetDailyStats(query bson.M) (data interface{}, err error) { + tasksStats, err := svc.getDailyTasksStats(query) + if err != nil { + return nil, err + } + return tasksStats, nil +} + +func (svc *Service) GetTaskStats(query bson.M) (data interface{}, err error) { + stats := bson.M{} + + // by status + stats["by_status"], err = svc.getTaskStatsByStatus(query) + if err != nil { + return nil, err + } + + // by node + stats["by_node"], err = svc.getTaskStatsByNode(query) + if err != nil { + return nil, err + } + + // by spider + stats["by_spider"], err = svc.getTaskStatsBySpider(query) + if err != nil { + return nil, err + } + + return stats, nil +} + +func (svc *Service) getDailyTasksStats(query bson.M) (data interface{}, err error) { + pipeline := mongo2.Pipeline{ + {{ + "$match", query, + }}, + {{ + "$addFields", + bson.M{ + "date": bson.M{ + "$dateToString": bson.M{ + "date": bson.M{"$toDate": "$_id"}, + "format": "%Y-%m-%d", + "timezone": "Asia/Shanghai", // TODO: parameterization + }, + }, + }, + }}, + {{ + "$group", + bson.M{ + "_id": "$date", + "tasks": bson.M{"$sum": 1}, + "results": bson.M{"$sum": "$result_count"}, + }, + }}, + {{ + "$sort", + bson.D{{"_id", 1}}, + }}, + } + var results []entity.StatsDailyItem + if err := mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Aggregate(pipeline, nil).All(&results); err != nil { + return nil, err + } + return results, nil +} + +func (svc *Service) getOverviewResults(query bson.M) (data interface{}, err error) { + pipeline := mongo2.Pipeline{ + {{"$match", query}}, + {{ + "$group", + bson.M{ + "_id": nil, + "results": bson.M{"$sum": "$result_count"}, + }, + }}, + } + var res bson.M + if err := mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Aggregate(pipeline, nil).One(&res); err != nil { + return nil, err + } + return res["results"], nil +} + +func (svc *Service) getTaskStatsByStatus(query bson.M) (data interface{}, err error) { + pipeline := mongo2.Pipeline{ + {{"$match", query}}, + {{ + "$group", + bson.M{ + "_id": "$status", + "tasks": bson.M{"$sum": 1}, + }, + }}, + {{ + "$project", + bson.M{ + "status": "$_id", + "tasks": "$tasks", + }, + }}, + } + var results []bson.M + if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).All(&results); err != nil { + return nil, err + } + return results, nil +} + +func (svc *Service) getTaskStatsByNode(query bson.M) (data interface{}, err error) { + pipeline := mongo2.Pipeline{ + {{"$match", query}}, + {{ + "$group", + bson.M{ + "_id": "$node_id", + "tasks": bson.M{"$sum": 1}, + }, + }}, + {{ + "$lookup", + bson.M{ + "from": interfaces.ModelColNameNode, + "localField": "_id", + "foreignField": "_id", + "as": "_n", + }, + }}, + {{ + "$project", + bson.M{ + "node_id": "$node_id", + "node": bson.M{"$arrayElemAt": bson.A{"$_n", 0}}, + "node_name": bson.M{"$arrayElemAt": bson.A{"$_n.name", 0}}, + "tasks": "$tasks", + }, + }}, + } + var results []bson.M + if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).All(&results); err != nil { + return nil, err + } + return results, nil +} + +func (svc *Service) getTaskStatsBySpider(query bson.M) (data interface{}, err error) { + pipeline := mongo2.Pipeline{ + {{"$match", query}}, + {{ + "$group", + bson.M{ + "_id": "$spider_id", + "tasks": bson.M{"$sum": 1}, + }, + }}, + {{ + "$lookup", + bson.M{ + "from": interfaces.ModelColNameSpider, + "localField": "_id", + "foreignField": "_id", + "as": "_s", + }, + }}, + {{ + "$project", + bson.M{ + "spider_id": "$spider_id", + "spider": bson.M{"$arrayElemAt": bson.A{"$_s", 0}}, + "spider_name": bson.M{"$arrayElemAt": bson.A{"$_s.name", 0}}, + "tasks": "$tasks", + }, + }}, + {{"$limit", 10}}, + } + var results []bson.M + if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).All(&results); err != nil { + return nil, err + } + return results, nil +} + +func (svc *Service) getTaskStatsHistogram(query bson.M) (data interface{}, err error) { + pipeline := mongo2.Pipeline{ + {{"$match", query}}, + {{ + "$lookup", + bson.M{ + "from": interfaces.ModelColNameTaskStat, + "localField": "_id", + "foreignField": "_id", + "as": "_ts", + }, + }}, + {{ + "$facet", + bson.M{ + "total_duration": bson.A{ + bson.M{ + "$bucketAuto": bson.M{ + "groupBy": "$_ts.td", + "buckets": 10, + "granularity": "1-2-5", + }, + }, + }, + }, + }}, + } + var res bson.M + if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).One(&res); err != nil { + return nil, err + } + return res, nil +} + +var svc interfaces.StatsService + +func GetStatsService() interfaces.StatsService { + if svc != nil { + return svc + } + + // service + svc = &Service{} + + return svc +} diff --git a/core/sys_exec/sys_exec.go b/core/sys_exec/sys_exec.go new file mode 100644 index 000000000..397e4357e --- /dev/null +++ b/core/sys_exec/sys_exec.go @@ -0,0 +1,89 @@ +package sys_exec + +import ( + "bufio" + "github.com/crawlab-team/go-trace" + "github.com/shirou/gopsutil/process" + "os/exec" + "time" +) + +type KillProcessOptions struct { + Timeout time.Duration + Force bool +} + +func KillProcess(cmd *exec.Cmd, opts *KillProcessOptions) error { + // process + p, err := process.NewProcess(int32(cmd.Process.Pid)) + if err != nil { + return err + } + + // kill function + killFunc := func(p *process.Process) error { + return killProcessRecursive(p, opts.Force) + } + + if opts.Timeout != 0 { + // with timeout + return killProcessWithTimeout(p, opts.Timeout, killFunc) + } else { + // without timeout + return killFunc(p) + } +} + +func killProcessWithTimeout(p *process.Process, timeout time.Duration, killFunc func(*process.Process) error) error { + go func() { + if err := killFunc(p); err != nil { + trace.PrintError(err) + } + }() + for i := 0; i < int(timeout.Seconds()); i++ { + ok, err := process.PidExists(p.Pid) + if err == nil && !ok { + return nil + } + time.Sleep(1 * time.Second) + } + return killProcess(p, true) +} + +func killProcessRecursive(p *process.Process, force bool) (err error) { + // children processes + cps, err := p.Children() + if err != nil { + return killProcess(p, force) + } + + // iterate children processes + for _, cp := range cps { + if err := killProcessRecursive(cp, force); err != nil { + return err + } + } + + return nil +} + +func killProcess(p *process.Process, force bool) (err error) { + if force { + err = p.Kill() + } else { + err = p.Terminate() + } + if err != nil { + return trace.TraceError(err) + } + return nil +} + +func ConfigureCmdLogging(cmd *exec.Cmd, fn func(scanner *bufio.Scanner)) { + stdout, _ := (*cmd).StdoutPipe() + stderr, _ := (*cmd).StderrPipe() + scannerStdout := bufio.NewScanner(stdout) + scannerStderr := bufio.NewScanner(stderr) + go fn(scannerStdout) + go fn(scannerStderr) +} diff --git a/core/sys_exec/sys_exec_darwin.go b/core/sys_exec/sys_exec_darwin.go new file mode 100644 index 000000000..130b7cb82 --- /dev/null +++ b/core/sys_exec/sys_exec_darwin.go @@ -0,0 +1,24 @@ +//go:build darwin +// +build darwin + +package sys_exec + +import ( + "os/exec" + "syscall" +) + +func BuildCmd(cmdStr string) *exec.Cmd { + return exec.Command("sh", "-c", cmdStr) +} + +func SetPgid(cmd *exec.Cmd) { + if cmd == nil { + return + } + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + } else { + cmd.SysProcAttr.Setpgid = true + } +} diff --git a/core/sys_exec/sys_exec_linux.go b/core/sys_exec/sys_exec_linux.go new file mode 100644 index 000000000..bf532a43d --- /dev/null +++ b/core/sys_exec/sys_exec_linux.go @@ -0,0 +1,24 @@ +//go:build linux +// +build linux + +package sys_exec + +import ( + "os/exec" + "syscall" +) + +func BuildCmd(cmdStr string) *exec.Cmd { + return exec.Command("sh", "-c", cmdStr) +} + +func SetPgid(cmd *exec.Cmd) { + if cmd == nil { + return + } + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + } else { + cmd.SysProcAttr.Setpgid = true + } +} diff --git a/core/sys_exec/sys_exec_windows.go b/core/sys_exec/sys_exec_windows.go new file mode 100644 index 000000000..e2678ccd8 --- /dev/null +++ b/core/sys_exec/sys_exec_windows.go @@ -0,0 +1,10 @@ +//go:build windows +// +build windows + +package sys_exec + +import "os/exec" + +func BuildCmd(cmdStr string) *exec.Cmd { + return exec.Command("cmd", "/C", cmdStr) +} diff --git a/core/system/service.go b/core/system/service.go new file mode 100644 index 000000000..10846740a --- /dev/null +++ b/core/system/service.go @@ -0,0 +1,86 @@ +package system + +import ( + mongo2 "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Service struct { + col *mongo2.Col + modelSvc service.ModelService +} + +func (svc *Service) Init() (err error) { + // initialize data + if err := svc.initData(); err != nil { + return err + } + + return nil +} + +func (svc *Service) initData() (err error) { + total, err := svc.col.Count(bson.M{ + "key": "site_title", + }) + if err != nil { + return err + } + if total > 0 { + return nil + } + + // data to initialize + settings := []models.Setting{ + { + Id: primitive.NewObjectID(), + Key: "site_title", + Value: bson.M{ + "customize_site_title": false, + "site_title": "", + }, + }, + } + var data []interface{} + for _, s := range settings { + data = append(data, s) + } + _, err = svc.col.InsertMany(data) + if err != nil { + return err + } + return nil +} + +func NewService() *Service { + // service + svc := &Service{ + col: mongo2.GetMongoCol(interfaces.ModelColNameSetting), + } + + // model service + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + svc.modelSvc = modelSvc + + if err := svc.Init(); err != nil { + panic(err) + } + + return svc +} + +var _service *Service + +func GetService() *Service { + if _service == nil { + _service = NewService() + } + return _service +} diff --git a/core/task/base.go b/core/task/base.go new file mode 100644 index 000000000..265186a3a --- /dev/null +++ b/core/task/base.go @@ -0,0 +1,97 @@ +package task + +import ( + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" +) + +type BaseService struct { + // dependencies + interfaces.WithConfigPath + modelSvc service.ModelService + + // internals + stopped bool +} + +func (svc *BaseService) Init() error { + // implement me + return nil +} + +func (svc *BaseService) Start() { + // implement me +} + +func (svc *BaseService) Wait() { + utils.DefaultWait() +} + +func (svc *BaseService) Stop() { + svc.stopped = true +} + +// SaveTask deprecated +func (svc *BaseService) SaveTask(t interfaces.Task, status string) (err error) { + // normalize status + if status == "" { + status = constants.TaskStatusPending + } + + // set task status + t.SetStatus(status) + + // attempt to get task from database + _, err = svc.modelSvc.GetTaskById(t.GetId()) + if err != nil { + // if task does not exist, add to database + if err == mongo.ErrNoDocuments { + if err := delegate.NewModelDelegate(t).Add(); err != nil { + return err + } + return nil + } else { + return err + } + } else { + // otherwise, update + if err := delegate.NewModelDelegate(t).Save(); err != nil { + return err + } + return nil + } +} + +func (svc *BaseService) IsStopped() (res bool) { + return svc.stopped +} + +func (svc *BaseService) GetQueue(nodeId primitive.ObjectID) (queue string) { + if nodeId.IsZero() { + return fmt.Sprintf("%s", constants.TaskListQueuePrefixPublic) + } else { + return fmt.Sprintf("%s:%s", constants.TaskListQueuePrefixNodes, nodeId.Hex()) + } +} + +func NewBaseService() (svc2 interfaces.TaskBaseService, err error) { + svc := &BaseService{} + + // dependency injection + if err := container.GetContainer().Invoke(func(cfgPath interfaces.WithConfigPath, modelSvc service.ModelService) { + svc.WithConfigPath = cfgPath + svc.modelSvc = modelSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + return svc, nil +} diff --git a/core/task/handler/options.go b/core/task/handler/options.go new file mode 100644 index 000000000..c2c035581 --- /dev/null +++ b/core/task/handler/options.go @@ -0,0 +1,40 @@ +package handler + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type Option func(svc interfaces.TaskHandlerService) + +func WithConfigPath(path string) Option { + return func(svc interfaces.TaskHandlerService) { + svc.SetConfigPath(path) + } +} + +func WithExitWatchDuration(duration time.Duration) Option { + return func(svc interfaces.TaskHandlerService) { + svc.SetExitWatchDuration(duration) + } +} + +func WithReportInterval(interval time.Duration) Option { + return func(svc interfaces.TaskHandlerService) { + svc.SetReportInterval(interval) + } +} + +func WithCancelTimeout(timeout time.Duration) Option { + return func(svc interfaces.TaskHandlerService) { + svc.SetCancelTimeout(timeout) + } +} + +type RunnerOption func(r interfaces.TaskRunner) + +func WithSubscribeTimeout(timeout time.Duration) RunnerOption { + return func(r interfaces.TaskRunner) { + r.SetSubscribeTimeout(timeout) + } +} diff --git a/core/task/handler/runner.go b/core/task/handler/runner.go new file mode 100644 index 000000000..119309100 --- /dev/null +++ b/core/task/handler/runner.go @@ -0,0 +1,691 @@ +package handler + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "github.com/apex/log" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + fs2 "github.com/crawlab-team/crawlab/core/fs" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/client" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/sys_exec" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "github.com/shirou/gopsutil/process" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +type Runner struct { + // dependencies + svc interfaces.TaskHandlerService // task handler service + fsSvc interfaces.FsServiceV2 // task fs service + hookSvc interfaces.TaskHookService // task hook service + + // settings + subscribeTimeout time.Duration + bufferSize int + + // internals + cmd *exec.Cmd // process command instance + pid int // process id + tid primitive.ObjectID // task id + t interfaces.Task // task model.Task + s interfaces.Spider // spider model.Spider + ch chan constants.TaskSignal // channel to communicate between Service and Runner + err error // standard process error + envs []models.Env // environment variables + cwd string // working directory + c interfaces.GrpcClient // grpc client + sub grpc.TaskService_SubscribeClient // grpc task service stream client + + // log internals + scannerStdout *bufio.Reader + scannerStderr *bufio.Reader + logBatchSize int +} + +func (r *Runner) Init() (err error) { + // update task + if err := r.updateTask("", nil); err != nil { + return err + } + + // start grpc client + if !r.c.IsStarted() { + r.c.Start() + } + + // working directory + workspacePath := viper.GetString("workspace") + r.cwd = filepath.Join(workspacePath, r.s.GetId().Hex()) + + // sync files from master + if !utils.IsMaster() { + if err := r.syncFiles(); err != nil { + return err + } + } + + // grpc task service stream client + if err := r.initSub(); err != nil { + return err + } + + // pre actions + if r.hookSvc != nil { + if err := r.hookSvc.PreActions(r.t, r.s, r.fsSvc, r.svc); err != nil { + return err + } + } + + return nil +} + +func (r *Runner) Run() (err error) { + // log task started + log.Infof("task[%s] started", r.tid.Hex()) + + // configure cmd + r.configureCmd() + + // configure environment variables + r.configureEnv() + + // configure logging + r.configureLogging() + + // start process + if err := r.cmd.Start(); err != nil { + return r.updateTask(constants.TaskStatusError, err) + } + + // start logging + go r.startLogging() + + // process id + if r.cmd.Process == nil { + return r.updateTask(constants.TaskStatusError, constants.ErrNotExists) + } + r.pid = r.cmd.Process.Pid + r.t.SetPid(r.pid) + + // update task status (processing) + if err := r.updateTask(constants.TaskStatusRunning, nil); err != nil { + return err + } + + // wait for process to finish + go r.wait() + + // start health check + go r.startHealthCheck() + + // declare task status + status := "" + + // wait for signal + signal := <-r.ch + switch signal { + case constants.TaskSignalFinish: + err = nil + status = constants.TaskStatusFinished + case constants.TaskSignalCancel: + err = constants.ErrTaskCancelled + status = constants.TaskStatusCancelled + case constants.TaskSignalError: + err = r.err + status = constants.TaskStatusError + case constants.TaskSignalLost: + err = constants.ErrTaskLost + status = constants.TaskStatusError + default: + err = constants.ErrInvalidSignal + status = constants.TaskStatusError + } + + // update task status + if err := r.updateTask(status, err); err != nil { + return err + } + + // post actions + if r.hookSvc != nil { + if err := r.hookSvc.PostActions(r.t, r.s, r.fsSvc, r.svc); err != nil { + return err + } + } + + return err +} + +func (r *Runner) Cancel() (err error) { + // kill process + opts := &sys_exec.KillProcessOptions{ + Timeout: r.svc.GetCancelTimeout(), + Force: true, + } + if err := sys_exec.KillProcess(r.cmd, opts); err != nil { + return err + } + + // make sure the process does not exist + op := func() error { + if exists, _ := process.PidExists(int32(r.pid)); exists { + return errors.ErrorTaskProcessStillExists + } + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), r.svc.GetExitWatchDuration()) + defer cancel() + b := backoff.WithContext(backoff.NewConstantBackOff(1*time.Second), ctx) + if err := backoff.Retry(op, b); err != nil { + return trace.TraceError(errors.ErrorTaskUnableToCancel) + } + + return nil +} + +// CleanUp clean up task runner +func (r *Runner) CleanUp() (err error) { + return nil +} + +func (r *Runner) SetSubscribeTimeout(timeout time.Duration) { + r.subscribeTimeout = timeout +} + +func (r *Runner) GetTaskId() (id primitive.ObjectID) { + return r.tid +} + +func (r *Runner) configureCmd() { + var cmdStr string + + // customized spider + if r.t.GetCmd() == "" { + cmdStr = r.s.GetCmd() + } else { + cmdStr = r.t.GetCmd() + } + + // parameters + if r.t.GetParam() != "" { + cmdStr += " " + r.t.GetParam() + } else if r.s.GetParam() != "" { + cmdStr += " " + r.s.GetParam() + } + + // get cmd instance + r.cmd = sys_exec.BuildCmd(cmdStr) + + // set working directory + r.cmd.Dir = r.cwd + + // configure pgid to allow killing sub processes + //sys_exec.SetPgid(r.cmd) +} + +func (r *Runner) configureLogging() { + // set stdout reader + stdout, _ := r.cmd.StdoutPipe() + r.scannerStdout = bufio.NewReaderSize(stdout, r.bufferSize) + + // set stderr reader + stderr, _ := r.cmd.StderrPipe() + r.scannerStderr = bufio.NewReaderSize(stderr, r.bufferSize) +} + +func (r *Runner) startLogging() { + // start reading stdout + go r.startLoggingReaderStdout() + + // start reading stderr + go r.startLoggingReaderStderr() +} + +func (r *Runner) startLoggingReaderStdout() { + for { + line, err := r.scannerStdout.ReadString(byte('\n')) + if err != nil { + break + } + line = strings.TrimSuffix(line, "\n") + r.writeLogLines([]string{line}) + } +} + +func (r *Runner) startLoggingReaderStderr() { + for { + line, err := r.scannerStderr.ReadString(byte('\n')) + if err != nil { + break + } + line = strings.TrimSuffix(line, "\n") + r.writeLogLines([]string{line}) + } +} + +func (r *Runner) startHealthCheck() { + if r.cmd.ProcessState == nil || r.cmd.ProcessState.Exited() { + return + } + for { + exists, _ := process.PidExists(int32(r.pid)) + if !exists { + // process lost + r.ch <- constants.TaskSignalLost + return + } + time.Sleep(1 * time.Second) + } +} + +func (r *Runner) configureEnv() { + // 默认把Node.js的全局node_modules加入环境变量 + envPath := os.Getenv("PATH") + nodePath := "/usr/lib/node_modules" + if !strings.Contains(envPath, nodePath) { + _ = os.Setenv("PATH", nodePath+":"+envPath) + } + _ = os.Setenv("NODE_PATH", nodePath) + + // default envs + r.cmd.Env = append(os.Environ(), "CRAWLAB_TASK_ID="+r.tid.Hex()) + if viper.GetString("grpc.address") != "" { + r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_ADDRESS="+viper.GetString("grpc.address")) + } + if viper.GetString("grpc.authKey") != "" { + r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_AUTH_KEY="+viper.GetString("grpc.authKey")) + } else { + r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_AUTH_KEY="+constants.DefaultGrpcAuthKey) + } + + // global environment variables + envs, err := r.svc.GetModelEnvironmentService().GetEnvironmentList(nil, nil) + if err != nil { + trace.PrintError(err) + return + } + for _, env := range envs { + r.cmd.Env = append(r.cmd.Env, env.GetKey()+"="+env.GetValue()) + } +} + +func (r *Runner) syncFiles() (err error) { + masterURL := fmt.Sprintf("%s/sync/%s", viper.GetString("api.endpoint"), r.s.GetId().Hex()) + workspacePath := viper.GetString("workspace") + workerDir := filepath.Join(workspacePath, r.s.GetId().Hex()) + + // get file list from master + resp, err := http.Get(masterURL + "/scan") + if err != nil { + fmt.Println("Error getting file list from master:", err) + return trace.TraceError(err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body:", err) + return trace.TraceError(err) + } + var masterFiles map[string]entity.FsFileInfo + err = json.Unmarshal(body, &masterFiles) + if err != nil { + fmt.Println("Error unmarshaling JSON:", err) + return trace.TraceError(err) + } + + // create a map for master files + masterFilesMap := make(map[string]entity.FsFileInfo) + for _, file := range masterFiles { + masterFilesMap[file.Path] = file + } + + // create worker directory if not exists + if _, err := os.Stat(workerDir); os.IsNotExist(err) { + if err := os.MkdirAll(workerDir, os.ModePerm); err != nil { + fmt.Println("Error creating worker directory:", err) + return trace.TraceError(err) + } + } + + // get file list from worker + workerFiles, err := utils.ScanDirectory(workerDir) + if err != nil { + fmt.Println("Error scanning worker directory:", err) + return trace.TraceError(err) + } + + // set up wait group and error channel + var wg sync.WaitGroup + errCh := make(chan error, 1) + + // delete files that are deleted on master node + for path, workerFile := range workerFiles { + if _, exists := masterFilesMap[path]; !exists { + fmt.Println("Deleting file:", path) + err := os.Remove(workerFile.FullPath) + if err != nil { + fmt.Println("Error deleting file:", err) + } + } + } + + // download files that are new or modified on master node + for path, masterFile := range masterFilesMap { + workerFile, exists := workerFiles[path] + if !exists || masterFile.Hash != workerFile.Hash { + wg.Add(1) + go func(path string, masterFile entity.FsFileInfo) { + defer wg.Done() + logrus.Infof("File needs to be synchronized: %s", path) + err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(workerDir, path)) + if err != nil { + logrus.Errorf("Error downloading file: %v", err) + select { + case errCh <- err: + default: + } + } + }(path, masterFile) + } + } + + wg.Wait() + close(errCh) + if err := <-errCh; err != nil { + return err + } + + return nil +} + +func (r *Runner) downloadFile(url string, filePath string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + out, err := os.Create(filePath) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + return err +} + +// wait for process to finish and send task signal (constants.TaskSignal) +// to task runner's channel (Runner.ch) according to exit code +func (r *Runner) wait() { + // wait for process to finish + if err := r.cmd.Wait(); err != nil { + exitError, ok := err.(*exec.ExitError) + if !ok { + r.ch <- constants.TaskSignalError + return + } + exitCode := exitError.ExitCode() + if exitCode == -1 { + // cancel error + r.ch <- constants.TaskSignalCancel + return + } + + // standard error + r.err = err + r.ch <- constants.TaskSignalError + return + } + + // success + r.ch <- constants.TaskSignalFinish +} + +// updateTask update and get updated info of task (Runner.t) +func (r *Runner) updateTask(status string, e error) (err error) { + if r.t != nil && status != "" { + // update task status + r.t.SetStatus(status) + if e != nil { + r.t.SetError(e.Error()) + } + if r.svc.GetNodeConfigService().IsMaster() { + if err := delegate.NewModelDelegate(r.t).Save(); err != nil { + return err + } + } else { + if err := client.NewModelDelegate(r.t, client.WithDelegateConfigPath(r.svc.GetConfigPath())).Save(); err != nil { + return err + } + } + + // send notification + go r.sendNotification() + + // update stats + go func() { + r._updateTaskStat(status) + r._updateSpiderStat(status) + }() + } + + // get task + r.t, err = r.svc.GetTaskById(r.tid) + if err != nil { + return err + } + + return nil +} + +func (r *Runner) initSub() (err error) { + r.sub, err = r.c.GetTaskClient().Subscribe(context.Background()) + if err != nil { + return trace.TraceError(err) + } + return nil +} + +func (r *Runner) writeLogLines(lines []string) { + data, err := json.Marshal(&entity.StreamMessageTaskData{ + TaskId: r.tid, + Logs: lines, + }) + if err != nil { + trace.PrintError(err) + return + } + msg := &grpc.StreamMessage{ + Code: grpc.StreamMessageCode_INSERT_LOGS, + Data: data, + } + if err := r.sub.Send(msg); err != nil { + trace.PrintError(err) + return + } +} + +func (r *Runner) _updateTaskStat(status string) { + ts, err := r.svc.GetModelTaskStatService().GetTaskStatById(r.tid) + if err != nil { + trace.PrintError(err) + return + } + switch status { + case constants.TaskStatusPending: + // do nothing + case constants.TaskStatusRunning: + ts.SetStartTs(time.Now()) + ts.SetWaitDuration(ts.GetStartTs().Sub(ts.GetCreateTs()).Milliseconds()) + case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: + ts.SetEndTs(time.Now()) + ts.SetRuntimeDuration(ts.GetEndTs().Sub(ts.GetStartTs()).Milliseconds()) + ts.SetTotalDuration(ts.GetEndTs().Sub(ts.GetCreateTs()).Milliseconds()) + } + if r.svc.GetNodeConfigService().IsMaster() { + if err := delegate.NewModelDelegate(ts).Save(); err != nil { + trace.PrintError(err) + return + } + } else { + if err := client.NewModelDelegate(ts, client.WithDelegateConfigPath(r.svc.GetConfigPath())).Save(); err != nil { + trace.PrintError(err) + return + } + } +} + +func (r *Runner) sendNotification() { + data, err := json.Marshal(r.t) + if err != nil { + trace.PrintError(err) + return + } + req := &grpc.Request{ + NodeKey: r.svc.GetNodeConfigService().GetNodeKey(), + Data: data, + } + _, err = r.c.GetTaskClient().SendNotification(context.Background(), req) + if err != nil { + trace.PrintError(err) + return + } +} + +func (r *Runner) _updateSpiderStat(status string) { + // task stat + ts, err := r.svc.GetModelTaskStatService().GetTaskStatById(r.tid) + if err != nil { + trace.PrintError(err) + return + } + + // update + var update bson.M + switch status { + case constants.TaskStatusPending, constants.TaskStatusRunning: + update = bson.M{ + "$set": bson.M{ + "last_task_id": r.tid, // last task id + }, + "$inc": bson.M{ + "tasks": 1, // task count + "wait_duration": ts.GetWaitDuration(), // wait duration + }, + } + case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: + update = bson.M{ + "$inc": bson.M{ + "results": ts.GetResultCount(), // results + "runtime_duration": ts.GetRuntimeDuration() / 1000, // runtime duration + "total_duration": ts.GetTotalDuration() / 1000, // total duration + }, + } + default: + trace.PrintError(errors.ErrorTaskInvalidType) + return + } + + // perform update + if r.svc.GetNodeConfigService().IsMaster() { + if err := mongo.GetMongoCol(interfaces.ModelColNameSpiderStat).UpdateId(r.s.GetId(), update); err != nil { + trace.PrintError(err) + return + } + } else { + modelSvc, err := client.NewBaseServiceDelegate( + client.WithBaseServiceModelId(interfaces.ModelIdSpiderStat), + client.WithBaseServiceConfigPath(r.svc.GetConfigPath()), + ) + if err != nil { + trace.PrintError(err) + return + } + if err := modelSvc.UpdateById(r.s.GetId(), update); err != nil { + trace.PrintError(err) + return + } + } + +} + +func NewTaskRunner(id primitive.ObjectID, svc interfaces.TaskHandlerService, opts ...RunnerOption) (r2 interfaces.TaskRunner, err error) { + // validate options + if id.IsZero() { + return nil, constants.ErrInvalidOptions + } + + // runner + r := &Runner{ + subscribeTimeout: 30 * time.Second, + bufferSize: 1024 * 1024, + svc: svc, + tid: id, + ch: make(chan constants.TaskSignal), + logBatchSize: 20, + } + + // apply options + for _, opt := range opts { + opt(r) + } + + // task + r.t, err = svc.GetTaskById(id) + if err != nil { + return nil, err + } + + // spider + r.s, err = svc.GetSpiderById(r.t.GetSpiderId()) + if err != nil { + return nil, err + } + + // task fs service + r.fsSvc = fs2.NewFsServiceV2(filepath.Join(viper.GetString("workspace"), r.s.GetId().Hex())) + + // dependency injection + if err := container.GetContainer().Invoke(func( + c interfaces.GrpcClient, + ) { + r.c = c + }); err != nil { + return nil, trace.TraceError(err) + } + + _ = container.GetContainer().Invoke(func(hookSvc interfaces.TaskHookService) { + r.hookSvc = hookSvc + }) + + // initialize task runner + if err := r.Init(); err != nil { + return r, err + } + + return r, nil +} diff --git a/core/task/handler/runner_test.go b/core/task/handler/runner_test.go new file mode 100644 index 000000000..175117a3a --- /dev/null +++ b/core/task/handler/runner_test.go @@ -0,0 +1,103 @@ +package handler + +import ( + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/google/uuid" + "github.com/spf13/viper" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type MockRunner struct { + mock.Mock + Runner +} + +func (m *MockRunner) downloadFile(url string, filePath string) error { + args := m.Called(url, filePath) + return args.Error(0) +} + +func newMockRunner() *MockRunner { + r := &MockRunner{} + r.s = &models.Spider{} + return r +} + +func TestSyncFiles_SuccessWithDummyFiles(t *testing.T) { + // Create a test server that responds with a list of files + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/scan") { + w.Write([]byte(`{"file1.txt":{"path": "file1.txt", "hash": "hash1"}, "file2.txt":{"path": "file2.txt", "hash": "hash2"}}`)) + return + } + + if strings.HasSuffix(r.URL.Path, "/download") { + w.Write([]byte("file content")) + return + } + })) + defer ts.Close() + + // Create a mock runner + r := newMockRunner() + r.On("downloadFile", mock.Anything, mock.Anything).Return(nil) + + // Set the master URL to the test server URL + viper.Set("api.endpoint", ts.URL) + + localPath := filepath.Join(os.TempDir(), uuid.New().String()) + os.MkdirAll(filepath.Join(localPath, r.s.GetId().Hex()), os.ModePerm) + defer os.RemoveAll(localPath) + viper.Set("workspace", localPath) + + // Call the method under test + err := r.syncFiles() + assert.NoError(t, err) + + // Assert that the files were downloaded + assert.FileExists(t, filepath.Join(localPath, r.s.GetId().Hex(), "file1.txt")) + assert.FileExists(t, filepath.Join(localPath, r.s.GetId().Hex(), "file2.txt")) +} + +func TestSyncFiles_DeletesFilesNotOnMaster(t *testing.T) { + // Create a test server that responds with an empty list of files + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/scan") { + w.Write([]byte(`{}`)) + return + } + })) + defer ts.Close() + + // Create a mock runner + r := newMockRunner() + r.On("downloadFile", mock.Anything, mock.Anything).Return(nil) + + // Set the master URL to the test server URL + viper.Set("api.endpoint", ts.URL) + + localPath := filepath.Join(os.TempDir(), uuid.New().String()) + os.MkdirAll(filepath.Join(localPath, r.s.GetId().Hex()), os.ModePerm) + defer os.RemoveAll(localPath) + viper.Set("workspace", localPath) + + // Create a dummy file that should be deleted + dummyFilePath := filepath.Join(localPath, r.s.GetId().Hex(), "dummy.txt") + dummyFile, _ := os.Create(dummyFilePath) + dummyFile.Close() + + // Call the method under test + err := r.syncFiles() + assert.NoError(t, err) + + // Assert that the dummy file was deleted + assert.NoFileExists(t, dummyFilePath) +} diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go new file mode 100644 index 000000000..5818f25c9 --- /dev/null +++ b/core/task/handler/runner_v2.go @@ -0,0 +1,671 @@ +package handler + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "github.com/apex/log" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + fs2 "github.com/crawlab-team/crawlab/core/fs" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/client" + "github.com/crawlab-team/crawlab/core/models/models" + service2 "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/sys_exec" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "github.com/shirou/gopsutil/process" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +type RunnerV2 struct { + // dependencies + svc *ServiceV2 // task handler service + fsSvc interfaces.FsServiceV2 // task fs service + + // settings + subscribeTimeout time.Duration + bufferSize int + + // internals + cmd *exec.Cmd // process command instance + pid int // process id + tid primitive.ObjectID // task id + t *models.TaskV2 // task model.Task + s *models.SpiderV2 // spider model.Spider + ch chan constants.TaskSignal // channel to communicate between Service and RunnerV2 + err error // standard process error + envs []models.Env // environment variables + cwd string // working directory + c interfaces.GrpcClient // grpc client + sub grpc.TaskService_SubscribeClient // grpc task service stream client + + // log internals + scannerStdout *bufio.Reader + scannerStderr *bufio.Reader + logBatchSize int +} + +func (r *RunnerV2) Init() (err error) { + // update task + if err := r.updateTask("", nil); err != nil { + return err + } + + // start grpc client + if !r.c.IsStarted() { + r.c.Start() + } + + // working directory + workspacePath := viper.GetString("workspace") + r.cwd = filepath.Join(workspacePath, r.s.Id.Hex()) + + // sync files from master + if !utils.IsMaster() { + if err := r.syncFiles(); err != nil { + return err + } + } + + // grpc task service stream client + if err := r.initSub(); err != nil { + return err + } + + return nil +} + +func (r *RunnerV2) Run() (err error) { + // log task started + log.Infof("task[%s] started", r.tid.Hex()) + + // configure cmd + r.configureCmd() + + // configure environment variables + r.configureEnv() + + // configure logging + r.configureLogging() + + // start process + if err := r.cmd.Start(); err != nil { + return r.updateTask(constants.TaskStatusError, err) + } + + // start logging + go r.startLogging() + + // process id + if r.cmd.Process == nil { + return r.updateTask(constants.TaskStatusError, constants.ErrNotExists) + } + r.pid = r.cmd.Process.Pid + r.t.Pid = r.pid + + // update task status (processing) + if err := r.updateTask(constants.TaskStatusRunning, nil); err != nil { + return err + } + + // wait for process to finish + go r.wait() + + // start health check + go r.startHealthCheck() + + // declare task status + status := "" + + // wait for signal + signal := <-r.ch + switch signal { + case constants.TaskSignalFinish: + err = nil + status = constants.TaskStatusFinished + case constants.TaskSignalCancel: + err = constants.ErrTaskCancelled + status = constants.TaskStatusCancelled + case constants.TaskSignalError: + err = r.err + status = constants.TaskStatusError + case constants.TaskSignalLost: + err = constants.ErrTaskLost + status = constants.TaskStatusError + default: + err = constants.ErrInvalidSignal + status = constants.TaskStatusError + } + + // update task status + if err := r.updateTask(status, err); err != nil { + return err + } + + return err +} + +func (r *RunnerV2) Cancel() (err error) { + // kill process + opts := &sys_exec.KillProcessOptions{ + Timeout: r.svc.GetCancelTimeout(), + Force: true, + } + if err := sys_exec.KillProcess(r.cmd, opts); err != nil { + return err + } + + // make sure the process does not exist + op := func() error { + if exists, _ := process.PidExists(int32(r.pid)); exists { + return errors.ErrorTaskProcessStillExists + } + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), r.svc.GetExitWatchDuration()) + defer cancel() + b := backoff.WithContext(backoff.NewConstantBackOff(1*time.Second), ctx) + if err := backoff.Retry(op, b); err != nil { + return trace.TraceError(errors.ErrorTaskUnableToCancel) + } + + return nil +} + +// CleanUp clean up task runner +func (r *RunnerV2) CleanUp() (err error) { + return nil +} + +func (r *RunnerV2) SetSubscribeTimeout(timeout time.Duration) { + r.subscribeTimeout = timeout +} + +func (r *RunnerV2) GetTaskId() (id primitive.ObjectID) { + return r.tid +} + +func (r *RunnerV2) configureCmd() { + var cmdStr string + + // customized spider + if r.t.Cmd == "" { + cmdStr = r.s.Cmd + } else { + cmdStr = r.t.Cmd + } + + // parameters + if r.t.Param != "" { + cmdStr += " " + r.t.Param + } else if r.s.Param != "" { + cmdStr += " " + r.s.Param + } + + // get cmd instance + r.cmd = sys_exec.BuildCmd(cmdStr) + + // set working directory + r.cmd.Dir = r.cwd + + // configure pgid to allow killing sub processes + //sys_exec.SetPgid(r.cmd) +} + +func (r *RunnerV2) configureLogging() { + // set stdout reader + stdout, _ := r.cmd.StdoutPipe() + r.scannerStdout = bufio.NewReaderSize(stdout, r.bufferSize) + + // set stderr reader + stderr, _ := r.cmd.StderrPipe() + r.scannerStderr = bufio.NewReaderSize(stderr, r.bufferSize) +} + +func (r *RunnerV2) startLogging() { + // start reading stdout + go r.startLoggingReaderStdout() + + // start reading stderr + go r.startLoggingReaderStderr() +} + +func (r *RunnerV2) startLoggingReaderStdout() { + for { + line, err := r.scannerStdout.ReadString(byte('\n')) + if err != nil { + break + } + line = strings.TrimSuffix(line, "\n") + r.writeLogLines([]string{line}) + } +} + +func (r *RunnerV2) startLoggingReaderStderr() { + for { + line, err := r.scannerStderr.ReadString(byte('\n')) + if err != nil { + break + } + line = strings.TrimSuffix(line, "\n") + r.writeLogLines([]string{line}) + } +} + +func (r *RunnerV2) startHealthCheck() { + if r.cmd.ProcessState == nil || r.cmd.ProcessState.Exited() { + return + } + for { + exists, _ := process.PidExists(int32(r.pid)) + if !exists { + // process lost + r.ch <- constants.TaskSignalLost + return + } + time.Sleep(1 * time.Second) + } +} + +func (r *RunnerV2) configureEnv() { + // 默认把Node.js的全局node_modules加入环境变量 + envPath := os.Getenv("PATH") + nodePath := "/usr/lib/node_modules" + if !strings.Contains(envPath, nodePath) { + _ = os.Setenv("PATH", nodePath+":"+envPath) + } + _ = os.Setenv("NODE_PATH", nodePath) + + // default envs + r.cmd.Env = append(os.Environ(), "CRAWLAB_TASK_ID="+r.tid.Hex()) + if viper.GetString("grpc.address") != "" { + r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_ADDRESS="+viper.GetString("grpc.address")) + } + if viper.GetString("grpc.authKey") != "" { + r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_AUTH_KEY="+viper.GetString("grpc.authKey")) + } else { + r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_AUTH_KEY="+constants.DefaultGrpcAuthKey) + } + + // global environment variables + envs, err := client.NewModelServiceV2[models.EnvironmentV2]().GetMany(nil, nil) + if err != nil { + trace.PrintError(err) + return + } + for _, env := range envs { + r.cmd.Env = append(r.cmd.Env, env.Key+"="+env.Value) + } +} + +func (r *RunnerV2) syncFiles() (err error) { + masterURL := fmt.Sprintf("%s/sync/%s", viper.GetString("api.endpoint"), r.s.Id.Hex()) + workspacePath := viper.GetString("workspace") + workerDir := filepath.Join(workspacePath, r.s.Id.Hex()) + + // get file list from master + resp, err := http.Get(masterURL + "/scan") + if err != nil { + fmt.Println("Error getting file list from master:", err) + return trace.TraceError(err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body:", err) + return trace.TraceError(err) + } + var masterFiles map[string]entity.FsFileInfo + err = json.Unmarshal(body, &masterFiles) + if err != nil { + fmt.Println("Error unmarshaling JSON:", err) + return trace.TraceError(err) + } + + // create a map for master files + masterFilesMap := make(map[string]entity.FsFileInfo) + for _, file := range masterFiles { + masterFilesMap[file.Path] = file + } + + // create worker directory if not exists + if _, err := os.Stat(workerDir); os.IsNotExist(err) { + if err := os.MkdirAll(workerDir, os.ModePerm); err != nil { + fmt.Println("Error creating worker directory:", err) + return trace.TraceError(err) + } + } + + // get file list from worker + workerFiles, err := utils.ScanDirectory(workerDir) + if err != nil { + fmt.Println("Error scanning worker directory:", err) + return trace.TraceError(err) + } + + // set up wait group and error channel + var wg sync.WaitGroup + errCh := make(chan error, 1) + + // delete files that are deleted on master node + for path, workerFile := range workerFiles { + if _, exists := masterFilesMap[path]; !exists { + fmt.Println("Deleting file:", path) + err := os.Remove(workerFile.FullPath) + if err != nil { + fmt.Println("Error deleting file:", err) + } + } + } + + // download files that are new or modified on master node + for path, masterFile := range masterFilesMap { + workerFile, exists := workerFiles[path] + if !exists || masterFile.Hash != workerFile.Hash { + wg.Add(1) + go func(path string, masterFile entity.FsFileInfo) { + defer wg.Done() + logrus.Infof("File needs to be synchronized: %s", path) + err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(workerDir, path)) + if err != nil { + logrus.Errorf("Error downloading file: %v", err) + select { + case errCh <- err: + default: + } + } + }(path, masterFile) + } + } + + wg.Wait() + close(errCh) + if err := <-errCh; err != nil { + return err + } + + return nil +} + +func (r *RunnerV2) downloadFile(url string, filePath string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + out, err := os.Create(filePath) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + return err +} + +// wait for process to finish and send task signal (constants.TaskSignal) +// to task runner's channel (RunnerV2.ch) according to exit code +func (r *RunnerV2) wait() { + // wait for process to finish + if err := r.cmd.Wait(); err != nil { + exitError, ok := err.(*exec.ExitError) + if !ok { + r.ch <- constants.TaskSignalError + return + } + exitCode := exitError.ExitCode() + if exitCode == -1 { + // cancel error + r.ch <- constants.TaskSignalCancel + return + } + + // standard error + r.err = err + r.ch <- constants.TaskSignalError + return + } + + // success + r.ch <- constants.TaskSignalFinish +} + +// updateTask update and get updated info of task (RunnerV2.t) +func (r *RunnerV2) updateTask(status string, e error) (err error) { + if r.t != nil && status != "" { + // update task status + r.t.Status = status + if e != nil { + r.t.Error = e.Error() + } + if r.svc.GetNodeConfigService().IsMaster() { + err = service2.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t) + if err != nil { + return err + } + } else { + err = client.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t) + if err != nil { + return err + } + } + + // send notification + go r.sendNotification() + + // update stats + go func() { + r._updateTaskStat(status) + r._updateSpiderStat(status) + }() + } + + // get task + r.t, err = r.svc.GetTaskById(r.tid) + if err != nil { + return err + } + + return nil +} + +func (r *RunnerV2) initSub() (err error) { + r.sub, err = r.c.GetTaskClient().Subscribe(context.Background()) + if err != nil { + return trace.TraceError(err) + } + return nil +} + +func (r *RunnerV2) writeLogLines(lines []string) { + data, err := json.Marshal(&entity.StreamMessageTaskData{ + TaskId: r.tid, + Logs: lines, + }) + if err != nil { + trace.PrintError(err) + return + } + msg := &grpc.StreamMessage{ + Code: grpc.StreamMessageCode_INSERT_LOGS, + Data: data, + } + if err := r.sub.Send(msg); err != nil { + trace.PrintError(err) + return + } +} + +func (r *RunnerV2) _updateTaskStat(status string) { + ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid) + if err != nil { + trace.PrintError(err) + return + } + switch status { + case constants.TaskStatusPending: + // do nothing + case constants.TaskStatusRunning: + ts.StartTs = time.Now() + ts.WaitDuration = ts.StartTs.Sub(ts.CreateTs).Milliseconds() + case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: + if ts.StartTs.IsZero() { + ts.StartTs = time.Now() + ts.WaitDuration = ts.StartTs.Sub(ts.CreateTs).Milliseconds() + } + ts.EndTs = time.Now() + ts.RuntimeDuration = ts.EndTs.Sub(ts.StartTs).Milliseconds() + ts.TotalDuration = ts.EndTs.Sub(ts.CreateTs).Milliseconds() + } + if r.svc.GetNodeConfigService().IsMaster() { + err = service2.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts) + if err != nil { + trace.PrintError(err) + return + } + } else { + err = client.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts) + if err != nil { + trace.PrintError(err) + return + } + } +} + +func (r *RunnerV2) sendNotification() { + data, err := json.Marshal(r.t) + if err != nil { + trace.PrintError(err) + return + } + req := &grpc.Request{ + NodeKey: r.svc.GetNodeConfigService().GetNodeKey(), + Data: data, + } + _, err = r.c.GetTaskClient().SendNotification(context.Background(), req) + if err != nil { + trace.PrintError(err) + return + } +} + +func (r *RunnerV2) _updateSpiderStat(status string) { + // task stat + ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid) + if err != nil { + trace.PrintError(err) + return + } + + // update + var update bson.M + switch status { + case constants.TaskStatusPending, constants.TaskStatusRunning: + update = bson.M{ + "$set": bson.M{ + "last_task_id": r.tid, // last task id + }, + "$inc": bson.M{ + "tasks": 1, // task count + "wait_duration": ts.WaitDuration, // wait duration + }, + } + case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: + update = bson.M{ + "$set": bson.M{ + "last_task_id": r.tid, // last task id + }, + "$inc": bson.M{ + "results": ts.ResultCount, // results + "runtime_duration": ts.RuntimeDuration / 1000, // runtime duration + "total_duration": ts.TotalDuration / 1000, // total duration + }, + } + default: + trace.PrintError(errors.ErrorTaskInvalidType) + return + } + + // perform update + if r.svc.GetNodeConfigService().IsMaster() { + err = service2.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update) + if err != nil { + trace.PrintError(err) + return + } + } else { + err = client.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update) + if err != nil { + trace.PrintError(err) + return + } + } + +} + +func NewTaskRunnerV2(id primitive.ObjectID, svc *ServiceV2) (r2 *RunnerV2, err error) { + // validate options + if id.IsZero() { + return nil, constants.ErrInvalidOptions + } + + // runner + r := &RunnerV2{ + subscribeTimeout: 30 * time.Second, + bufferSize: 1024 * 1024, + svc: svc, + tid: id, + ch: make(chan constants.TaskSignal), + logBatchSize: 20, + } + + // task + r.t, err = svc.GetTaskById(id) + if err != nil { + return nil, err + } + + // spider + r.s, err = svc.GetSpiderById(r.t.SpiderId) + if err != nil { + return nil, err + } + + // task fs service + r.fsSvc = fs2.NewFsServiceV2(filepath.Join(viper.GetString("workspace"), r.s.Id.Hex())) + + // dependency injection + if err := container.GetContainer().Invoke(func( + c interfaces.GrpcClient, + ) { + r.c = c + }); err != nil { + return nil, trace.TraceError(err) + } + + // initialize task runner + if err := r.Init(); err != nil { + return r, err + } + + return r, nil +} diff --git a/core/task/handler/service.go b/core/task/handler/service.go new file mode 100644 index 000000000..84a929ada --- /dev/null +++ b/core/task/handler/service.go @@ -0,0 +1,506 @@ +package handler + +import ( + "context" + "encoding/json" + "errors" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + errors2 "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/client" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/task" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "sync" + "time" +) + +type Service struct { + // dependencies + interfaces.TaskBaseService + cfgSvc interfaces.NodeConfigService + modelSvc service.ModelService + clientModelSvc interfaces.GrpcClientModelService + clientModelNodeSvc interfaces.GrpcClientModelNodeService + clientModelSpiderSvc interfaces.GrpcClientModelSpiderService + clientModelTaskSvc interfaces.GrpcClientModelTaskService + clientModelTaskStatSvc interfaces.GrpcClientModelTaskStatService + clientModelEnvironmentSvc interfaces.GrpcClientModelEnvironmentService + c interfaces.GrpcClient // grpc client + + // settings + //maxRunners int + exitWatchDuration time.Duration + reportInterval time.Duration + fetchInterval time.Duration + fetchTimeout time.Duration + cancelTimeout time.Duration + + // internals variables + stopped bool + mu sync.Mutex + runners sync.Map // pool of task runners started + syncLocks sync.Map // files sync locks map of task runners +} + +func (svc *Service) Start() { + // Initialize gRPC if not started + if !svc.c.IsStarted() { + svc.c.Start() + } + + go svc.ReportStatus() + go svc.Fetch() +} + +func (svc *Service) Run(taskId primitive.ObjectID) (err error) { + return svc.run(taskId) +} + +func (svc *Service) Reset() { + svc.mu.Lock() + defer svc.mu.Unlock() +} + +func (svc *Service) Cancel(taskId primitive.ObjectID) (err error) { + r, err := svc.getRunner(taskId) + if err != nil { + return err + } + if err := r.Cancel(); err != nil { + return err + } + return nil +} + +func (svc *Service) Fetch() { + for { + // wait + time.Sleep(svc.fetchInterval) + + // current node + n, err := svc.GetCurrentNode() + if err != nil { + continue + } + + // skip if node is not active or enabled + if !n.GetActive() || !n.GetEnabled() { + continue + } + + // validate if there are available runners + if svc.getRunnerCount() >= n.GetMaxRunners() { + continue + } + + // stop + if svc.stopped { + return + } + + // fetch task + tid, err := svc.fetch() + if err != nil { + trace.PrintError(err) + continue + } + + // skip if no task id + if tid.IsZero() { + continue + } + + // run task + if err := svc.run(tid); err != nil { + trace.PrintError(err) + t, err := svc.GetTaskById(tid) + if err == nil && t.GetStatus() != constants.TaskStatusCancelled { + t.SetError(err.Error()) + _ = svc.SaveTask(t, constants.TaskStatusError) + continue + } + continue + } + } +} + +func (svc *Service) ReportStatus() { + for { + if svc.stopped { + return + } + + // report handler status + if err := svc.reportStatus(); err != nil { + trace.PrintError(err) + } + + // wait + time.Sleep(svc.reportInterval) + } +} + +func (svc *Service) IsSyncLocked(path string) (ok bool) { + _, ok = svc.syncLocks.Load(path) + return ok +} + +func (svc *Service) LockSync(path string) { + svc.syncLocks.Store(path, true) +} + +func (svc *Service) UnlockSync(path string) { + svc.syncLocks.Delete(path) +} + +//func (svc *Service) GetMaxRunners() (maxRunners int) { +// return svc.maxRunners +//} +// +//func (svc *Service) SetMaxRunners(maxRunners int) { +// svc.maxRunners = maxRunners +//} + +func (svc *Service) GetExitWatchDuration() (duration time.Duration) { + return svc.exitWatchDuration +} + +func (svc *Service) SetExitWatchDuration(duration time.Duration) { + svc.exitWatchDuration = duration +} + +func (svc *Service) GetFetchInterval() (interval time.Duration) { + return svc.fetchInterval +} + +func (svc *Service) SetFetchInterval(interval time.Duration) { + svc.fetchInterval = interval +} + +func (svc *Service) GetReportInterval() (interval time.Duration) { + return svc.reportInterval +} + +func (svc *Service) SetReportInterval(interval time.Duration) { + svc.reportInterval = interval +} + +func (svc *Service) GetCancelTimeout() (timeout time.Duration) { + return svc.cancelTimeout +} + +func (svc *Service) SetCancelTimeout(timeout time.Duration) { + svc.cancelTimeout = timeout +} + +func (svc *Service) GetModelService() (modelSvc interfaces.GrpcClientModelService) { + return svc.clientModelSvc +} + +func (svc *Service) GetModelSpiderService() (modelSpiderSvc interfaces.GrpcClientModelSpiderService) { + return svc.clientModelSpiderSvc +} + +func (svc *Service) GetModelTaskService() (modelTaskSvc interfaces.GrpcClientModelTaskService) { + return svc.clientModelTaskSvc +} + +func (svc *Service) GetModelTaskStatService() (modelTaskSvc interfaces.GrpcClientModelTaskStatService) { + return svc.clientModelTaskStatSvc +} + +func (svc *Service) GetModelEnvironmentService() (modelTaskSvc interfaces.GrpcClientModelEnvironmentService) { + return svc.clientModelEnvironmentSvc +} + +func (svc *Service) GetNodeConfigService() (cfgSvc interfaces.NodeConfigService) { + return svc.cfgSvc +} + +func (svc *Service) GetCurrentNode() (n interfaces.Node, err error) { + // node key + nodeKey := svc.cfgSvc.GetNodeKey() + + // current node + if svc.cfgSvc.IsMaster() { + n, err = svc.modelSvc.GetNodeByKey(nodeKey, nil) + } else { + n, err = svc.clientModelNodeSvc.GetNodeByKey(nodeKey) + } + if err != nil { + return nil, err + } + + return n, nil +} + +func (svc *Service) GetTaskById(id primitive.ObjectID) (t interfaces.Task, err error) { + if svc.cfgSvc.IsMaster() { + t, err = svc.modelSvc.GetTaskById(id) + } else { + t, err = svc.clientModelTaskSvc.GetTaskById(id) + } + if err != nil { + return nil, err + } + + return t, nil +} + +func (svc *Service) GetSpiderById(id primitive.ObjectID) (s interfaces.Spider, err error) { + if svc.cfgSvc.IsMaster() { + s, err = svc.modelSvc.GetSpiderById(id) + } else { + s, err = svc.clientModelSpiderSvc.GetSpiderById(id) + } + if err != nil { + return nil, err + } + + return s, nil +} + +func (svc *Service) getRunners() (runners []*Runner) { + svc.mu.Lock() + defer svc.mu.Unlock() + svc.runners.Range(func(key, value interface{}) bool { + r := value.(Runner) + runners = append(runners, &r) + return true + }) + return runners +} + +func (svc *Service) getRunnerCount() (count int) { + n, err := svc.GetCurrentNode() + if err != nil { + trace.PrintError(err) + return + } + query := bson.M{ + "node_id": n.GetId(), + "status": constants.TaskStatusRunning, + } + if svc.cfgSvc.IsMaster() { + count, err = svc.modelSvc.GetBaseService(interfaces.ModelIdTask).Count(query) + if err != nil { + trace.PrintError(err) + return + } + } else { + count, err = svc.clientModelTaskSvc.Count(query) + if err != nil { + trace.PrintError(err) + return + } + } + return count +} + +func (svc *Service) getRunner(taskId primitive.ObjectID) (r interfaces.TaskRunner, err error) { + log.Debugf("[TaskHandlerService] getRunner: taskId[%v]", taskId) + v, ok := svc.runners.Load(taskId) + if !ok { + return nil, trace.TraceError(errors2.ErrorTaskNotExists) + } + switch v.(type) { + case interfaces.TaskRunner: + r = v.(interfaces.TaskRunner) + default: + return nil, trace.TraceError(errors2.ErrorModelInvalidType) + } + return r, nil +} + +func (svc *Service) addRunner(taskId primitive.ObjectID, r interfaces.TaskRunner) { + log.Debugf("[TaskHandlerService] addRunner: taskId[%v]", taskId) + svc.runners.Store(taskId, r) +} + +func (svc *Service) deleteRunner(taskId primitive.ObjectID) { + log.Debugf("[TaskHandlerService] deleteRunner: taskId[%v]", taskId) + svc.runners.Delete(taskId) +} + +func (svc *Service) saveTask(t interfaces.Task, status string) (err error) { + // normalize status + if status == "" { + status = constants.TaskStatusPending + } + + // set task status + t.SetStatus(status) + + // attempt to get task from database + _, err = svc.clientModelTaskSvc.GetTaskById(t.GetId()) + if err != nil { + // if task does not exist, add to database + if err == mongo.ErrNoDocuments { + if err := client.NewModelDelegate(t, client.WithDelegateConfigPath(svc.GetConfigPath())).Add(); err != nil { + return err + } + return nil + } else { + return err + } + } else { + // otherwise, update + if err := client.NewModelDelegate(t, client.WithDelegateConfigPath(svc.GetConfigPath())).Save(); err != nil { + return err + } + return nil + } +} + +func (svc *Service) reportStatus() (err error) { + // current node + n, err := svc.GetCurrentNode() + if err != nil { + return err + } + + // available runners of handler + ar := n.GetMaxRunners() - svc.getRunnerCount() + + // set available runners + n.SetAvailableRunners(ar) + + // save node + if svc.cfgSvc.IsMaster() { + err = delegate.NewModelDelegate(n).Save() + } else { + err = client.NewModelDelegate(n, client.WithDelegateConfigPath(svc.GetConfigPath())).Save() + } + if err != nil { + return err + } + + return nil +} + +func (svc *Service) fetch() (tid primitive.ObjectID, err error) { + ctx, cancel := context.WithTimeout(context.Background(), svc.fetchTimeout) + defer cancel() + res, err := svc.c.GetTaskClient().Fetch(ctx, svc.c.NewRequest(nil)) + if err != nil { + return tid, trace.TraceError(err) + } + if err := json.Unmarshal(res.Data, &tid); err != nil { + return tid, trace.TraceError(err) + } + return tid, nil +} + +func (svc *Service) run(taskId primitive.ObjectID) (err error) { + // attempt to get runner from pool + _, ok := svc.runners.Load(taskId) + if ok { + return trace.TraceError(errors2.ErrorTaskAlreadyExists) + } + + // create a new task runner + r, err := NewTaskRunner(taskId, svc) + if err != nil { + return trace.TraceError(err) + } + + // add runner to pool + svc.addRunner(taskId, r) + + // create a goroutine to run task + go func() { + // delete runner from pool + defer svc.deleteRunner(r.GetTaskId()) + defer func(r interfaces.TaskRunner) { + err := r.CleanUp() + if err != nil { + log.Errorf("task[%s] clean up error: %v", r.GetTaskId().Hex(), err) + } + }(r) + // run task process (blocking) + // error or finish after task runner ends + if err := r.Run(); err != nil { + switch { + case errors.Is(err, constants.ErrTaskError): + log.Errorf("task[%s] finished with error: %v", r.GetTaskId().Hex(), err) + case errors.Is(err, constants.ErrTaskCancelled): + log.Errorf("task[%s] cancelled", r.GetTaskId().Hex()) + default: + log.Errorf("task[%s] finished with unknown error: %v", r.GetTaskId().Hex(), err) + } + } + log.Infof("task[%s] finished", r.GetTaskId().Hex()) + }() + + return nil +} + +func NewTaskHandlerService() (svc2 interfaces.TaskHandlerService, err error) { + // base service + baseSvc, err := task.NewBaseService() + if err != nil { + return nil, trace.TraceError(err) + } + + // service + svc := &Service{ + TaskBaseService: baseSvc, + exitWatchDuration: 60 * time.Second, + fetchInterval: 1 * time.Second, + fetchTimeout: 15 * time.Second, + reportInterval: 5 * time.Second, + cancelTimeout: 5 * time.Second, + mu: sync.Mutex{}, + runners: sync.Map{}, + syncLocks: sync.Map{}, + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + cfgSvc interfaces.NodeConfigService, + modelSvc service.ModelService, + clientModelSvc interfaces.GrpcClientModelService, + clientModelNodeSvc interfaces.GrpcClientModelNodeService, + clientModelSpiderSvc interfaces.GrpcClientModelSpiderService, + clientModelTaskSvc interfaces.GrpcClientModelTaskService, + clientModelTaskStatSvc interfaces.GrpcClientModelTaskStatService, + clientModelEnvironmentSvc interfaces.GrpcClientModelEnvironmentService, + c interfaces.GrpcClient, + ) { + svc.cfgSvc = cfgSvc + svc.modelSvc = modelSvc + svc.clientModelSvc = clientModelSvc + svc.clientModelNodeSvc = clientModelNodeSvc + svc.clientModelSpiderSvc = clientModelSpiderSvc + svc.clientModelTaskSvc = clientModelTaskSvc + svc.clientModelTaskStatSvc = clientModelTaskStatSvc + svc.clientModelEnvironmentSvc = clientModelEnvironmentSvc + svc.c = c + }); err != nil { + return nil, trace.TraceError(err) + } + + log.Debugf("[NewTaskHandlerService] svc[cfgPath: %s]", svc.cfgSvc.GetConfigPath()) + + return svc, nil +} + +var _service interfaces.TaskHandlerService + +func GetTaskHandlerService() (svr interfaces.TaskHandlerService, err error) { + if _service != nil { + return _service, nil + } + _service, err = NewTaskHandlerService() + if err != nil { + return nil, err + } + return _service, nil +} diff --git a/core/task/handler/service_v2.go b/core/task/handler/service_v2.go new file mode 100644 index 000000000..397d5f112 --- /dev/null +++ b/core/task/handler/service_v2.go @@ -0,0 +1,426 @@ +package handler + +import ( + "context" + "encoding/json" + "errors" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/constants" + errors2 "github.com/crawlab-team/crawlab/core/errors" + grpcclient "github.com/crawlab-team/crawlab/core/grpc/client" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/client" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "sync" + "time" +) + +type ServiceV2 struct { + // dependencies + cfgSvc interfaces.NodeConfigService + c *grpcclient.GrpcClientV2 // grpc client + + // settings + //maxRunners int + exitWatchDuration time.Duration + reportInterval time.Duration + fetchInterval time.Duration + fetchTimeout time.Duration + cancelTimeout time.Duration + + // internals variables + stopped bool + mu sync.Mutex + runners sync.Map // pool of task runners started + syncLocks sync.Map // files sync locks map of task runners +} + +func (svc *ServiceV2) Start() { + // Initialize gRPC if not started + if !svc.c.IsStarted() { + svc.c.Start() + } + + go svc.ReportStatus() + go svc.Fetch() +} + +func (svc *ServiceV2) Run(taskId primitive.ObjectID) (err error) { + return svc.run(taskId) +} + +func (svc *ServiceV2) Reset() { + svc.mu.Lock() + defer svc.mu.Unlock() +} + +func (svc *ServiceV2) Cancel(taskId primitive.ObjectID) (err error) { + r, err := svc.getRunner(taskId) + if err != nil { + return err + } + if err := r.Cancel(); err != nil { + return err + } + return nil +} + +func (svc *ServiceV2) Fetch() { + for { + // wait + time.Sleep(svc.fetchInterval) + + // current node + n, err := svc.GetCurrentNode() + if err != nil { + continue + } + + // skip if node is not active or enabled + if !n.Active || !n.Enabled { + continue + } + + // validate if there are available runners + if svc.getRunnerCount() >= n.MaxRunners { + continue + } + + // stop + if svc.stopped { + return + } + + // fetch task + tid, err := svc.fetch() + if err != nil { + trace.PrintError(err) + continue + } + + // skip if no task id + if tid.IsZero() { + continue + } + + // run task + if err := svc.run(tid); err != nil { + trace.PrintError(err) + t, err := svc.GetTaskById(tid) + if err == nil && t.Status != constants.TaskStatusCancelled { + t.Error = err.Error() + t.Status = constants.TaskStatusError + t.SetUpdated(t.CreatedBy) + continue + } + continue + } + } +} + +func (svc *ServiceV2) ReportStatus() { + for { + if svc.stopped { + return + } + + // report handler status + if err := svc.reportStatus(); err != nil { + trace.PrintError(err) + } + + // wait + time.Sleep(svc.reportInterval) + } +} + +func (svc *ServiceV2) IsSyncLocked(path string) (ok bool) { + _, ok = svc.syncLocks.Load(path) + return ok +} + +func (svc *ServiceV2) LockSync(path string) { + svc.syncLocks.Store(path, true) +} + +func (svc *ServiceV2) UnlockSync(path string) { + svc.syncLocks.Delete(path) +} + +//func (svc *ServiceV2) GetMaxRunners() (maxRunners int) { +// return svc.maxRunners +//} +// +//func (svc *ServiceV2) SetMaxRunners(maxRunners int) { +// svc.maxRunners = maxRunners +//} + +func (svc *ServiceV2) GetExitWatchDuration() (duration time.Duration) { + return svc.exitWatchDuration +} + +func (svc *ServiceV2) SetExitWatchDuration(duration time.Duration) { + svc.exitWatchDuration = duration +} + +func (svc *ServiceV2) GetFetchInterval() (interval time.Duration) { + return svc.fetchInterval +} + +func (svc *ServiceV2) SetFetchInterval(interval time.Duration) { + svc.fetchInterval = interval +} + +func (svc *ServiceV2) GetReportInterval() (interval time.Duration) { + return svc.reportInterval +} + +func (svc *ServiceV2) SetReportInterval(interval time.Duration) { + svc.reportInterval = interval +} + +func (svc *ServiceV2) GetCancelTimeout() (timeout time.Duration) { + return svc.cancelTimeout +} + +func (svc *ServiceV2) SetCancelTimeout(timeout time.Duration) { + svc.cancelTimeout = timeout +} + +func (svc *ServiceV2) GetNodeConfigService() (cfgSvc interfaces.NodeConfigService) { + return svc.cfgSvc +} + +func (svc *ServiceV2) GetCurrentNode() (n *models.NodeV2, err error) { + // node key + nodeKey := svc.cfgSvc.GetNodeKey() + + // current node + if svc.cfgSvc.IsMaster() { + n, err = service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + } else { + n, err = client.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + } + if err != nil { + return nil, err + } + + return n, nil +} + +func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models.TaskV2, err error) { + if svc.cfgSvc.IsMaster() { + t, err = service.NewModelServiceV2[models.TaskV2]().GetById(id) + } else { + t, err = client.NewModelServiceV2[models.TaskV2]().GetById(id) + } + if err != nil { + return nil, err + } + + return t, nil +} + +func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models.SpiderV2, err error) { + if svc.cfgSvc.IsMaster() { + s, err = service.NewModelServiceV2[models.SpiderV2]().GetById(id) + } else { + s, err = client.NewModelServiceV2[models.SpiderV2]().GetById(id) + } + if err != nil { + return nil, err + } + + return s, nil +} + +func (svc *ServiceV2) getRunners() (runners []*Runner) { + svc.mu.Lock() + defer svc.mu.Unlock() + svc.runners.Range(func(key, value interface{}) bool { + r := value.(Runner) + runners = append(runners, &r) + return true + }) + return runners +} + +func (svc *ServiceV2) getRunnerCount() (count int) { + n, err := svc.GetCurrentNode() + if err != nil { + trace.PrintError(err) + return + } + query := bson.M{ + "node_id": n.Id, + "status": constants.TaskStatusRunning, + } + if svc.cfgSvc.IsMaster() { + count, err = service.NewModelServiceV2[models.TaskV2]().Count(query) + if err != nil { + trace.PrintError(err) + return + } + } else { + count, err = client.NewModelServiceV2[models.TaskV2]().Count(query) + if err != nil { + trace.PrintError(err) + return + } + } + return count +} + +func (svc *ServiceV2) getRunner(taskId primitive.ObjectID) (r interfaces.TaskRunner, err error) { + log.Debugf("[TaskHandlerService] getRunner: taskId[%v]", taskId) + v, ok := svc.runners.Load(taskId) + if !ok { + return nil, trace.TraceError(errors2.ErrorTaskNotExists) + } + switch v.(type) { + case interfaces.TaskRunner: + r = v.(interfaces.TaskRunner) + default: + return nil, trace.TraceError(errors2.ErrorModelInvalidType) + } + return r, nil +} + +func (svc *ServiceV2) addRunner(taskId primitive.ObjectID, r interfaces.TaskRunner) { + log.Debugf("[TaskHandlerService] addRunner: taskId[%v]", taskId) + svc.runners.Store(taskId, r) +} + +func (svc *ServiceV2) deleteRunner(taskId primitive.ObjectID) { + log.Debugf("[TaskHandlerService] deleteRunner: taskId[%v]", taskId) + svc.runners.Delete(taskId) +} + +func (svc *ServiceV2) reportStatus() (err error) { + // current node + n, err := svc.GetCurrentNode() + if err != nil { + return err + } + + // available runners of handler + ar := n.MaxRunners - svc.getRunnerCount() + + // set available runners + n.AvailableRunners = ar + + // save node + n.SetUpdated(n.CreatedBy) + if svc.cfgSvc.IsMaster() { + err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n) + } else { + err = client.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n) + } + if err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) fetch() (tid primitive.ObjectID, err error) { + ctx, cancel := context.WithTimeout(context.Background(), svc.fetchTimeout) + defer cancel() + res, err := svc.c.TaskClient.Fetch(ctx, svc.c.NewRequest(nil)) + if err != nil { + return tid, trace.TraceError(err) + } + if err := json.Unmarshal(res.Data, &tid); err != nil { + return tid, trace.TraceError(err) + } + return tid, nil +} + +func (svc *ServiceV2) run(taskId primitive.ObjectID) (err error) { + // attempt to get runner from pool + _, ok := svc.runners.Load(taskId) + if ok { + return trace.TraceError(errors2.ErrorTaskAlreadyExists) + } + + // create a new task runner + r, err := NewTaskRunnerV2(taskId, svc) + if err != nil { + return trace.TraceError(err) + } + + // add runner to pool + svc.addRunner(taskId, r) + + // create a goroutine to run task + go func() { + // delete runner from pool + defer svc.deleteRunner(r.GetTaskId()) + defer func(r interfaces.TaskRunner) { + err := r.CleanUp() + if err != nil { + log.Errorf("task[%s] clean up error: %v", r.GetTaskId().Hex(), err) + } + }(r) + // run task process (blocking) + // error or finish after task runner ends + if err := r.Run(); err != nil { + switch { + case errors.Is(err, constants.ErrTaskError): + log.Errorf("task[%s] finished with error: %v", r.GetTaskId().Hex(), err) + case errors.Is(err, constants.ErrTaskCancelled): + log.Errorf("task[%s] cancelled", r.GetTaskId().Hex()) + default: + log.Errorf("task[%s] finished with unknown error: %v", r.GetTaskId().Hex(), err) + } + } + log.Infof("task[%s] finished", r.GetTaskId().Hex()) + }() + + return nil +} + +func NewTaskHandlerServiceV2() (svc2 *ServiceV2, err error) { + // service + svc := &ServiceV2{ + exitWatchDuration: 60 * time.Second, + fetchInterval: 1 * time.Second, + fetchTimeout: 15 * time.Second, + reportInterval: 5 * time.Second, + cancelTimeout: 5 * time.Second, + mu: sync.Mutex{}, + runners: sync.Map{}, + syncLocks: sync.Map{}, + } + + // dependency injection + svc.cfgSvc = nodeconfig.GetNodeConfigService() + + // grpc client + svc.c, err = grpcclient.NewGrpcClientV2() + if err != nil { + return nil, err + } + + log.Debugf("[NewTaskHandlerService] svc[cfgPath: %s]", svc.cfgSvc.GetConfigPath()) + + return svc, nil +} + +var _serviceV2 *ServiceV2 + +func GetTaskHandlerServiceV2() (svr *ServiceV2, err error) { + if _serviceV2 != nil { + return _serviceV2, nil + } + _serviceV2, err = NewTaskHandlerServiceV2() + if err != nil { + return nil, err + } + return _serviceV2, nil +} diff --git a/core/task/log/constants.go b/core/task/log/constants.go new file mode 100644 index 000000000..736768511 --- /dev/null +++ b/core/task/log/constants.go @@ -0,0 +1,12 @@ +package log + +const ( + MetadataName = "metadata.json" +) + +const ( + DriverTypeFile = "file" // raw file + DriverTypeFs = "fs" // file system (SeaweedFS) + DriverTypeMongo = "mongo" // mongodb + DriverTypeEs = "es" // elastic search +) diff --git a/core/task/log/default.go b/core/task/log/default.go new file mode 100644 index 000000000..2ac37f816 --- /dev/null +++ b/core/task/log/default.go @@ -0,0 +1,5 @@ +package log + +import "time" + +var DefaultLogTtl = 30 * 24 * time.Hour diff --git a/core/task/log/driver.go b/core/task/log/driver.go new file mode 100644 index 000000000..139ec44d4 --- /dev/null +++ b/core/task/log/driver.go @@ -0,0 +1,18 @@ +package log + +func GetLogDriver(logDriverType string) (driver Driver, err error) { + switch logDriverType { + case DriverTypeFile: + driver, err = GetFileLogDriver() + if err != nil { + return driver, err + } + case DriverTypeMongo: + return driver, ErrNotImplemented + case DriverTypeEs: + return driver, ErrNotImplemented + default: + return driver, ErrInvalidType + } + return driver, nil +} diff --git a/core/task/log/entity.go b/core/task/log/entity.go new file mode 100644 index 000000000..0583e41c7 --- /dev/null +++ b/core/task/log/entity.go @@ -0,0 +1,16 @@ +package log + +import "time" + +type Message struct { + Id int64 `json:"id" bson:"id"` + Msg string `json:"msg" bson:"msg"` + Ts time.Time `json:"ts" bson:"ts"` +} + +type Metadata struct { + Size int64 `json:"size,omitempty" bson:"size"` + TotalLines int64 `json:"total_lines,omitempty" bson:"total_lines"` + TotalBytes int64 `json:"total_bytes,omitempty" bson:"total_bytes"` + Md5 string `json:"md5,omitempty" bson:"md5"` +} diff --git a/core/task/log/errors.go b/core/task/log/errors.go new file mode 100644 index 000000000..eb7c15409 --- /dev/null +++ b/core/task/log/errors.go @@ -0,0 +1,8 @@ +package log + +import "errors" + +var ( + ErrInvalidType = errors.New("invalid type") + ErrNotImplemented = errors.New("not implemented") +) diff --git a/core/task/log/file_driver.go b/core/task/log/file_driver.go new file mode 100644 index 000000000..590a1c9a5 --- /dev/null +++ b/core/task/log/file_driver.go @@ -0,0 +1,284 @@ +package log + +import ( + "bufio" + "bytes" + "errors" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/spf13/viper" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +type FileLogDriver struct { + // settings + logFileName string + rootPath string + + // internals + mu sync.Mutex +} + +func (d *FileLogDriver) Init() (err error) { + go d.cleanup() + + return nil +} + +func (d *FileLogDriver) Close() (err error) { + return nil +} + +func (d *FileLogDriver) WriteLine(id string, line string) (err error) { + d.initDir(id) + + d.mu.Lock() + defer d.mu.Unlock() + filePath := d.getLogFilePath(id, d.logFileName) + + f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(0760)) + if err != nil { + return trace.TraceError(err) + } + defer func(f *os.File) { + err := f.Close() + if err != nil { + log.Errorf("close file error: %s", err.Error()) + } + }(f) + + _, err = f.WriteString(line + "\n") + if err != nil { + return trace.TraceError(err) + } + + return nil +} + +func (d *FileLogDriver) WriteLines(id string, lines []string) (err error) { + linesString := strings.Join(lines, "\n") + if err := d.WriteLine(id, linesString); err != nil { + return err + } + return nil +} + +func (d *FileLogDriver) Find(id string, pattern string, skip int, limit int) (lines []string, err error) { + if pattern != "" { + return lines, errors.New("not implemented") + } + if !utils.Exists(d.getLogFilePath(id, d.logFileName)) { + return nil, nil + } + + f, err := os.Open(d.getLogFilePath(id, d.logFileName)) + if err != nil { + return nil, trace.TraceError(err) + } + defer f.Close() + + sc := bufio.NewReaderSize(f, 1024*1024*10) + + i := -1 + for { + line, err := sc.ReadString(byte('\n')) + if err != nil { + break + } + line = strings.TrimSuffix(line, "\n") + + i++ + + if i < skip { + continue + } + + if i >= skip+limit { + break + } + + lines = append(lines, line) + } + + return lines, nil +} + +func (d *FileLogDriver) Count(id string, pattern string) (n int, err error) { + if pattern != "" { + return n, errors.New("not implemented") + } + if !utils.Exists(d.getLogFilePath(id, d.logFileName)) { + return 0, nil + } + + f, err := os.Open(d.getLogFilePath(id, d.logFileName)) + if err != nil { + return n, trace.TraceError(err) + } + return d.lineCounter(f) +} + +func (d *FileLogDriver) Flush() (err error) { + return nil +} + +func (d *FileLogDriver) getLogPath() (logPath string) { + return viper.GetString("log.path") +} + +func (d *FileLogDriver) getBasePath(id string) (filePath string) { + return filepath.Join(d.getLogPath(), id) +} + +func (d *FileLogDriver) getMetadataPath(id string) (filePath string) { + return filepath.Join(d.getBasePath(id), MetadataName) +} + +func (d *FileLogDriver) getLogFilePath(id, fileName string) (filePath string) { + return filepath.Join(d.getBasePath(id), fileName) +} + +func (d *FileLogDriver) getLogFiles(id string) (files []os.FileInfo) { + // 增加了对返回异常的捕获 + files, err := utils.ListDir(d.getBasePath(id)) + if err != nil { + trace.PrintError(err) + return nil + } + return +} + +func (d *FileLogDriver) initDir(id string) { + if !utils.Exists(d.getBasePath(id)) { + if err := os.MkdirAll(d.getBasePath(id), os.FileMode(0770)); err != nil { + trace.PrintError(err) + } + } +} + +func (d *FileLogDriver) lineCounter(r io.Reader) (n int, err error) { + buf := make([]byte, 32*1024) + count := 0 + lineSep := []byte{'\n'} + + for { + c, err := r.Read(buf) + count += bytes.Count(buf[:c], lineSep) + + switch { + case err == io.EOF: + return count, nil + + case err != nil: + return count, err + } + } +} + +func (d *FileLogDriver) getTtl() time.Duration { + ttl := viper.GetString("log.ttl") + if ttl == "" { + return DefaultLogTtl + } + + if strings.HasSuffix(ttl, "s") { + ttl = strings.TrimSuffix(ttl, "s") + n, err := strconv.Atoi(ttl) + if err != nil { + return DefaultLogTtl + } + return time.Duration(n) * time.Second + } else if strings.HasSuffix(ttl, "m") { + ttl = strings.TrimSuffix(ttl, "m") + n, err := strconv.Atoi(ttl) + if err != nil { + return DefaultLogTtl + } + return time.Duration(n) * time.Minute + } else if strings.HasSuffix(ttl, "h") { + ttl = strings.TrimSuffix(ttl, "h") + n, err := strconv.Atoi(ttl) + if err != nil { + return DefaultLogTtl + } + return time.Duration(n) * time.Hour + + } else if strings.HasSuffix(ttl, "d") { + ttl = strings.TrimSuffix(ttl, "d") + n, err := strconv.Atoi(ttl) + if err != nil { + return DefaultLogTtl + } + return time.Duration(n) * 24 * time.Hour + } else { + return DefaultLogTtl + } +} + +func (d *FileLogDriver) cleanup() { + if d.getLogPath() == "" { + return + } + if !utils.Exists(d.getLogPath()) { + if err := os.MkdirAll(d.getLogPath(), os.FileMode(0770)); err != nil { + log.Errorf("failed to create log directory: %s", d.getLogPath()) + trace.PrintError(err) + return + } + } + for { + // 增加对目录不存在的判断 + dirs, err := utils.ListDir(d.getLogPath()) + if err != nil { + trace.PrintError(err) + time.Sleep(10 * time.Minute) + continue + } + for _, dir := range dirs { + if time.Now().After(dir.ModTime().Add(d.getTtl())) { + if err := os.RemoveAll(d.getBasePath(dir.Name())); err != nil { + trace.PrintError(err) + continue + } + log.Infof("removed outdated log directory: %s", d.getBasePath(dir.Name())) + } + } + + time.Sleep(10 * time.Minute) + } +} + +var logDriver Driver + +func newFileLogDriver() (driver Driver, err error) { + // driver + driver = &FileLogDriver{ + logFileName: "log.txt", + mu: sync.Mutex{}, + } + + // init + if err := driver.Init(); err != nil { + return nil, err + } + + return driver, nil +} + +func GetFileLogDriver() (driver Driver, err error) { + if logDriver != nil { + return logDriver, nil + } + logDriver, err = newFileLogDriver() + if err != nil { + return nil, err + } + return logDriver, nil +} diff --git a/core/task/log/file_driver_test.go b/core/task/log/file_driver_test.go new file mode 100644 index 000000000..260edf717 --- /dev/null +++ b/core/task/log/file_driver_test.go @@ -0,0 +1,132 @@ +package log + +import ( + "fmt" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson/primitive" + "os" + "strings" + "testing" +) + +func setupFileDriverTest() { + cleanupFileDriverTest() + _ = os.MkdirAll("./tmp", os.ModePerm) +} + +func cleanupFileDriverTest() { + _ = os.RemoveAll("./tmp") +} + +func TestFileDriver_WriteLine(t *testing.T) { + setupFileDriverTest() + t.Cleanup(cleanupFileDriverTest) + + d, err := newFileLogDriver(nil) + require.Nil(t, err) + defer d.Close() + + id := primitive.NewObjectID() + + err = d.WriteLine(id.Hex(), "it works") + require.Nil(t, err) + + logFilePath := fmt.Sprintf("/var/log/crawlab/%s/log.txt", id.Hex()) + require.FileExists(t, logFilePath) + text, err := os.ReadFile(logFilePath) + require.Nil(t, err) + require.Equal(t, "it works\n", string(text)) +} + +func TestFileDriver_WriteLines(t *testing.T) { + setupFileDriverTest() + t.Cleanup(cleanupFileDriverTest) + + d, err := newFileLogDriver(nil) + require.Nil(t, err) + defer d.Close() + + id := primitive.NewObjectID() + + for i := 0; i < 100; i++ { + err = d.WriteLine(id.Hex(), "it works") + require.Nil(t, err) + } + + logFilePath := fmt.Sprintf("/var/log/crawlab/%s/log.txt", id.Hex()) + require.FileExists(t, logFilePath) + text, err := os.ReadFile(logFilePath) + require.Nil(t, err) + require.Contains(t, string(text), "it works\n") + lines := strings.Split(string(text), "\n") + require.Equal(t, 101, len(lines)) +} + +func TestFileDriver_Find(t *testing.T) { + setupFileDriverTest() + t.Cleanup(cleanupFileDriverTest) + + d, err := newFileLogDriver(nil) + require.Nil(t, err) + defer d.Close() + + id := primitive.NewObjectID() + + batch := 1000 + var lines []string + for i := 0; i < 10; i++ { + for j := 0; j < batch; j++ { + line := fmt.Sprintf("line: %d", i*batch+j+1) + lines = append(lines, line) + } + err = d.WriteLines(id.Hex(), lines) + require.Nil(t, err) + lines = []string{} + } + + driver := d + + lines, err = driver.Find(id.Hex(), "", 0, 10) + require.Nil(t, err) + require.Equal(t, 10, len(lines)) + require.Equal(t, "line: 1", lines[0]) + require.Equal(t, "line: 10", lines[len(lines)-1]) + + lines, err = driver.Find(id.Hex(), "", 0, 1) + require.Nil(t, err) + require.Equal(t, 1, len(lines)) + require.Equal(t, "line: 1", lines[0]) + require.Equal(t, "line: 1", lines[len(lines)-1]) + + lines, err = driver.Find(id.Hex(), "", 0, 1000) + require.Nil(t, err) + require.Equal(t, 1000, len(lines)) + require.Equal(t, "line: 1", lines[0]) + require.Equal(t, "line: 1000", lines[len(lines)-1]) + + lines, err = driver.Find(id.Hex(), "", 1000, 1000) + require.Nil(t, err) + require.Equal(t, 1000, len(lines)) + require.Equal(t, "line: 1001", lines[0]) + require.Equal(t, "line: 2000", lines[len(lines)-1]) + + lines, err = driver.Find(id.Hex(), "", 1001, 1000) + require.Nil(t, err) + require.Equal(t, 1000, len(lines)) + require.Equal(t, "line: 1002", lines[0]) + require.Equal(t, "line: 2001", lines[len(lines)-1]) + + lines, err = driver.Find(id.Hex(), "", 1001, 999) + require.Nil(t, err) + require.Equal(t, 999, len(lines)) + require.Equal(t, "line: 1002", lines[0]) + require.Equal(t, "line: 2000", lines[len(lines)-1]) + + lines, err = driver.Find(id.Hex(), "", 999, 2001) + require.Nil(t, err) + require.Equal(t, 2001, len(lines)) + require.Equal(t, "line: 1000", lines[0]) + require.Equal(t, "line: 3000", lines[len(lines)-1]) + + cleanupFileDriverTest() +} diff --git a/core/task/log/interface.go b/core/task/log/interface.go new file mode 100644 index 000000000..fd3745cb9 --- /dev/null +++ b/core/task/log/interface.go @@ -0,0 +1,10 @@ +package log + +type Driver interface { + Init() (err error) + Close() (err error) + WriteLine(id string, line string) (err error) + WriteLines(id string, lines []string) (err error) + Find(id string, pattern string, skip int, limit int) (lines []string, err error) + Count(id string, pattern string) (n int, err error) +} diff --git a/core/task/scheduler/options.go b/core/task/scheduler/options.go new file mode 100644 index 000000000..a6600fab2 --- /dev/null +++ b/core/task/scheduler/options.go @@ -0,0 +1,20 @@ +package scheduler + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type Option func(svc interfaces.TaskSchedulerService) + +func WithConfigPath(path string) Option { + return func(svc interfaces.TaskSchedulerService) { + svc.SetConfigPath(path) + } +} + +func WithInterval(interval time.Duration) Option { + return func(svc interfaces.TaskSchedulerService) { + svc.SetInterval(interval) + } +} diff --git a/core/task/scheduler/service.go b/core/task/scheduler/service.go new file mode 100644 index 000000000..fb8585456 --- /dev/null +++ b/core/task/scheduler/service.go @@ -0,0 +1,264 @@ +package scheduler + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/task" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "time" +) + +type Service struct { + // dependencies + interfaces.TaskBaseService + nodeCfgSvc interfaces.NodeConfigService + modelSvc service.ModelService + svr interfaces.GrpcServer + handlerSvc interfaces.TaskHandlerService + + // settings + interval time.Duration +} + +func (svc *Service) Start() { + go svc.initTaskStatus() + go svc.cleanupTasks() + svc.Wait() + svc.Stop() +} + +func (svc *Service) Enqueue(t interfaces.Task) (t2 interfaces.Task, err error) { + // set task status + t.SetStatus(constants.TaskStatusPending) + + // user + var u *models.User + if !t.GetUserId().IsZero() { + u, _ = svc.modelSvc.GetUserById(t.GetUserId()) + } + + // add task + if err = delegate.NewModelDelegate(t, u).Add(); err != nil { + return nil, err + } + + // task queue item + tq := &models.TaskQueueItem{ + Id: t.GetId(), + Priority: t.GetPriority(), + NodeId: t.GetNodeId(), + } + + // task stat + ts := &models.TaskStat{ + Id: t.GetId(), + CreateTs: time.Now(), + } + + // enqueue task + _, err = mongo.GetMongoCol(interfaces.ModelColNameTaskQueue).Insert(tq) + if err != nil { + return nil, trace.TraceError(err) + } + + // add task stat + _, err = mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Insert(ts) + if err != nil { + return nil, trace.TraceError(err) + } + + // success + return t, nil +} + +func (svc *Service) Cancel(id primitive.ObjectID, args ...interface{}) (err error) { + // task + t, err := svc.modelSvc.GetTaskById(id) + if err != nil { + return trace.TraceError(err) + } + + // initial status + initialStatus := t.Status + + // set task status as "cancelled" + _ = svc.SaveTask(t, constants.TaskStatusCancelled) + + // set status of pending tasks as "cancelled" and remove from task item queue + if initialStatus == constants.TaskStatusPending { + // remove from task item queue + if err := mongo.GetMongoCol(interfaces.ModelColNameTaskQueue).DeleteId(t.GetId()); err != nil { + return trace.TraceError(err) + } + return nil + } + + // whether task is running on master node + isMasterTask, err := svc.isMasterNode(t) + if err != nil { + // when error, force status being set as "cancelled" + return svc.SaveTask(t, constants.TaskStatusCancelled) + } + + // node + n, err := svc.modelSvc.GetNodeById(t.GetNodeId()) + if err != nil { + return trace.TraceError(err) + } + + if isMasterTask { + // cancel task on master + if err := svc.handlerSvc.Cancel(id); err != nil { + return trace.TraceError(err) + } + // cancel success + return nil + } else { + // send to cancel task on worker nodes + if err := svc.svr.SendStreamMessageWithData("node:"+n.GetKey(), grpc.StreamMessageCode_CANCEL_TASK, t); err != nil { + return trace.TraceError(err) + } + // cancel success + return nil + } +} + +func (svc *Service) SetInterval(interval time.Duration) { + svc.interval = interval +} + +// initTaskStatus initialize task status of existing tasks +func (svc *Service) initTaskStatus() { + // set status of running tasks as TaskStatusAbnormal + runningTasks, err := svc.modelSvc.GetTaskList(bson.M{ + "status": bson.M{ + "$in": []string{ + constants.TaskStatusPending, + constants.TaskStatusRunning, + }, + }, + }, nil) + if err != nil { + if err == mongo2.ErrNoDocuments { + return + } + trace.PrintError(err) + } + for _, t := range runningTasks { + go func(t *models.Task) { + if err := svc.SaveTask(t, constants.TaskStatusAbnormal); err != nil { + trace.PrintError(err) + } + }(&t) + } + if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTaskQueue).DeleteList(nil); err != nil { + return + } +} + +func (svc *Service) isMasterNode(t *models.Task) (ok bool, err error) { + if t.GetNodeId().IsZero() { + return false, trace.TraceError(errors.ErrorTaskNoNodeId) + } + n, err := svc.modelSvc.GetNodeById(t.GetNodeId()) + if err != nil { + if err == mongo2.ErrNoDocuments { + return false, trace.TraceError(errors.ErrorTaskNodeNotFound) + } + return false, trace.TraceError(err) + } + return n.IsMaster, nil +} + +func (svc *Service) cleanupTasks() { + for { + // task stats over 30 days ago + taskStats, err := svc.modelSvc.GetTaskStatList(bson.M{ + "create_ts": bson.M{ + "$lt": time.Now().Add(-30 * 24 * time.Hour), + }, + }, nil) + if err != nil { + time.Sleep(30 * time.Minute) + continue + } + + // task ids + var ids []primitive.ObjectID + for _, ts := range taskStats { + ids = append(ids, ts.Id) + } + + if len(ids) > 0 { + // remove tasks + if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTask).DeleteList(bson.M{ + "_id": bson.M{"$in": ids}, + }); err != nil { + trace.PrintError(err) + } + + // remove task stats + if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTaskStat).DeleteList(bson.M{ + "_id": bson.M{"$in": ids}, + }); err != nil { + trace.PrintError(err) + } + } + + time.Sleep(30 * time.Minute) + } +} + +func NewTaskSchedulerService() (svc2 interfaces.TaskSchedulerService, err error) { + // base service + baseSvc, err := task.NewBaseService() + if err != nil { + return nil, trace.TraceError(err) + } + + // service + svc := &Service{ + TaskBaseService: baseSvc, + interval: 5 * time.Second, + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + nodeCfgSvc interfaces.NodeConfigService, + modelSvc service.ModelService, + svr interfaces.GrpcServer, + handlerSvc interfaces.TaskHandlerService, + ) { + svc.nodeCfgSvc = nodeCfgSvc + svc.modelSvc = modelSvc + svc.svr = svr + svc.handlerSvc = handlerSvc + }); err != nil { + return nil, err + } + + return svc, nil +} + +var svc interfaces.TaskSchedulerService + +func GetTaskSchedulerService() (svr interfaces.TaskSchedulerService, err error) { + if svc != nil { + return svc, nil + } + svc, err = NewTaskSchedulerService() + if err != nil { + return nil, err + } + return svc, nil +} diff --git a/core/task/scheduler/service_v2.go b/core/task/scheduler/service_v2.go new file mode 100644 index 000000000..b5b6cfa4c --- /dev/null +++ b/core/task/scheduler/service_v2.go @@ -0,0 +1,264 @@ +package scheduler + +import ( + errors2 "errors" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "time" +) + +type ServiceV2 struct { + // dependencies + nodeCfgSvc interfaces.NodeConfigService + svr interfaces.GrpcServer + handlerSvc interfaces.TaskHandlerService + + // settings + interval time.Duration +} + +func (svc *ServiceV2) Start() { + go svc.initTaskStatus() + go svc.cleanupTasks() + utils.DefaultWait() +} + +func (svc *ServiceV2) Enqueue(t *models.TaskV2, by primitive.ObjectID) (t2 *models.TaskV2, err error) { + // set task status + t.Status = constants.TaskStatusPending + t.SetCreatedBy(by) + t.SetUpdated(by) + + // add task + taskModelSvc := service.NewModelServiceV2[models.TaskV2]() + _, err = taskModelSvc.InsertOne(*t) + if err != nil { + return nil, err + } + + // task queue item + tq := models.TaskQueueItemV2{ + Priority: t.Priority, + NodeId: t.NodeId, + } + tq.SetId(t.Id) + + // task stat + ts := models.TaskStatV2{ + CreateTs: time.Now(), + } + ts.SetId(t.Id) + + // enqueue task + _, err = service.NewModelServiceV2[models.TaskQueueItemV2]().InsertOne(tq) + if err != nil { + return nil, trace.TraceError(err) + } + + // add task stat + _, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(ts) + if err != nil { + return nil, trace.TraceError(err) + } + + // success + return t, nil +} + +func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err error) { + // task + t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + if err != nil { + return trace.TraceError(err) + } + + // initial status + initialStatus := t.Status + + // set task status as "cancelled" + t.Status = constants.TaskStatusCancelled + _ = svc.SaveTask(t, by) + + // set status of pending tasks as "cancelled" and remove from task item queue + if initialStatus == constants.TaskStatusPending { + // remove from task item queue + if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteById(t.Id); err != nil { + return trace.TraceError(err) + } + return nil + } + + // whether task is running on master node + isMasterTask, err := svc.isMasterNode(t) + if err != nil { + // when error, force status being set as "cancelled" + t.Status = constants.TaskStatusCancelled + return svc.SaveTask(t, by) + } + + // node + n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId) + if err != nil { + return trace.TraceError(err) + } + + if isMasterTask { + // cancel task on master + if err := svc.handlerSvc.Cancel(id); err != nil { + return trace.TraceError(err) + } + // cancel success + return nil + } else { + // send to cancel task on worker nodes + if err := svc.svr.SendStreamMessageWithData("node:"+n.Key, grpc.StreamMessageCode_CANCEL_TASK, t); err != nil { + return trace.TraceError(err) + } + // cancel success + return nil + } +} + +func (svc *ServiceV2) SetInterval(interval time.Duration) { + svc.interval = interval +} + +func (svc *ServiceV2) SaveTask(t *models.TaskV2, by primitive.ObjectID) (err error) { + if t.Id.IsZero() { + t.SetCreated(by) + t.SetUpdated(by) + _, err = service.NewModelServiceV2[models.TaskV2]().InsertOne(*t) + return err + } else { + t.SetUpdated(by) + return service.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t) + } +} + +// initTaskStatus initialize task status of existing tasks +func (svc *ServiceV2) initTaskStatus() { + // set status of running tasks as TaskStatusAbnormal + runningTasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{ + "status": bson.M{ + "$in": []string{ + constants.TaskStatusPending, + constants.TaskStatusRunning, + }, + }, + }, nil) + if err != nil { + if errors2.Is(err, mongo2.ErrNoDocuments) { + return + } + trace.PrintError(err) + } + for _, t := range runningTasks { + go func(t *models.TaskV2) { + t.Status = constants.TaskStatusAbnormal + if err := svc.SaveTask(t, primitive.NilObjectID); err != nil { + trace.PrintError(err) + } + }(&t) + } + if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteMany(nil); err != nil { + return + } +} + +func (svc *ServiceV2) isMasterNode(t *models.TaskV2) (ok bool, err error) { + if t.NodeId.IsZero() { + return false, trace.TraceError(errors.ErrorTaskNoNodeId) + } + n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId) + if err != nil { + if errors2.Is(err, mongo2.ErrNoDocuments) { + return false, trace.TraceError(errors.ErrorTaskNodeNotFound) + } + return false, trace.TraceError(err) + } + return n.IsMaster, nil +} + +func (svc *ServiceV2) cleanupTasks() { + for { + // task stats over 30 days ago + taskStats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(bson.M{ + "create_ts": bson.M{ + "$lt": time.Now().Add(-30 * 24 * time.Hour), + }, + }, nil) + if err != nil { + time.Sleep(30 * time.Minute) + continue + } + + // task ids + var ids []primitive.ObjectID + for _, ts := range taskStats { + ids = append(ids, ts.Id) + } + + if len(ids) > 0 { + // remove tasks + if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{ + "_id": bson.M{"$in": ids}, + }); err != nil { + trace.PrintError(err) + } + + // remove task stats + if err := service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{ + "_id": bson.M{"$in": ids}, + }); err != nil { + trace.PrintError(err) + } + } + + time.Sleep(30 * time.Minute) + } +} + +func NewTaskSchedulerServiceV2() (svc2 *ServiceV2, err error) { + // service + svc := &ServiceV2{ + interval: 5 * time.Second, + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + nodeCfgSvc interfaces.NodeConfigService, + svr interfaces.GrpcServer, + handlerSvc interfaces.TaskHandlerService, + ) { + svc.nodeCfgSvc = nodeCfgSvc + svc.svr = svr + svc.handlerSvc = handlerSvc + }); err != nil { + return nil, err + } + + return svc, nil +} + +var svcV2 *ServiceV2 + +func GetTaskSchedulerServiceV2() (svr *ServiceV2, err error) { + if svcV2 != nil { + return svcV2, nil + } + svcV2, err = NewTaskSchedulerServiceV2() + if err != nil { + return nil, err + } + return svcV2, nil +} diff --git a/core/task/stats/options.go b/core/task/stats/options.go new file mode 100644 index 000000000..37e7a8239 --- /dev/null +++ b/core/task/stats/options.go @@ -0,0 +1,11 @@ +package stats + +import "github.com/crawlab-team/crawlab/core/interfaces" + +type Option func(service interfaces.TaskStatsService) + +func WithConfigPath(path string) Option { + return func(svc interfaces.TaskStatsService) { + svc.SetConfigPath(path) + } +} diff --git a/core/task/stats/service.go b/core/task/stats/service.go new file mode 100644 index 000000000..ee12d7626 --- /dev/null +++ b/core/task/stats/service.go @@ -0,0 +1,155 @@ +package stats + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/result" + "github.com/crawlab-team/crawlab/core/task" + "github.com/crawlab-team/crawlab/core/task/log" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "sync" + "time" +) + +type Service struct { + // dependencies + interfaces.TaskBaseService + nodeCfgSvc interfaces.NodeConfigService + modelSvc service.ModelService + + // internals + mu sync.Mutex + resultServices sync.Map + rsTtl time.Duration + logDriver log.Driver +} + +func (svc *Service) Init() (err error) { + go svc.cleanup() + return nil +} + +func (svc *Service) InsertData(id primitive.ObjectID, records ...interface{}) (err error) { + resultSvc, err := svc.getResultService(id) + if err != nil { + return err + } + if err := resultSvc.Insert(records...); err != nil { + return err + } + go svc.updateTaskStats(id, len(records)) + return nil +} + +func (svc *Service) InsertLogs(id primitive.ObjectID, logs ...string) (err error) { + return svc.logDriver.WriteLines(id.Hex(), logs) +} + +func (svc *Service) getResultService(id primitive.ObjectID) (resultSvc interfaces.ResultService, err error) { + // atomic operation + svc.mu.Lock() + defer svc.mu.Unlock() + + // attempt to get from cache + res, _ := svc.resultServices.Load(id.Hex()) + if res != nil { + // hit in cache + resultSvc, ok := res.(interfaces.ResultService) + resultSvc.SetTime(time.Now()) + if ok { + return resultSvc, nil + } + } + + // task + t, err := svc.modelSvc.GetTaskById(id) + if err != nil { + return nil, err + } + + // result service + resultSvc, err = result.GetResultService(t.SpiderId) + if err != nil { + return nil, err + } + + // store in cache + svc.resultServices.Store(id.Hex(), resultSvc) + + return resultSvc, nil +} + +func (svc *Service) updateTaskStats(id primitive.ObjectID, resultCount int) { + _ = mongo.GetMongoCol(interfaces.ModelColNameTaskStat).UpdateId(id, bson.M{ + "$inc": bson.M{ + "result_count": resultCount, + }, + }) +} + +func (svc *Service) cleanup() { + for { + // atomic operation + svc.mu.Lock() + + svc.resultServices.Range(func(key, value interface{}) bool { + rs := value.(interfaces.ResultService) + if time.Now().After(rs.GetTime().Add(svc.rsTtl)) { + svc.resultServices.Delete(key) + } + return true + }) + + svc.mu.Unlock() + + time.Sleep(10 * time.Minute) + } +} + +func NewTaskStatsService() (svc2 interfaces.TaskStatsService, err error) { + // base service + baseSvc, err := task.NewBaseService() + if err != nil { + return nil, trace.TraceError(err) + } + + // service + svc := &Service{ + mu: sync.Mutex{}, + TaskBaseService: baseSvc, + resultServices: sync.Map{}, + } + + // dependency injection + if err := container.GetContainer().Invoke(func(nodeCfgSvc interfaces.NodeConfigService, modelSvc service.ModelService) { + svc.nodeCfgSvc = nodeCfgSvc + svc.modelSvc = modelSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + // log driver + svc.logDriver, err = log.GetLogDriver(log.DriverTypeFile) + if err != nil { + return nil, err + } + + return svc, nil +} + +var _service interfaces.TaskStatsService + +func GetTaskStatsService() (svr interfaces.TaskStatsService, err error) { + if _service != nil { + return _service, nil + } + _service, err = NewTaskStatsService() + if err != nil { + return nil, err + } + return _service, nil +} diff --git a/core/task/stats/service_v2.go b/core/task/stats/service_v2.go new file mode 100644 index 000000000..b198c95d9 --- /dev/null +++ b/core/task/stats/service_v2.go @@ -0,0 +1,143 @@ +package stats + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/core/result" + "github.com/crawlab-team/crawlab/core/task/log" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "sync" + "time" +) + +type ServiceV2 struct { + // dependencies + nodeCfgSvc interfaces.NodeConfigService + modelSvc service.ModelService + + // internals + mu sync.Mutex + resultServices sync.Map + rsTtl time.Duration + logDriver log.Driver +} + +func (svc *ServiceV2) Init() (err error) { + go svc.cleanup() + return nil +} + +func (svc *ServiceV2) InsertData(id primitive.ObjectID, records ...interface{}) (err error) { + resultSvc, err := svc.getResultService(id) + if err != nil { + return err + } + if err := resultSvc.Insert(records...); err != nil { + return err + } + go svc.updateTaskStats(id, len(records)) + return nil +} + +func (svc *ServiceV2) InsertLogs(id primitive.ObjectID, logs ...string) (err error) { + return svc.logDriver.WriteLines(id.Hex(), logs) +} + +func (svc *ServiceV2) getResultService(id primitive.ObjectID) (resultSvc interfaces.ResultService, err error) { + // atomic operation + svc.mu.Lock() + defer svc.mu.Unlock() + + // attempt to get from cache + res, _ := svc.resultServices.Load(id.Hex()) + if res != nil { + // hit in cache + resultSvc, ok := res.(interfaces.ResultService) + resultSvc.SetTime(time.Now()) + if ok { + return resultSvc, nil + } + } + + // task + t, err := svc.modelSvc.GetTaskById(id) + if err != nil { + return nil, err + } + + // result service + resultSvc, err = result.GetResultService(t.SpiderId) + if err != nil { + return nil, err + } + + // store in cache + svc.resultServices.Store(id.Hex(), resultSvc) + + return resultSvc, nil +} + +func (svc *ServiceV2) updateTaskStats(id primitive.ObjectID, resultCount int) { + err := service.NewModelServiceV2[models.TaskStatV2]().UpdateById(id, bson.M{ + "$inc": bson.M{ + "result_count": resultCount, + }, + }) + if err != nil { + trace.PrintError(err) + } +} + +func (svc *ServiceV2) cleanup() { + for { + // atomic operation + svc.mu.Lock() + + svc.resultServices.Range(func(key, value interface{}) bool { + rs := value.(interfaces.ResultService) + if time.Now().After(rs.GetTime().Add(svc.rsTtl)) { + svc.resultServices.Delete(key) + } + return true + }) + + svc.mu.Unlock() + + time.Sleep(10 * time.Minute) + } +} + +func NewTaskStatsServiceV2() (svc2 *ServiceV2, err error) { + // service + svc := &ServiceV2{ + mu: sync.Mutex{}, + resultServices: sync.Map{}, + } + + svc.nodeCfgSvc = nodeconfig.GetNodeConfigService() + + // log driver + svc.logDriver, err = log.GetLogDriver(log.DriverTypeFile) + if err != nil { + return nil, err + } + + return svc, nil +} + +var _serviceV2 *ServiceV2 + +func GetTaskStatsServiceV2() (svr *ServiceV2, err error) { + if _serviceV2 != nil { + return _serviceV2, nil + } + _serviceV2, err = NewTaskStatsServiceV2() + if err != nil { + return nil, err + } + return _serviceV2, nil +} diff --git a/core/user/options.go b/core/user/options.go new file mode 100644 index 000000000..81c208d76 --- /dev/null +++ b/core/user/options.go @@ -0,0 +1,11 @@ +package user + +import "github.com/crawlab-team/crawlab/core/interfaces" + +type Option func(svc interfaces.UserService) + +func WithJwtSecret(secret string) Option { + return func(svc interfaces.UserService) { + svc.SetJwtSecret(secret) + } +} diff --git a/core/user/service.go b/core/user/service.go new file mode 100644 index 000000000..97a2ad9de --- /dev/null +++ b/core/user/service.go @@ -0,0 +1,238 @@ +package user + +import ( + mongo2 "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt/v5" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "time" +) + +type Service struct { + // settings variables + jwtSecret string + jwtSigningMethod jwt.SigningMethod + + // dependencies + modelSvc service.ModelService +} + +func (svc *Service) Init() (err error) { + _, err = svc.modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil) + if err == nil { + return nil + } + if err.Error() != mongo.ErrNoDocuments.Error() { + return err + } + return svc.Create(&interfaces.UserCreateOptions{ + Username: constants.DefaultAdminUsername, + Password: constants.DefaultAdminPassword, + Role: constants.RoleAdmin, + }) +} + +func (svc *Service) SetJwtSecret(secret string) { + svc.jwtSecret = secret +} + +func (svc *Service) SetJwtSigningMethod(method jwt.SigningMethod) { + svc.jwtSigningMethod = method +} + +func (svc *Service) Create(opts *interfaces.UserCreateOptions, args ...interface{}) (err error) { + actor := utils.GetUserFromArgs(args...) + + // validate options + if opts.Username == "" || opts.Password == "" { + return trace.TraceError(errors.ErrorUserMissingRequiredFields) + } + if len(opts.Password) < 5 { + return trace.TraceError(errors.ErrorUserInvalidPassword) + } + + // normalize options + if opts.Role == "" { + opts.Role = constants.RoleNormal + } + + // check if user exists + if u, err := svc.modelSvc.GetUserByUsername(opts.Username, nil); err == nil && u != nil && !u.Id.IsZero() { + return trace.TraceError(errors.ErrorUserAlreadyExists) + } + + // transaction + return mongo2.RunTransaction(func(ctx mongo.SessionContext) error { + // add user + u := &models.User{ + Username: opts.Username, + Role: opts.Role, + Email: opts.Email, + } + if err := delegate.NewModelDelegate(u, actor).Add(); err != nil { + return err + } + + // add password + p := &models.Password{ + Id: u.Id, + Password: utils.EncryptMd5(opts.Password), + } + if err := delegate.NewModelDelegate(p, actor).Add(); err != nil { + return err + } + + return nil + }) +} + +func (svc *Service) Login(opts *interfaces.UserLoginOptions) (token string, u interfaces.User, err error) { + u, err = svc.modelSvc.GetUserByUsername(opts.Username, nil) + if err != nil { + return "", nil, err + } + p, err := svc.modelSvc.GetPasswordById(u.GetId()) + if err != nil { + return "", nil, err + } + if p.Password != utils.EncryptMd5(opts.Password) { + return "", nil, errors.ErrorUserMismatch + } + token, err = svc.makeToken(u) + if err != nil { + return "", nil, err + } + return token, u, nil +} + +func (svc *Service) CheckToken(tokenStr string) (u interfaces.User, err error) { + return svc.checkToken(tokenStr) +} + +func (svc *Service) ChangePassword(id primitive.ObjectID, password string, args ...interface{}) (err error) { + actor := utils.GetUserFromArgs(args...) + + p, err := svc.modelSvc.GetPasswordById(id) + if err != nil { + return err + } + p.Password = utils.EncryptMd5(password) + if err := delegate.NewModelDelegate(p, actor).Save(); err != nil { + return err + } + return nil +} + +func (svc *Service) MakeToken(user interfaces.User) (tokenStr string, err error) { + return svc.makeToken(user) +} + +func (svc *Service) GetCurrentUser(c *gin.Context) (user interfaces.User, err error) { + // token string + tokenStr := c.GetHeader("Authorization") + + // user + u, err := userSvc.CheckToken(tokenStr) + if err != nil { + return nil, err + } + + return u, nil +} + +func (svc *Service) makeToken(user interfaces.User) (tokenStr string, err error) { + token := jwt.NewWithClaims(svc.jwtSigningMethod, jwt.MapClaims{ + "id": user.GetId(), + "username": user.GetUsername(), + "nbf": time.Now().Unix(), + }) + return token.SignedString([]byte(svc.jwtSecret)) +} + +func (svc *Service) checkToken(tokenStr string) (user interfaces.User, err error) { + token, err := jwt.Parse(tokenStr, svc.getSecretFunc()) + if err != nil { + return + } + + claim, ok := token.Claims.(jwt.MapClaims) + if !ok { + err = errors.ErrorUserInvalidType + return + } + + if !token.Valid { + err = errors.ErrorUserInvalidToken + return + } + + id, err := primitive.ObjectIDFromHex(claim["id"].(string)) + if err != nil { + return user, err + } + username := claim["username"].(string) + user, err = svc.modelSvc.GetUserById(id) + if err != nil { + err = errors.ErrorUserNotExists + return + } + + if username != user.GetUsername() { + err = errors.ErrorUserMismatch + return + } + + return +} + +func (svc *Service) getSecretFunc() jwt.Keyfunc { + return func(token *jwt.Token) (interface{}, error) { + return []byte(svc.jwtSecret), nil + } +} + +func NewUserService() (svc2 interfaces.UserService, err error) { + // service + svc := &Service{ + jwtSecret: "crawlab", + jwtSigningMethod: jwt.SigningMethodHS256, + } + + // dependency injection + if err := container.GetContainer().Invoke(func(modelSvc service.ModelService) { + svc.modelSvc = modelSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + // initialize + if err := svc.Init(); err != nil { + return nil, trace.TraceError(err) + } + + return svc, nil +} + +var userSvc interfaces.UserService + +func GetUserService() (svc interfaces.UserService, err error) { + if userSvc != nil { + return userSvc, nil + } + svc, err = NewUserService() + if err != nil { + return nil, err + } + userSvc = svc + return svc, nil +} diff --git a/core/user/service_v2.go b/core/user/service_v2.go new file mode 100644 index 000000000..e4aa72b6c --- /dev/null +++ b/core/user/service_v2.go @@ -0,0 +1,212 @@ +package user + +import ( + mongo2 "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt/v5" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "time" +) + +type ServiceV2 struct { + jwtSecret string + jwtSigningMethod jwt.SigningMethod + modelSvc *service.ModelServiceV2[models.UserV2] +} + +func (svc *ServiceV2) Init() (err error) { + _, err = svc.modelSvc.GetOne(bson.M{"username": constants.DefaultAdminUsername}, nil) + if err == nil { + return nil + } + if err.Error() != mongo.ErrNoDocuments.Error() { + return err + } + return svc.Create( + constants.DefaultAdminUsername, + constants.DefaultAdminPassword, + constants.RoleAdmin, + "", + primitive.NilObjectID, + ) +} + +func (svc *ServiceV2) SetJwtSecret(secret string) { + svc.jwtSecret = secret +} + +func (svc *ServiceV2) SetJwtSigningMethod(method jwt.SigningMethod) { + svc.jwtSigningMethod = method +} + +func (svc *ServiceV2) Create(username, password, role, email string, by primitive.ObjectID) (err error) { + // validate options + if username == "" || password == "" { + return trace.TraceError(errors.ErrorUserMissingRequiredFields) + } + if len(password) < 5 { + return trace.TraceError(errors.ErrorUserInvalidPassword) + } + + // normalize options + if role == "" { + role = constants.RoleNormal + } + + // check if user exists + if u, err := svc.modelSvc.GetOne(bson.M{"username": username}, nil); err == nil && u != nil && !u.Id.IsZero() { + return trace.TraceError(errors.ErrorUserAlreadyExists) + } + + // transaction + return mongo2.RunTransaction(func(ctx mongo.SessionContext) error { + // add user + u := models.UserV2{ + Username: username, + Role: role, + Password: utils.EncryptMd5(password), + Email: email, + } + u.SetCreated(by) + u.SetUpdated(by) + _, err = svc.modelSvc.InsertOne(u) + + return err + }) +} + +func (svc *ServiceV2) Login(username, password string) (token string, u *models.UserV2, err error) { + u, err = svc.modelSvc.GetOne(bson.M{"username": username}, nil) + if err != nil { + return "", nil, err + } + if u.Password != utils.EncryptMd5(password) { + return "", nil, errors.ErrorUserMismatch + } + token, err = svc.makeToken(u) + if err != nil { + return "", nil, err + } + return token, u, nil +} + +func (svc *ServiceV2) CheckToken(tokenStr string) (u *models.UserV2, err error) { + return svc.checkToken(tokenStr) +} + +func (svc *ServiceV2) ChangePassword(id primitive.ObjectID, password string, by primitive.ObjectID) (err error) { + u, err := svc.modelSvc.GetById(id) + if err != nil { + return err + } + u.Password = utils.EncryptMd5(password) + u.SetCreatedBy(by) + return svc.modelSvc.ReplaceById(id, *u) +} + +func (svc *ServiceV2) MakeToken(user *models.UserV2) (tokenStr string, err error) { + return svc.makeToken(user) +} + +func (svc *ServiceV2) GetCurrentUser(c *gin.Context) (user interfaces.User, err error) { + // token string + tokenStr := c.GetHeader("Authorization") + + // user + u, err := userSvc.CheckToken(tokenStr) + if err != nil { + return nil, err + } + + return u, nil +} + +func (svc *ServiceV2) makeToken(user *models.UserV2) (tokenStr string, err error) { + token := jwt.NewWithClaims(svc.jwtSigningMethod, jwt.MapClaims{ + "id": user.Id, + "username": user.Username, + "nbf": time.Now().Unix(), + }) + return token.SignedString([]byte(svc.jwtSecret)) +} + +func (svc *ServiceV2) checkToken(tokenStr string) (user *models.UserV2, err error) { + token, err := jwt.Parse(tokenStr, svc.getSecretFunc()) + if err != nil { + return + } + + claim, ok := token.Claims.(jwt.MapClaims) + if !ok { + err = errors.ErrorUserInvalidType + return + } + + if !token.Valid { + err = errors.ErrorUserInvalidToken + return + } + + id, err := primitive.ObjectIDFromHex(claim["id"].(string)) + if err != nil { + return user, err + } + username := claim["username"].(string) + user, err = svc.modelSvc.GetById(id) + if err != nil { + err = errors.ErrorUserNotExists + return + } + + if username != user.Username { + err = errors.ErrorUserMismatch + return + } + + return +} + +func (svc *ServiceV2) getSecretFunc() jwt.Keyfunc { + return func(token *jwt.Token) (interface{}, error) { + return []byte(svc.jwtSecret), nil + } +} + +func NewUserServiceV2() (svc *ServiceV2, err error) { + // service + svc = &ServiceV2{ + modelSvc: service.NewModelServiceV2[models.UserV2](), + jwtSecret: "crawlab", + jwtSigningMethod: jwt.SigningMethodHS256, + } + + // initialize + if err := svc.Init(); err != nil { + return nil, trace.TraceError(err) + } + + return svc, nil +} + +var userSvcV2 *ServiceV2 + +func GetUserServiceV2() (svc *ServiceV2, err error) { + if userSvcV2 != nil { + return userSvcV2, nil + } + svc, err = NewUserServiceV2() + if err != nil { + return nil, err + } + userSvcV2 = svc + return svc, nil +} diff --git a/core/user/test/base.go b/core/user/test/base.go new file mode 100644 index 000000000..392a7e066 --- /dev/null +++ b/core/user/test/base.go @@ -0,0 +1,65 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/user" + "go.uber.org/dig" + "testing" +) + +func init() { + var err error + T, err = NewTest() + if err != nil { + panic(err) + } +} + +var T *Test + +type Test struct { + // dependencies + modelSvc service.ModelService + userSvc interfaces.UserService + + // test data + TestUsername string + TestPassword string + TestNewPassword string +} + +func (t *Test) Setup(t2 *testing.T) { + var err error + t.userSvc, err = user.NewUserService() + if err != nil { + panic(err) + } + t2.Cleanup(t.Cleanup) +} + +func (t *Test) Cleanup() { + _ = t.modelSvc.GetBaseService(interfaces.ModelIdUser).DeleteList(nil) +} + +func NewTest() (t *Test, err error) { + // test + t = &Test{ + TestUsername: "test_username", + TestPassword: "test_password", + TestNewPassword: "test_new_password", + } + + // dependency injection + c := dig.New() + if err := c.Provide(service.GetService); err != nil { + return nil, err + } + if err := c.Invoke(func(modelSvc service.ModelService) { + t.modelSvc = modelSvc + }); err != nil { + return nil, err + } + + return t, nil +} diff --git a/core/user/test/user_service_test.go b/core/user/test/user_service_test.go new file mode 100644 index 000000000..0806c31ed --- /dev/null +++ b/core/user/test/user_service_test.go @@ -0,0 +1,61 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/stretchr/testify/require" + "testing" +) + +func TestUserService_Init(t *testing.T) { + var err error + T.Setup(t) + + u, err := T.modelSvc.GetUserByUsernameWithPassword(constants.DefaultAdminUsername, nil) + require.Nil(t, err) + require.Equal(t, constants.DefaultAdminUsername, u.Username) + require.Equal(t, utils.EncryptMd5(constants.DefaultAdminPassword), u.Password) +} + +func TestUserService_Create_Login_CheckToken(t *testing.T) { + var err error + T.Setup(t) + + err = T.userSvc.Create(&interfaces.UserCreateOptions{ + Username: T.TestUsername, + Password: T.TestPassword, + }) + require.Nil(t, err) + + u, err := T.modelSvc.GetUserByUsernameWithPassword(T.TestUsername, nil) + require.Nil(t, err) + require.Equal(t, T.TestUsername, u.Username) + require.Equal(t, utils.EncryptMd5(T.TestPassword), u.Password) + + token, u2, err := T.userSvc.Login(&interfaces.UserLoginOptions{ + Username: T.TestUsername, + Password: T.TestPassword, + }) + require.Nil(t, err) + require.Greater(t, len(token), 10) + require.Equal(t, u.Username, u2.GetUsername()) + + u3, err := T.userSvc.CheckToken(token) + require.Nil(t, err) + require.Equal(t, u.Username, u3.GetUsername()) +} + +func TestUserService_ChangePassword(t *testing.T) { + var err error + T.Setup(t) + + u, err := T.modelSvc.GetUserByUsernameWithPassword(constants.DefaultAdminUsername, nil) + require.Nil(t, err) + err = T.userSvc.ChangePassword(u.Id, T.TestNewPassword) + require.Nil(t, err) + + u2, err := T.modelSvc.GetUserByUsernameWithPassword(constants.DefaultAdminUsername, nil) + require.Nil(t, err) + require.Equal(t, utils.EncryptMd5(T.TestNewPassword), u2.Password) +} diff --git a/core/utils/args.go b/core/utils/args.go new file mode 100644 index 000000000..e83f540a9 --- /dev/null +++ b/core/utils/args.go @@ -0,0 +1,17 @@ +package utils + +import "github.com/crawlab-team/crawlab/core/interfaces" + +func GetUserFromArgs(args ...interface{}) (u interfaces.User) { + for _, arg := range args { + switch arg.(type) { + case interfaces.User: + var ok bool + u, ok = arg.(interfaces.User) + if ok { + return u + } + } + } + return nil +} diff --git a/core/utils/array.go b/core/utils/array.go new file mode 100644 index 000000000..a5e958c69 --- /dev/null +++ b/core/utils/array.go @@ -0,0 +1,46 @@ +package utils + +import ( + "errors" + "math/rand" + "reflect" + "time" +) + +func StringArrayContains(arr []string, str string) bool { + for _, s := range arr { + if s == str { + return true + } + } + return false +} + +func GetArrayItems(array interface{}) (res []interface{}, err error) { + switch reflect.TypeOf(array).Kind() { + case reflect.Slice, reflect.Array: + s := reflect.ValueOf(array) + for i := 0; i < s.Len(); i++ { + obj, ok := s.Index(i).Interface().(interface{}) + if !ok { + return nil, errors.New("invalid type") + } + res = append(res, obj) + } + default: + return nil, errors.New("invalid type") + } + return res, nil +} + +func ShuffleArray(slice []interface{}) (err error) { + r := rand.New(rand.NewSource(time.Now().Unix())) + for len(slice) > 0 { + n := len(slice) + randIndex := r.Intn(n) + slice[n-1], slice[randIndex] = slice[randIndex], slice[n-1] + slice = slice[:n-1] + } + + return nil +} diff --git a/core/utils/backoff.go b/core/utils/backoff.go new file mode 100644 index 000000000..53658a8d7 --- /dev/null +++ b/core/utils/backoff.go @@ -0,0 +1,15 @@ +package utils + +import ( + "github.com/apex/log" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/go-trace" + "time" +) + +func BackoffErrorNotify(prefix string) backoff.Notify { + return func(err error, duration time.Duration) { + log.Errorf("%s error: %v. reattempt in %.1f seconds...", prefix, err, duration.Seconds()) + trace.PrintError(err) + } +} diff --git a/core/utils/binders/binder_col_name.go b/core/utils/binders/binder_col_name.go new file mode 100644 index 000000000..9e76224d3 --- /dev/null +++ b/core/utils/binders/binder_col_name.go @@ -0,0 +1,99 @@ +package binders + +import ( + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" +) + +func NewColNameBinder(id interfaces.ModelId) (b *ColNameBinder) { + return &ColNameBinder{id: id} +} + +type ColNameBinder struct { + id interfaces.ModelId +} + +func (b *ColNameBinder) Bind() (res interface{}, err error) { + switch b.id { + // system models + case interfaces.ModelIdArtifact: + return interfaces.ModelColNameArtifact, nil + case interfaces.ModelIdTag: + return interfaces.ModelColNameTag, nil + + // operation models + case interfaces.ModelIdNode: + return interfaces.ModelColNameNode, nil + case interfaces.ModelIdProject: + return interfaces.ModelColNameProject, nil + case interfaces.ModelIdSpider: + return interfaces.ModelColNameSpider, nil + case interfaces.ModelIdTask: + return interfaces.ModelColNameTask, nil + case interfaces.ModelIdJob: + return interfaces.ModelColNameJob, nil + case interfaces.ModelIdSchedule: + return interfaces.ModelColNameSchedule, nil + case interfaces.ModelIdUser: + return interfaces.ModelColNameUser, nil + case interfaces.ModelIdSetting: + return interfaces.ModelColNameSetting, nil + case interfaces.ModelIdToken: + return interfaces.ModelColNameToken, nil + case interfaces.ModelIdVariable: + return interfaces.ModelColNameVariable, nil + case interfaces.ModelIdTaskQueue: + return interfaces.ModelColNameTaskQueue, nil + case interfaces.ModelIdTaskStat: + return interfaces.ModelColNameTaskStat, nil + case interfaces.ModelIdSpiderStat: + return interfaces.ModelColNameSpiderStat, nil + case interfaces.ModelIdDataSource: + return interfaces.ModelColNameDataSource, nil + case interfaces.ModelIdDataCollection: + return interfaces.ModelColNameDataCollection, nil + case interfaces.ModelIdPassword: + return interfaces.ModelColNamePasswords, nil + case interfaces.ModelIdExtraValue: + return interfaces.ModelColNameExtraValues, nil + case interfaces.ModelIdGit: + return interfaces.ModelColNameGit, nil + case interfaces.ModelIdRole: + return interfaces.ModelColNameRole, nil + case interfaces.ModelIdUserRole: + return interfaces.ModelColNameUserRole, nil + case interfaces.ModelIdPermission: + return interfaces.ModelColNamePermission, nil + case interfaces.ModelIdRolePermission: + return interfaces.ModelColNameRolePermission, nil + case interfaces.ModelIdEnvironment: + return interfaces.ModelColNameEnvironment, nil + case interfaces.ModelIdDependencySetting: + return interfaces.ModelColNameDependencySetting, nil + + // invalid + default: + return res, errors.ErrorModelNotImplemented + } +} + +func (b *ColNameBinder) MustBind() (res interface{}) { + res, err := b.Bind() + if err != nil { + panic(err) + } + return res +} + +func (b *ColNameBinder) BindString() (res string, err error) { + res_, err := b.Bind() + if err != nil { + return "", err + } + res = res_.(string) + return res, nil +} + +func (b *ColNameBinder) MustBindString() (res string) { + return b.MustBind().(string) +} diff --git a/core/utils/bool.go b/core/utils/bool.go new file mode 100644 index 000000000..8d569683e --- /dev/null +++ b/core/utils/bool.go @@ -0,0 +1,12 @@ +package utils + +import "github.com/spf13/viper" + +func EnvIsTrue(key string, defaultOk bool) bool { + isTrueBool := viper.GetBool(key) + isTrueString := viper.GetString(key) + if isTrueString == "" { + return defaultOk + } + return isTrueBool || isTrueString == "Y" +} diff --git a/core/utils/bson.go b/core/utils/bson.go new file mode 100644 index 000000000..e6443e888 --- /dev/null +++ b/core/utils/bson.go @@ -0,0 +1,125 @@ +package utils + +import ( + "github.com/emirpasic/gods/sets/hashset" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "reflect" +) + +func BsonMEqual(v1, v2 bson.M) (ok bool) { + //ok = reflect.DeepEqual(v1, v2) + ok = bsonMEqual(v1, v2) + return ok +} + +func bsonMEqual(v1, v2 bson.M) (ok bool) { + // all keys + allKeys := hashset.New() + for key := range v1 { + allKeys.Add(key) + } + for key := range v2 { + allKeys.Add(key) + } + + for _, keyRes := range allKeys.Values() { + key := keyRes.(string) + v1Value, ok := v1[key] + if !ok { + return false + } + v2Value, ok := v2[key] + if !ok { + return false + } + + mode := 0 + + var v1ValueBsonM bson.M + var v1ValueBsonA bson.A + switch v1Value.(type) { + case bson.M: + mode = 1 + v1ValueBsonM = v1Value.(bson.M) + case bson.A: + mode = 2 + v1ValueBsonA = v1Value.(bson.A) + } + + var v2ValueBsonM bson.M + var v2ValueBsonA bson.A + switch v2Value.(type) { + case bson.M: + if mode != 1 { + return false + } + v2ValueBsonM = v2Value.(bson.M) + case bson.A: + if mode != 2 { + return false + } + v2ValueBsonA = v2Value.(bson.A) + } + + switch mode { + case 0: + if v1Value != v2Value { + return false + } + case 1: + if !bsonMEqual(v1ValueBsonM, v2ValueBsonM) { + return false + } + case 2: + if !reflect.DeepEqual(v1ValueBsonA, v2ValueBsonA) { + return false + } + default: + // not reachable + return false + } + } + + return true +} + +func NormalizeBsonMObjectId(m bson.M) (res bson.M) { + for k, v := range m { + switch v.(type) { + case string: + oid, err := primitive.ObjectIDFromHex(v.(string)) + if err == nil { + m[k] = oid + } + case bson.M: + m[k] = NormalizeBsonMObjectId(v.(bson.M)) + } + } + return m +} + +func DenormalizeBsonMObjectId(m bson.M) (res bson.M) { + for k, v := range m { + switch v.(type) { + case primitive.ObjectID: + m[k] = v.(primitive.ObjectID).Hex() + case bson.M: + m[k] = NormalizeBsonMObjectId(v.(bson.M)) + } + } + return m +} + +func NormalizeObjectId(v interface{}) (res interface{}) { + switch v.(type) { + case string: + oid, err := primitive.ObjectIDFromHex(v.(string)) + if err != nil { + return v + } + return oid + default: + return v + } +} diff --git a/core/utils/cache.go b/core/utils/cache.go new file mode 100644 index 000000000..b875b316f --- /dev/null +++ b/core/utils/cache.go @@ -0,0 +1,57 @@ +package utils + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "time" +) + +func GetFromDbCache(key string, getFn func() (string, error)) (res string, err error) { + col := mongo.GetMongoCol(constants.CacheColName) + + var d bson.M + if err := col.Find(bson.M{ + constants.CacheColKey: key, + }, nil).One(&d); err != nil { + if err != mongo2.ErrNoDocuments { + return "", err + } + + // get cache value + res, err = getFn() + if err != nil { + return "", err + } + + // save cache + d = bson.M{ + constants.CacheColKey: key, + constants.CacheColValue: res, + constants.CacheColTime: time.Now(), + } + if _, err := col.Insert(d); err != nil { + return "", err + } + return res, nil + } + + // type conversion + r, ok := d[constants.CacheColValue] + if !ok { + if err := col.Delete(bson.M{constants.CacheColKey: key}); err != nil { + return "", err + } + return GetFromDbCache(key, getFn) + } + res, ok = r.(string) + if !ok { + if err := col.Delete(bson.M{constants.CacheColKey: key}); err != nil { + return "", err + } + return GetFromDbCache(key, getFn) + } + + return res, nil +} diff --git a/core/utils/chan.go b/core/utils/chan.go new file mode 100644 index 000000000..c0144340d --- /dev/null +++ b/core/utils/chan.go @@ -0,0 +1,40 @@ +package utils + +import ( + "sync" +) + +var TaskExecChanMap = NewChanMap() + +type ChanMap struct { + m sync.Map +} + +func NewChanMap() *ChanMap { + return &ChanMap{m: sync.Map{}} +} + +func (cm *ChanMap) Chan(key string) chan string { + if ch, ok := cm.m.Load(key); ok { + return ch.(interface{}).(chan string) + } + ch := make(chan string, 10) + cm.m.Store(key, ch) + return ch +} + +func (cm *ChanMap) ChanBlocked(key string) chan string { + if ch, ok := cm.m.Load(key); ok { + return ch.(interface{}).(chan string) + } + ch := make(chan string) + cm.m.Store(key, ch) + return ch +} + +func (cm *ChanMap) HasChanKey(key string) bool { + if _, ok := cm.m.Load(key); ok { + return true + } + return false +} diff --git a/core/utils/chan_test.go b/core/utils/chan_test.go new file mode 100644 index 000000000..4bc759179 --- /dev/null +++ b/core/utils/chan_test.go @@ -0,0 +1,78 @@ +package utils + +import ( + . "github.com/smartystreets/goconvey/convey" + "sync" + "testing" +) + +func TestNewChanMap(t *testing.T) { + mapTest := sync.Map{} + chanTest := make(chan string) + test := "test" + + Convey("Call NewChanMap to generate ChanMap", t, func() { + mapTest.Store("test", chanTest) + chanMapTest := ChanMap{mapTest} + chanMap := NewChanMap() + chanMap.m.Store("test", chanTest) + + Convey(test, func() { + v1, ok := chanMap.m.Load("test") + So(ok, ShouldBeTrue) + v2, ok := chanMapTest.m.Load("test") + So(ok, ShouldBeTrue) + So(v1, ShouldResemble, v2) + }) + }) +} + +func TestChan(t *testing.T) { + mapTest := sync.Map{} + chanTest := make(chan string) + mapTest.Store("test", chanTest) + chanMapTest := ChanMap{mapTest} + + Convey("Test Chan use exist key", t, func() { + ch1 := chanMapTest.Chan("test") + Convey("ch1 should equal chanTest", func() { + So(ch1, ShouldEqual, chanTest) + }) + }) + Convey("Test Chan use no-exist key", t, func() { + ch2 := chanMapTest.Chan("test2") + Convey("ch2 should equal chanMapTest.m[test2]", func() { + v, ok := chanMapTest.m.Load("test2") + So(ok, ShouldBeTrue) + So(v, ShouldEqual, ch2) + }) + Convey("Cap of chanMapTest.m[test2] should equal 10", func() { + So(10, ShouldEqual, cap(ch2)) + }) + }) +} + +func TestChanBlocked(t *testing.T) { + mapTest := sync.Map{} + chanTest := make(chan string) + mapTest.Store("test", chanTest) + chanMapTest := ChanMap{mapTest} + + Convey("Test Chan use exist key", t, func() { + ch1 := chanMapTest.ChanBlocked("test") + Convey("ch1 should equal chanTest", func() { + So(ch1, ShouldEqual, chanTest) + }) + }) + Convey("Test Chan use no-exist key", t, func() { + ch2 := chanMapTest.ChanBlocked("test2") + Convey("ch2 should equal chanMapTest.m[test2]", func() { + v, ok := chanMapTest.m.Load("test2") + So(ok, ShouldBeTrue) + So(v, ShouldEqual, ch2) + }) + Convey("Cap of chanMapTest.m[test2] should equal 10", func() { + So(0, ShouldEqual, cap(ch2)) + }) + }) +} diff --git a/core/utils/cockroachdb.go b/core/utils/cockroachdb.go new file mode 100644 index 000000000..650c32f1e --- /dev/null +++ b/core/utils/cockroachdb.go @@ -0,0 +1,60 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/mssql" + "time" +) + +func GetCockroachdbSession(ds *models.DataSource) (s db.Session, err error) { + return getCockroachdbSession(context.Background(), ds) +} + +func GetCockroachdbSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getCockroachdbSession(ctx, ds) +} + +func getCockroachdbSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultCockroachdbPort + } + + // connect settings + settings := mssql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mssql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/cron.go b/core/utils/cron.go new file mode 100644 index 000000000..e8838d2f5 --- /dev/null +++ b/core/utils/cron.go @@ -0,0 +1,176 @@ +package utils + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +// cronBounds provides a range of acceptable values (plus a map of name to value). +type cronBounds struct { + min, max uint + names map[string]uint +} + +type cronUtils struct { + // The cronBounds for each field. + seconds cronBounds + minutes cronBounds + hours cronBounds + dom cronBounds + months cronBounds + dow cronBounds + + // Set the top bit if a star was included in the expression. + starBit uint64 +} + +// getRange returns the bits indicated by the given expression: +// number | number "-" number [ "/" number ] +// or error parsing range. +func (u *cronUtils) getRange(expr string, r cronBounds) (uint64, error) { + var ( + start, end, step uint + rangeAndStep = strings.Split(expr, "/") + lowAndHigh = strings.Split(rangeAndStep[0], "-") + singleDigit = len(lowAndHigh) == 1 + err error + ) + + var extra uint64 + if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" { + start = r.min + end = r.max + extra = CronUtils.starBit + } else { + start, err = u.parseIntOrName(lowAndHigh[0], r.names) + if err != nil { + return 0, err + } + switch len(lowAndHigh) { + case 1: + end = start + case 2: + end, err = u.parseIntOrName(lowAndHigh[1], r.names) + if err != nil { + return 0, err + } + default: + return 0, fmt.Errorf("too many hyphens: %s", expr) + } + } + + switch len(rangeAndStep) { + case 1: + step = 1 + case 2: + step, err = u.mustParseInt(rangeAndStep[1]) + if err != nil { + return 0, err + } + + // Special handling: "N/step" means "N-max/step". + if singleDigit { + end = r.max + } + if step > 1 { + extra = 0 + } + default: + return 0, fmt.Errorf("too many slashes: %s", expr) + } + + if start < r.min { + return 0, fmt.Errorf("beginning of range (%d) below minimum (%d): %s", start, r.min, expr) + } + if end > r.max { + return 0, fmt.Errorf("end of range (%d) above maximum (%d): %s", end, r.max, expr) + } + if start > end { + return 0, fmt.Errorf("beginning of range (%d) beyond end of range (%d): %s", start, end, expr) + } + if step == 0 { + return 0, fmt.Errorf("step of range should be a positive number: %s", expr) + } + + return u.getBits(start, end, step) | extra, nil +} + +// parseIntOrName returns the (possibly-named) integer contained in expr. +func (u *cronUtils) parseIntOrName(expr string, names map[string]uint) (uint, error) { + if names != nil { + if namedInt, ok := names[strings.ToLower(expr)]; ok { + return namedInt, nil + } + } + return u.mustParseInt(expr) +} + +// mustParseInt parses the given expression as an int or returns an error. +func (u *cronUtils) mustParseInt(expr string) (uint, error) { + num, err := strconv.Atoi(expr) + if err != nil { + return 0, fmt.Errorf("failed to parse int from %s: %s", expr, err) + } + if num < 0 { + return 0, fmt.Errorf("negative number (%d) not allowed: %s", num, expr) + } + + return uint(num), nil +} + +// getBits sets all bits in the range [min, max], modulo the given step size. +func (u *cronUtils) getBits(min, max, step uint) uint64 { + var bits uint64 + + // If step is 1, use shifts. + if step == 1 { + return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min) + } + + // Else, use a simple loop. + for i := min; i <= max; i += step { + bits |= 1 << i + } + return bits +} + +// all returns all bits within the given cronBounds. (plus the star bit) +func (u *cronUtils) all(r cronBounds) uint64 { + return u.getBits(r.min, r.max, 1) | CronUtils.starBit +} + +var CronUtils = cronUtils{ + // The cronBounds for each field. + seconds: cronBounds{0, 59, nil}, + minutes: cronBounds{0, 59, nil}, + hours: cronBounds{0, 23, nil}, + dom: cronBounds{1, 31, nil}, + months: cronBounds{1, 12, map[string]uint{ + "jan": 1, + "feb": 2, + "mar": 3, + "apr": 4, + "may": 5, + "jun": 6, + "jul": 7, + "aug": 8, + "sep": 9, + "oct": 10, + "nov": 11, + "dec": 12, + }}, + dow: cronBounds{0, 6, map[string]uint{ + "sun": 0, + "mon": 1, + "tue": 2, + "wed": 3, + "thu": 4, + "fri": 5, + "sat": 6, + }}, + + // Set the top bit if a star was included in the expression. + starBit: 1 << 63, +} diff --git a/core/utils/debug.go b/core/utils/debug.go new file mode 100644 index 000000000..09ef3098e --- /dev/null +++ b/core/utils/debug.go @@ -0,0 +1,18 @@ +package utils + +import ( + "fmt" + "github.com/spf13/viper" + "time" +) + +func IsDebug() bool { + return viper.GetBool("debug") +} + +func LogDebug(msg string) { + if !IsDebug() { + return + } + fmt.Println(fmt.Sprintf("[DEBUG] %s: %s", time.Now().Format("2006-01-02 15:04:05"), msg)) +} diff --git a/core/utils/demo.go b/core/utils/demo.go new file mode 100644 index 000000000..19e31dc51 --- /dev/null +++ b/core/utils/demo.go @@ -0,0 +1,57 @@ +package utils + +import ( + "fmt" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/sys_exec" + "github.com/crawlab-team/go-trace" + "github.com/spf13/viper" +) + +func GetApiAddress() (res string) { + apiAddress := viper.GetString("api.address") + if apiAddress == "" { + return "http://localhost:8000" + } + return apiAddress +} + +func IsDemo() (ok bool) { + return EnvIsTrue("demo", true) +} + +func InitializedDemo() (ok bool) { + col := mongo.GetMongoCol("users") + n, err := col.Count(nil) + if err != nil { + return true + } + return n > 0 +} + +func ImportDemo() (err error) { + cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo import", GetApiAddress()) + cmd := sys_exec.BuildCmd(cmdStr) + if err := cmd.Run(); err != nil { + trace.PrintError(err) + } + return nil +} + +func ReimportDemo() (err error) { + cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo reimport", GetApiAddress()) + cmd := sys_exec.BuildCmd(cmdStr) + if err := cmd.Run(); err != nil { + trace.PrintError(err) + } + return nil +} + +func CleanupDemo() (err error) { + cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo reimport", GetApiAddress()) + cmd := sys_exec.BuildCmd(cmdStr) + if err := cmd.Run(); err != nil { + trace.PrintError(err) + } + return nil +} diff --git a/core/utils/di.go b/core/utils/di.go new file mode 100644 index 000000000..8c467228e --- /dev/null +++ b/core/utils/di.go @@ -0,0 +1,18 @@ +package utils + +import ( + "github.com/crawlab-team/go-trace" + "github.com/spf13/viper" + "go.uber.org/dig" + "os" +) + +func VisualizeContainer(c *dig.Container) (err error) { + if !viper.GetBool("debug.di.visualize") { + return nil + } + if err := dig.Visualize(c, os.Stdout); err != nil { + return trace.TraceError(err) + } + return nil +} diff --git a/core/utils/docker.go b/core/utils/docker.go new file mode 100644 index 000000000..a1c98ed18 --- /dev/null +++ b/core/utils/docker.go @@ -0,0 +1,5 @@ +package utils + +func IsDocker() (ok bool) { + return EnvIsTrue("docker", false) +} diff --git a/core/utils/encrypt.go b/core/utils/encrypt.go new file mode 100644 index 000000000..d07540540 --- /dev/null +++ b/core/utils/encrypt.go @@ -0,0 +1,80 @@ +package utils + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/md5" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "io" +) + +func GetSecretKey() string { + return constants.DefaultEncryptServerKey +} + +func GetSecretKeyBytes() []byte { + return []byte(GetSecretKey()) +} + +func ComputeHmacSha256(message string, secret string) string { + key := []byte(secret) + h := hmac.New(sha256.New, key) + h.Write([]byte(message)) + sha := hex.EncodeToString(h.Sum(nil)) + return base64.StdEncoding.EncodeToString([]byte(sha)) +} + +func EncryptMd5(str string) string { + w := md5.New() + _, _ = io.WriteString(w, str) + md5str := fmt.Sprintf("%x", w.Sum(nil)) + return md5str +} + +func padding(src []byte, blockSize int) []byte { + padNum := blockSize - len(src)%blockSize + pad := bytes.Repeat([]byte{byte(padNum)}, padNum) + return append(src, pad...) +} + +func unPadding(src []byte) []byte { + n := len(src) + unPadNum := int(src[n-1]) + return src[:n-unPadNum] +} + +func EncryptAES(src string) (res string, err error) { + srcBytes := []byte(src) + key := GetSecretKeyBytes() + block, err := aes.NewCipher(key) + if err != nil { + return res, err + } + srcBytes = padding(srcBytes, block.BlockSize()) + blockMode := cipher.NewCBCEncrypter(block, key) + blockMode.CryptBlocks(srcBytes, srcBytes) + res = hex.EncodeToString(srcBytes) + return res, nil +} + +func DecryptAES(src string) (res string, err error) { + srcBytes, err := hex.DecodeString(src) + if err != nil { + return res, err + } + key := GetSecretKeyBytes() + block, err := aes.NewCipher(key) + if err != nil { + return res, err + } + blockMode := cipher.NewCBCDecrypter(block, key) + blockMode.CryptBlocks(srcBytes, srcBytes) + res = string(unPadding(srcBytes)) + return res, nil +} diff --git a/core/utils/encrypt_test.go b/core/utils/encrypt_test.go new file mode 100644 index 000000000..d393fa5bc --- /dev/null +++ b/core/utils/encrypt_test.go @@ -0,0 +1,20 @@ +package utils + +import ( + "fmt" + "github.com/stretchr/testify/require" + "testing" +) + +func TestEncryptAesPassword(t *testing.T) { + plainText := "crawlab" + encryptedText, err := EncryptAES(plainText) + require.Nil(t, err) + decryptedText, err := DecryptAES(encryptedText) + require.Nil(t, err) + fmt.Println(fmt.Sprintf("plainText: %s", plainText)) + fmt.Println(fmt.Sprintf("encryptedText: %s", encryptedText)) + fmt.Println(fmt.Sprintf("decryptedText: %s", decryptedText)) + require.Equal(t, decryptedText, plainText) + require.NotEqual(t, decryptedText, encryptedText) +} diff --git a/core/utils/es.go b/core/utils/es.go new file mode 100644 index 000000000..587e17157 --- /dev/null +++ b/core/utils/es.go @@ -0,0 +1,159 @@ +package utils + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/go-trace" + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +func GetElasticsearchClient(ds *models.DataSource) (c *elasticsearch.Client, err error) { + return getElasticsearchClient(context.Background(), ds) +} + +func GetElasticsearchClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c *elasticsearch.Client, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getElasticsearchClient(ctx, ds) +} + +func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elasticsearch.Client, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultElasticsearchPort + } + + // es hosts + addresses := []string{ + fmt.Sprintf("http://%s:%s", host, port), + } + + // retry backoff + rb := backoff.NewExponentialBackOff() + + // es client options + cfg := elasticsearch.Config{ + Addresses: addresses, + Username: ds.Username, + Password: ds.Password, + //CloudID: "", + //APIKey: "", + //ServiceToken: "", + //CertificateFingerprint: "", + //Header: nil, + //CACert: nil, + //RetryOnStatus: nil, + //DisableRetry: false, + //EnableRetryOnTimeout: false, + //MaxRetries: 0, + //CompressRequestBody: false, + //DiscoverNodesOnStart: false, + //DiscoverNodesInterval: 0, + //EnableMetrics: false, + //EnableDebugLogger: false, + //EnableCompatibilityMode: false, + //DisableMetaHeader: false, + //UseResponseCheckOnly: false, + RetryBackoff: func(i int) time.Duration { + if i == 1 { + rb.Reset() + } + return rb.NextBackOff() + }, + //Transport: nil, + //Logger: nil, + //Selector: nil, + //ConnectionPoolFunc: nil, + } + + // es client + done := make(chan struct{}) + go func() { + c, err = elasticsearch.NewClient(cfg) + if err != nil { + return + } + var res *esapi.Response + res, err = c.Info() + fmt.Println(res) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return c, err +} + +func GetElasticsearchQuery(query generic.ListQuery) (buf *bytes.Buffer) { + q := map[string]interface{}{} + if len(query) > 0 { + match := getElasticsearchQueryMatch(query) + q["query"] = map[string]interface{}{ + "match": match, + } + } + buf = &bytes.Buffer{} + if err := json.NewEncoder(buf).Encode(q); err != nil { + trace.PrintError(err) + } + return buf +} + +func GetElasticsearchQueryWithOptions(query generic.ListQuery, opts *generic.ListOptions) (buf *bytes.Buffer) { + q := map[string]interface{}{ + "size": opts.Limit, + "from": opts.Skip, + // TODO: sort + } + if len(query) > 0 { + match := getElasticsearchQueryMatch(query) + q["query"] = map[string]interface{}{ + "match": match, + } + } + buf = &bytes.Buffer{} + if err := json.NewEncoder(buf).Encode(q); err != nil { + trace.PrintError(err) + } + return buf +} + +func getElasticsearchQueryMatch(query generic.ListQuery) (match map[string]interface{}) { + match = map[string]interface{}{} + for _, c := range query { + switch c.Value.(type) { + case primitive.ObjectID: + c.Value = c.Value.(primitive.ObjectID).Hex() + } + switch c.Op { + case generic.OpEqual: + match[c.Key] = c.Value + default: + match[c.Key] = map[string]interface{}{ + c.Op: c.Value, + } + } + } + return match +} diff --git a/core/utils/file.go b/core/utils/file.go new file mode 100644 index 000000000..216a129a9 --- /dev/null +++ b/core/utils/file.go @@ -0,0 +1,382 @@ +package utils + +import ( + "archive/zip" + "crypto/md5" + "encoding/hex" + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "runtime/debug" +) + +func OpenFile(fileName string) *os.File { + file, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, os.ModePerm) + if err != nil { + log.Errorf("create file error: %s, file_name: %s", err.Error(), fileName) + debug.PrintStack() + return nil + } + return file +} + +func Exists(path string) bool { + _, err := os.Stat(path) //os.Stat获取文件信息 + if err != nil { + return os.IsExist(err) + } + return true +} + +func IsDir(path string) bool { + s, err := os.Stat(path) + if err != nil { + return false + } + return s.IsDir() +} + +// ListDir Add: 增加error类型作为第二返回值 +// 在其他函数如 /task/log/file_driver.go中的 *FileLogDriver.cleanup()函数调用时 +// 可以通过判断err是否为nil来判断是否有错误发生 +func ListDir(path string) ([]fs.FileInfo, error) { + list, err := os.ReadDir(path) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return nil, err + } + + var res []fs.FileInfo + for _, item := range list { + info, err := item.Info() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return nil, err + } + res = append(res, info) + } + return res, nil +} + +func DeCompress(srcFile *os.File, dstPath string) error { + // 如果保存路径不存在，创建一个 + if !Exists(dstPath) { + if err := os.MkdirAll(dstPath, os.ModePerm); err != nil { + debug.PrintStack() + return err + } + } + + // 读取zip文件 + zipFile, err := zip.OpenReader(srcFile.Name()) + if err != nil { + log.Errorf("Unzip File Error：" + err.Error()) + debug.PrintStack() + return err + } + defer Close(zipFile) + + // 遍历zip内所有文件和目录 + for _, innerFile := range zipFile.File { + // 获取该文件数据 + info := innerFile.FileInfo() + + // 如果是目录，则创建一个 + if info.IsDir() { + err = os.MkdirAll(filepath.Join(dstPath, innerFile.Name), os.ModeDir|os.ModePerm) + if err != nil { + log.Errorf("Unzip File Error : " + err.Error()) + debug.PrintStack() + return err + } + continue + } + + // 如果文件目录不存在，则创建一个 + dirPath := filepath.Join(dstPath, filepath.Dir(innerFile.Name)) + if !Exists(dirPath) { + if err = os.MkdirAll(dirPath, os.ModeDir|os.ModePerm); err != nil { + log.Errorf("Unzip File Error : " + err.Error()) + debug.PrintStack() + return err + } + } + + // 打开该文件 + srcFile, err := innerFile.Open() + if err != nil { + log.Errorf("Unzip File Error : " + err.Error()) + debug.PrintStack() + continue + } + + // 创建新文件 + newFilePath := filepath.Join(dstPath, innerFile.Name) + newFile, err := os.OpenFile(newFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, info.Mode()) + if err != nil { + log.Errorf("Unzip File Error : " + err.Error()) + debug.PrintStack() + continue + } + + // 拷贝该文件到新文件中 + if _, err := io.Copy(newFile, srcFile); err != nil { + debug.PrintStack() + return err + } + + // 关闭该文件 + if err := srcFile.Close(); err != nil { + debug.PrintStack() + return err + } + + // 关闭新文件 + if err := newFile.Close(); err != nil { + debug.PrintStack() + return err + } + } + return nil +} + +// Compress 压缩文件 +// files 文件数组，可以是不同dir下的文件或者文件夹 +// dest 压缩文件存放地址 +func Compress(files []*os.File, dest string) error { + d, _ := os.Create(dest) + defer Close(d) + w := zip.NewWriter(d) + defer Close(w) + for _, file := range files { + if err := _Compress(file, "", w); err != nil { + return err + } + } + return nil +} + +func _Compress(file *os.File, prefix string, zw *zip.Writer) error { + info, err := file.Stat() + if err != nil { + debug.PrintStack() + return err + } + if info.IsDir() { + prefix = prefix + "/" + info.Name() + fileInfos, err := file.Readdir(-1) + if err != nil { + debug.PrintStack() + return err + } + for _, fi := range fileInfos { + f, err := os.Open(file.Name() + "/" + fi.Name()) + if err != nil { + debug.PrintStack() + return err + } + err = _Compress(f, prefix, zw) + if err != nil { + debug.PrintStack() + return err + } + } + } else { + header, err := zip.FileInfoHeader(info) + if err != nil { + debug.PrintStack() + return err + } + header.Name = prefix + "/" + header.Name + writer, err := zw.CreateHeader(header) + if err != nil { + debug.PrintStack() + return err + } + _, err = io.Copy(writer, file) + Close(file) + if err != nil { + debug.PrintStack() + return err + } + } + return nil +} + +func TrimFileData(data []byte) (res []byte) { + if string(data) == constants.EmptyFileData { + return res + } + return data +} + +func ZipDirectory(dir, zipfile string) error { + zipFile, err := os.Create(zipfile) + if err != nil { + return err + } + defer zipFile.Close() + + zipWriter := zip.NewWriter(zipFile) + defer zipWriter.Close() + + baseDir := filepath.Dir(dir) + + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + relPath, err := filepath.Rel(baseDir, path) + if err != nil { + return err + } + + zipFile, err := zipWriter.Create(relPath) + if err != nil { + return err + } + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + _, err = io.Copy(zipFile, file) + if err != nil { + return err + } + + return nil + }) + + return err +} + +// CopyFile File copies a single file from src to dst +func CopyFile(src, dst string) error { + var err error + var srcFd *os.File + var dstFd *os.File + var srcInfo os.FileInfo + + if srcFd, err = os.Open(src); err != nil { + return err + } + defer srcFd.Close() + + if dstFd, err = os.Create(dst); err != nil { + return err + } + defer dstFd.Close() + + if _, err = io.Copy(dstFd, srcFd); err != nil { + return err + } + if srcInfo, err = os.Stat(src); err != nil { + return err + } + return os.Chmod(dst, srcInfo.Mode()) +} + +// CopyDir Dir copies a whole directory recursively +func CopyDir(src string, dst string) error { + var err error + var fds []os.DirEntry + var srcInfo os.FileInfo + + if srcInfo, err = os.Stat(src); err != nil { + return err + } + + if err = os.MkdirAll(dst, srcInfo.Mode()); err != nil { + return err + } + + if fds, err = os.ReadDir(src); err != nil { + return err + } + for _, fd := range fds { + srcfp := path.Join(src, fd.Name()) + dstfp := path.Join(dst, fd.Name()) + + if fd.IsDir() { + if err = CopyDir(srcfp, dstfp); err != nil { + fmt.Println(err) + } + } else { + if err = CopyFile(srcfp, dstfp); err != nil { + fmt.Println(err) + } + } + } + return nil +} + +func GetFileHash(filePath string) (res string, err error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + defer file.Close() + + hash := md5.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func ScanDirectory(dir string) (res map[string]entity.FsFileInfo, err error) { + files := make(map[string]entity.FsFileInfo) + + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + hash, err := GetFileHash(path) + if err != nil { + return err + } + + relPath, err := filepath.Rel(dir, path) + if err != nil { + return err + } + + files[relPath] = entity.FsFileInfo{ + Name: info.Name(), + Path: relPath, + FullPath: path, + Extension: filepath.Ext(path), + FileSize: info.Size(), + ModTime: info.ModTime(), + Mode: info.Mode(), + Hash: hash, + } + return nil + }) + if err != nil { + return nil, err + } + + return files, nil +} diff --git a/core/utils/file_test.go b/core/utils/file_test.go new file mode 100644 index 000000000..4af32d0db --- /dev/null +++ b/core/utils/file_test.go @@ -0,0 +1,129 @@ +package utils + +import ( + "archive/zip" + . "github.com/smartystreets/goconvey/convey" + "io" + "log" + "os" + "runtime/debug" + "testing" +) + +func TestExists(t *testing.T) { + var pathString = "../config" + var wrongPathString = "test" + + Convey("Test path or file is Exists or not", t, func() { + res := Exists(pathString) + Convey("The result should be true", func() { + So(res, ShouldEqual, true) + }) + wrongRes := Exists(wrongPathString) + Convey("The result should be false", func() { + So(wrongRes, ShouldEqual, false) + }) + }) +} + +func TestIsDir(t *testing.T) { + var pathString = "../config" + var fileString = "../config/config.go" + var wrongString = "test" + + Convey("Test path is folder or not", t, func() { + res := IsDir(pathString) + So(res, ShouldEqual, true) + fileRes := IsDir(fileString) + So(fileRes, ShouldEqual, false) + wrongRes := IsDir(wrongString) + So(wrongRes, ShouldEqual, false) + }) +} + +func TestCompress(t *testing.T) { + err := os.Mkdir("testCompress", os.ModePerm) + if err != nil { + t.Error("create testCompress failed") + } + var pathString = "testCompress" + var files []*os.File + var disPath = "testCompress" + file, err := os.Open(pathString) + if err != nil { + t.Error("open source path failed") + } + files = append(files, file) + Convey("Verify dispath is valid path", t, func() { + er := Compress(files, disPath) + Convey("err should be nil", func() { + So(er, ShouldEqual, nil) + }) + }) + _ = os.RemoveAll("testCompress") + +} +func Zip(zipFile string, fileList []string) error { + // 创建 zip 包文件 + fw, err := os.Create(zipFile) + if err != nil { + log.Fatal() + } + defer Close(fw) + + // 实例化新的 zip.Writer + zw := zip.NewWriter(fw) + defer Close(zw) + + for _, fileName := range fileList { + fr, err := os.Open(fileName) + if err != nil { + return err + } + fi, err := fr.Stat() + if err != nil { + return err + } + // 写入文件的头信息 + fh, err := zip.FileInfoHeader(fi) + if err != nil { + return err + } + w, err := zw.CreateHeader(fh) + if err != nil { + return err + } + // 写入文件内容 + _, err = io.Copy(w, fr) + if err != nil { + return err + } + } + return nil +} + +func TestDeCompress(t *testing.T) { + err := os.Mkdir("testDeCompress", os.ModePerm) + if err != nil { + t.Error(err) + + } + err = Zip("demo.zip", []string{}) + if err != nil { + t.Error("create zip file failed") + } + tmpFile, err := os.OpenFile("demo.zip", os.O_RDONLY, 0777) + if err != nil { + debug.PrintStack() + t.Error("open demo.zip failed") + } + var dstPath = "./testDeCompress" + Convey("Test DeCopmress func", t, func() { + + err := DeCompress(tmpFile, dstPath) + So(err, ShouldEqual, nil) + }) + _ = os.RemoveAll("testDeCompress") + _ = os.Remove("demo.zip") + +} diff --git a/core/utils/filter.go b/core/utils/filter.go new file mode 100644 index 000000000..13911bd32 --- /dev/null +++ b/core/utils/filter.go @@ -0,0 +1,49 @@ +package utils + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" +) + +// FilterToQuery Translate entity.Filter to bson.M +func FilterToQuery(f interfaces.Filter) (q bson.M, err error) { + if f == nil || f.IsNil() { + return nil, nil + } + + q = bson.M{} + for _, cond := range f.GetConditions() { + key := cond.GetKey() + op := cond.GetOp() + value := cond.GetValue() + switch op { + case constants.FilterOpNotSet: + // do nothing + case constants.FilterOpEqual: + q[key] = cond.GetValue() + case constants.FilterOpNotEqual: + q[key] = bson.M{"$ne": value} + case constants.FilterOpContains, constants.FilterOpRegex, constants.FilterOpSearch: + q[key] = bson.M{"$regex": value, "$options": "i"} + case constants.FilterOpNotContains: + q[key] = bson.M{"$not": bson.M{"$regex": value}} + case constants.FilterOpIn: + q[key] = bson.M{"$in": value} + case constants.FilterOpNotIn: + q[key] = bson.M{"$nin": value} + case constants.FilterOpGreaterThan: + q[key] = bson.M{"$gt": value} + case constants.FilterOpGreaterThanEqual: + q[key] = bson.M{"$gte": value} + case constants.FilterOpLessThan: + q[key] = bson.M{"$lt": value} + case constants.FilterOpLessThanEqual: + q[key] = bson.M{"$lte": value} + default: + return nil, errors.ErrorFilterInvalidOperation + } + } + return q, nil +} diff --git a/core/utils/git.go b/core/utils/git.go new file mode 100644 index 000000000..1065aad59 --- /dev/null +++ b/core/utils/git.go @@ -0,0 +1,36 @@ +package utils + +import ( + vcs "github.com/crawlab-team/crawlab-vcs" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" +) + +func InitGitClientAuth(g interfaces.Git, gitClient *vcs.GitClient) { + // set auth + switch g.GetAuthType() { + case constants.GitAuthTypeHttp: + gitClient.SetAuthType(vcs.GitAuthTypeHTTP) + gitClient.SetUsername(g.GetUsername()) + gitClient.SetPassword(g.GetPassword()) + case constants.GitAuthTypeSsh: + gitClient.SetAuthType(vcs.GitAuthTypeSSH) + gitClient.SetUsername(g.GetUsername()) + gitClient.SetPrivateKey(g.GetPassword()) + } +} + +func InitGitClientAuthV2(g *models.GitV2, gitClient *vcs.GitClient) { + // set auth + switch g.AuthType { + case constants.GitAuthTypeHttp: + gitClient.SetAuthType(vcs.GitAuthTypeHTTP) + gitClient.SetUsername(g.Username) + gitClient.SetPassword(g.Password) + case constants.GitAuthTypeSsh: + gitClient.SetAuthType(vcs.GitAuthTypeSSH) + gitClient.SetUsername(g.Username) + gitClient.SetPrivateKey(g.Password) + } +} diff --git a/core/utils/helpers.go b/core/utils/helpers.go new file mode 100644 index 000000000..338166f80 --- /dev/null +++ b/core/utils/helpers.go @@ -0,0 +1,36 @@ +package utils + +import ( + "github.com/crawlab-team/go-trace" + "io" + "reflect" + "unsafe" +) + +func BytesToString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} + +func Close(c io.Closer) { + err := c.Close() + if err != nil { + trace.PrintError(err) + } +} + +func Contains(array interface{}, val interface{}) (fla bool) { + fla = false + switch reflect.TypeOf(array).Kind() { + case reflect.Slice: + { + s := reflect.ValueOf(array) + for i := 0; i < s.Len(); i++ { + if reflect.DeepEqual(val, s.Index(i).Interface()) { + fla = true + return + } + } + } + } + return +} diff --git a/core/utils/http.go b/core/utils/http.go new file mode 100644 index 000000000..3b2c0f667 --- /dev/null +++ b/core/utils/http.go @@ -0,0 +1,32 @@ +package utils + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "net/http" +) + +func handleError(statusCode int, c *gin.Context, err error, print bool) { + if print { + trace.PrintError(err) + } + c.AbortWithStatusJSON(statusCode, entity.Response{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageError, + Error: err.Error(), + }) +} + +func HandleError(statusCode int, c *gin.Context, err error) { + handleError(statusCode, c, err, true) +} + +func HandleErrorUnauthorized(c *gin.Context, err error) { + HandleError(http.StatusUnauthorized, c, err) +} + +func HandleErrorInternalServerError(c *gin.Context, err error) { + HandleError(http.StatusInternalServerError, c, err) +} diff --git a/core/utils/init.go b/core/utils/init.go new file mode 100644 index 000000000..21a12bd8c --- /dev/null +++ b/core/utils/init.go @@ -0,0 +1,30 @@ +package utils + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "sync" +) + +var moduleInitializedMap = sync.Map{} + +func InitModule(id interfaces.ModuleId, fn func() error) (err error) { + res, ok := moduleInitializedMap.Load(id) + if ok { + initialized, _ := res.(bool) + if initialized { + return nil + } + } + + if err := fn(); err != nil { + return err + } + + moduleInitializedMap.Store(id, true) + + return nil +} + +func ForceInitModule(fn func() error) (err error) { + return fn() +} diff --git a/core/utils/json.go b/core/utils/json.go new file mode 100644 index 000000000..564a67eb1 --- /dev/null +++ b/core/utils/json.go @@ -0,0 +1,12 @@ +package utils + +import "encoding/json" + +func JsonToBytes(d interface{}) (bytes []byte, err error) { + switch d.(type) { + case []byte: + return d.([]byte), nil + default: + return json.Marshal(d) + } +} diff --git a/core/utils/kafka.go b/core/utils/kafka.go new file mode 100644 index 000000000..1bf005f2a --- /dev/null +++ b/core/utils/kafka.go @@ -0,0 +1,41 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/segmentio/kafka-go" + "time" +) + +func GetKafkaConnection(ds *models.DataSource) (c *kafka.Conn, err error) { + return getKafkaConnection(context.Background(), ds) +} + +func GetKafkaConnectionWithTimeout(ds *models.DataSource, timeout time.Duration) (c *kafka.Conn, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getKafkaConnection(ctx, ds) +} + +func getKafkaConnection(ctx context.Context, ds *models.DataSource) (c *kafka.Conn, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultKafkaPort + } + + // kafka connection address + network := "tcp" + address := fmt.Sprintf("%s:%s", host, port) + topic := ds.Database + partition := 0 // TODO: parameterize + + // kafka connection + return kafka.DialLeader(ctx, network, address, topic, partition) +} diff --git a/core/utils/mongo.go b/core/utils/mongo.go new file mode 100644 index 000000000..4b1ff502c --- /dev/null +++ b/core/utils/mongo.go @@ -0,0 +1,94 @@ +package utils + +import ( + "context" + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "time" +) + +func GetMongoQuery(query generic.ListQuery) (res bson.M) { + res = bson.M{} + for _, c := range query { + switch c.Op { + case generic.OpEqual: + res[c.Key] = c.Value + default: + res[c.Key] = bson.M{ + c.Op: c.Value, + } + } + } + return res +} + +func GetMongoOpts(opts *generic.ListOptions) (res *mongo.FindOptions) { + var sort bson.D + for _, s := range opts.Sort { + direction := 1 + if s.Direction == generic.SortDirectionAsc { + direction = 1 + } else if s.Direction == generic.SortDirectionDesc { + direction = -1 + } + sort = append(sort, bson.E{Key: s.Key, Value: direction}) + } + return &mongo.FindOptions{ + Skip: opts.Skip, + Limit: opts.Limit, + Sort: sort, + } +} + +func GetMongoClient(ds *models.DataSource) (c *mongo2.Client, err error) { + return getMongoClient(context.Background(), ds) +} + +func GetMongoClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c *mongo2.Client, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMongoClient(ctx, ds) +} + +func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Client, err error) { + // normalize settings + if ds.Host == "" { + ds.Host = constants.DefaultHost + } + if ds.Port == "" { + ds.Port = constants.DefaultMongoPort + } + + // options + var opts []mongo.ClientOption + opts = append(opts, mongo.WithContext(ctx)) + opts = append(opts, mongo.WithUri(ds.Url)) + opts = append(opts, mongo.WithHost(ds.Host)) + opts = append(opts, mongo.WithPort(ds.Port)) + opts = append(opts, mongo.WithDb(ds.Database)) + opts = append(opts, mongo.WithUsername(ds.Username)) + opts = append(opts, mongo.WithPassword(ds.Password)) + opts = append(opts, mongo.WithHosts(ds.Hosts)) + + // extra + if ds.Extra != nil { + // auth source + authSource, ok := ds.Extra["auth_source"] + if ok { + opts = append(opts, mongo.WithAuthSource(authSource)) + } + + // auth mechanism + authMechanism, ok := ds.Extra["auth_mechanism"] + if ok { + opts = append(opts, mongo.WithAuthMechanism(authMechanism)) + } + } + + // client + return mongo.GetMongoClient(opts...) +} diff --git a/core/utils/mssql.go b/core/utils/mssql.go new file mode 100644 index 000000000..0fb21353b --- /dev/null +++ b/core/utils/mssql.go @@ -0,0 +1,60 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/mssql" + "time" +) + +func GetMssqlSession(ds *models.DataSource) (s db.Session, err error) { + return getMssqlSession(context.Background(), ds) +} + +func GetMssqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMssqlSession(ctx, ds) +} + +func getMssqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultMssqlPort + } + + // connect settings + settings := mssql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mssql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/mysql.go b/core/utils/mysql.go new file mode 100644 index 000000000..8fc9ae18f --- /dev/null +++ b/core/utils/mysql.go @@ -0,0 +1,60 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/mysql" + "time" +) + +func GetMysqlSession(ds *models.DataSource) (s db.Session, err error) { + return getMysqlSession(context.Background(), ds) +} + +func GetMysqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMysqlSession(ctx, ds) +} + +func getMysqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultMysqlPort + } + + // connect settings + settings := mysql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mysql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/node.go b/core/utils/node.go new file mode 100644 index 000000000..1f20ade16 --- /dev/null +++ b/core/utils/node.go @@ -0,0 +1,13 @@ +package utils + +func IsMaster() bool { + return EnvIsTrue("node.master", false) +} + +func GetNodeType() string { + if IsMaster() { + return "master" + } else { + return "worker" + } +} diff --git a/core/utils/os.go b/core/utils/os.go new file mode 100644 index 000000000..183512527 --- /dev/null +++ b/core/utils/os.go @@ -0,0 +1,13 @@ +package utils + +import ( + "os" + "os/signal" + "syscall" +) + +func DefaultWait() { + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit +} diff --git a/core/utils/postgresql.go b/core/utils/postgresql.go new file mode 100644 index 000000000..40c8a208a --- /dev/null +++ b/core/utils/postgresql.go @@ -0,0 +1,60 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/postgresql" + "time" +) + +func GetPostgresqlSession(ds *models.DataSource) (s db.Session, err error) { + return getPostgresqlSession(context.Background(), ds) +} + +func GetPostgresqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getPostgresqlSession(ctx, ds) +} + +func getPostgresqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultPostgresqlPort + } + + // connect settings + settings := postgresql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = postgresql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/result.go b/core/utils/result.go new file mode 100644 index 000000000..624503090 --- /dev/null +++ b/core/utils/result.go @@ -0,0 +1,23 @@ +package utils + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/interfaces" +) + +func GetResultHash(value interface{}, keys []string) (res string, err error) { + m := make(map[string]interface{}) + for _, k := range keys { + _value, ok := value.(interfaces.Result) + if !ok { + continue + } + v := _value.GetValue(k) + m[k] = v + } + data, err := json.Marshal(m) + if err != nil { + return "", err + } + return EncryptMd5(string(data)), nil +} diff --git a/core/utils/rpc.go b/core/utils/rpc.go new file mode 100644 index 000000000..03414199e --- /dev/null +++ b/core/utils/rpc.go @@ -0,0 +1,14 @@ +package utils + +import "encoding/json" + +// Object 转化为 String +func ObjectToString(params interface{}) string { + bytes, _ := json.Marshal(params) + return BytesToString(bytes) +} + +// 获取 RPC 参数 +func GetRpcParam(key string, params map[string]string) string { + return params[key] +} diff --git a/core/utils/spider.go b/core/utils/spider.go new file mode 100644 index 000000000..4484ccf0a --- /dev/null +++ b/core/utils/spider.go @@ -0,0 +1,8 @@ +package utils + +func GetSpiderCol(col string, name string) string { + if col == "" { + return "results_" + name + } + return col +} diff --git a/core/utils/sql.go b/core/utils/sql.go new file mode 100644 index 000000000..f231d7556 --- /dev/null +++ b/core/utils/sql.go @@ -0,0 +1,27 @@ +package utils + +import ( + "github.com/crawlab-team/crawlab-db/generic" + "github.com/upper/db/v4" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func GetSqlQuery(query generic.ListQuery) (res db.Cond) { + res = db.Cond{} + for _, c := range query { + switch c.Value.(type) { + case primitive.ObjectID: + c.Value = c.Value.(primitive.ObjectID).Hex() + } + switch c.Op { + case generic.OpEqual: + res[c.Key] = c.Value + default: + res[c.Key] = db.Cond{ + c.Op: c.Value, + } + } + } + // TODO: sort + return res +} diff --git a/core/utils/sqlite.go b/core/utils/sqlite.go new file mode 100644 index 000000000..1d6ff682a --- /dev/null +++ b/core/utils/sqlite.go @@ -0,0 +1,45 @@ +package utils + +import ( + "context" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/sqlite" + "time" +) + +func GetSqliteSession(ds *models.DataSource) (s db.Session, err error) { + return getSqliteSession(context.Background(), ds) +} + +func GetSqliteSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getSqliteSession(ctx, ds) +} + +func getSqliteSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // connect settings + settings := sqlite.ConnectionURL{ + Database: ds.Database, + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = sqlite.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/stats.go b/core/utils/stats.go new file mode 100644 index 000000000..d4b585bf7 --- /dev/null +++ b/core/utils/stats.go @@ -0,0 +1 @@ +package utils diff --git a/core/utils/system.go b/core/utils/system.go new file mode 100644 index 000000000..9b161f112 --- /dev/null +++ b/core/utils/system.go @@ -0,0 +1,7 @@ +package utils + +import "github.com/spf13/viper" + +func IsPro() bool { + return viper.GetString("info.edition") == "global.edition.pro" +} diff --git a/core/utils/task.go b/core/utils/task.go new file mode 100644 index 000000000..94bfbb3b2 --- /dev/null +++ b/core/utils/task.go @@ -0,0 +1,13 @@ +package utils + +import "github.com/crawlab-team/crawlab/core/constants" + +func IsCancellable(status string) bool { + switch status { + case constants.TaskStatusPending, + constants.TaskStatusRunning: + return true + default: + return false + } +} diff --git a/core/utils/time.go b/core/utils/time.go new file mode 100644 index 000000000..689e2c284 --- /dev/null +++ b/core/utils/time.go @@ -0,0 +1,18 @@ +package utils + +import ( + "time" +) + +func GetLocalTime(t time.Time) time.Time { + return t.In(time.Local) +} + +func GetTimeString(t time.Time) string { + return t.Format("2006-01-02 15:04:05") +} + +func GetLocalTimeString(t time.Time) string { + t = GetLocalTime(t) + return GetTimeString(t) +} diff --git a/core/utils/uuid.go b/core/utils/uuid.go new file mode 100644 index 000000000..72c5191cc --- /dev/null +++ b/core/utils/uuid.go @@ -0,0 +1,8 @@ +package utils + +import "github.com/google/uuid" + +func NewUUIDString() (res string) { + id, _ := uuid.NewUUID() + return id.String() +} diff --git a/grpc/.gitignore b/grpc/.gitignore new file mode 100644 index 000000000..f32e31af4 --- /dev/null +++ b/grpc/.gitignore @@ -0,0 +1,2 @@ +.idea/ +.DS_Store diff --git a/grpc/LICENSE b/grpc/LICENSE new file mode 100644 index 000000000..3c0946b9b --- /dev/null +++ b/grpc/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2021, Crawlab Team +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/grpc/README.md b/grpc/README.md new file mode 100644 index 000000000..207b4216b --- /dev/null +++ b/grpc/README.md @@ -0,0 +1,2 @@ +# crawlab-grpc +gRPC for Crawlab diff --git a/grpc/bin/compile.sh b/grpc/bin/compile.sh new file mode 100755 index 000000000..b0d5c65fe --- /dev/null +++ b/grpc/bin/compile.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +if [ -L $0 ] +then + BASE_DIR=`dirname $(readlink $0)` +else + BASE_DIR=`dirname $0` +fi +base_path=$(cd $BASE_DIR/..; pwd) + +cd $base_path && \ + rm -rf dist | true + +cd $base_path && \ + mkdir -p dist/python | true && \ + mkdir -p dist/js | true && \ + mkdir -p dist/ts | true && \ + mkdir -p dist/java | true && \ + mkdir -p dist/csharp | true && \ + mkdir -p dist/php | true && \ + mkdir -p dist/ruby | true + +cd $base_path && \ + protoc -I ./proto \ + --go_out=. \ + --go-grpc_out=. \ + --python_out=dist/python \ + --js_out=dist/js \ + --java_out=dist/java \ + --csharp_out=dist/csharp \ + ./proto/**/*.proto + +# python +cd $base_path && \ + python3 -m grpc_tools.protoc -I ./proto \ + --grpc_python_out=dist/python \ + ./proto/**/*.proto diff --git a/grpc/dependencies_service_v2_request.pb.go b/grpc/dependencies_service_v2_request.pb.go new file mode 100644 index 000000000..cae03422f --- /dev/null +++ b/grpc/dependencies_service_v2_request.pb.go @@ -0,0 +1,488 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/dependencies_service_v2_request.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type Dependency struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` +} + +func (x *Dependency) Reset() { + *x = Dependency{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Dependency) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Dependency) ProtoMessage() {} + +func (x *Dependency) ProtoReflect() protoreflect.Message { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Dependency.ProtoReflect.Descriptor instead. +func (*Dependency) Descriptor() ([]byte, []int) { + return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{0} +} + +func (x *Dependency) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Dependency) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +type DependenciesServiceV2ConnectRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` +} + +func (x *DependenciesServiceV2ConnectRequest) Reset() { + *x = DependenciesServiceV2ConnectRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DependenciesServiceV2ConnectRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DependenciesServiceV2ConnectRequest) ProtoMessage() {} + +func (x *DependenciesServiceV2ConnectRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DependenciesServiceV2ConnectRequest.ProtoReflect.Descriptor instead. +func (*DependenciesServiceV2ConnectRequest) Descriptor() ([]byte, []int) { + return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{1} +} + +func (x *DependenciesServiceV2ConnectRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +type DependenciesServiceV2SyncRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Lang string `protobuf:"bytes,2,opt,name=lang,proto3" json:"lang,omitempty"` + Dependencies []*Dependency `protobuf:"bytes,3,rep,name=dependencies,proto3" json:"dependencies,omitempty"` +} + +func (x *DependenciesServiceV2SyncRequest) Reset() { + *x = DependenciesServiceV2SyncRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DependenciesServiceV2SyncRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DependenciesServiceV2SyncRequest) ProtoMessage() {} + +func (x *DependenciesServiceV2SyncRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DependenciesServiceV2SyncRequest.ProtoReflect.Descriptor instead. +func (*DependenciesServiceV2SyncRequest) Descriptor() ([]byte, []int) { + return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{2} +} + +func (x *DependenciesServiceV2SyncRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *DependenciesServiceV2SyncRequest) GetLang() string { + if x != nil { + return x.Lang + } + return "" +} + +func (x *DependenciesServiceV2SyncRequest) GetDependencies() []*Dependency { + if x != nil { + return x.Dependencies + } + return nil +} + +type DependenciesServiceV2InstallRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Lang string `protobuf:"bytes,2,opt,name=lang,proto3" json:"lang,omitempty"` + Dependencies []*Dependency `protobuf:"bytes,3,rep,name=dependencies,proto3" json:"dependencies,omitempty"` + Proxy string `protobuf:"bytes,4,opt,name=proxy,proto3" json:"proxy,omitempty"` +} + +func (x *DependenciesServiceV2InstallRequest) Reset() { + *x = DependenciesServiceV2InstallRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DependenciesServiceV2InstallRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DependenciesServiceV2InstallRequest) ProtoMessage() {} + +func (x *DependenciesServiceV2InstallRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DependenciesServiceV2InstallRequest.ProtoReflect.Descriptor instead. +func (*DependenciesServiceV2InstallRequest) Descriptor() ([]byte, []int) { + return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{3} +} + +func (x *DependenciesServiceV2InstallRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *DependenciesServiceV2InstallRequest) GetLang() string { + if x != nil { + return x.Lang + } + return "" +} + +func (x *DependenciesServiceV2InstallRequest) GetDependencies() []*Dependency { + if x != nil { + return x.Dependencies + } + return nil +} + +func (x *DependenciesServiceV2InstallRequest) GetProxy() string { + if x != nil { + return x.Proxy + } + return "" +} + +type DependenciesServiceV2UninstallRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Lang string `protobuf:"bytes,2,opt,name=lang,proto3" json:"lang,omitempty"` + Dependencies []*Dependency `protobuf:"bytes,3,rep,name=dependencies,proto3" json:"dependencies,omitempty"` +} + +func (x *DependenciesServiceV2UninstallRequest) Reset() { + *x = DependenciesServiceV2UninstallRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DependenciesServiceV2UninstallRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DependenciesServiceV2UninstallRequest) ProtoMessage() {} + +func (x *DependenciesServiceV2UninstallRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DependenciesServiceV2UninstallRequest.ProtoReflect.Descriptor instead. +func (*DependenciesServiceV2UninstallRequest) Descriptor() ([]byte, []int) { + return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{4} +} + +func (x *DependenciesServiceV2UninstallRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *DependenciesServiceV2UninstallRequest) GetLang() string { + if x != nil { + return x.Lang + } + return "" +} + +func (x *DependenciesServiceV2UninstallRequest) GetDependencies() []*Dependency { + if x != nil { + return x.Dependencies + } + return nil +} + +var File_entity_dependencies_service_v2_request_proto protoreflect.FileDescriptor + +var file_entity_dependencies_service_v2_request_proto_rawDesc = []byte{ + 0x0a, 0x2c, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, + 0x6e, 0x63, 0x69, 0x65, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, + 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, + 0x67, 0x72, 0x70, 0x63, 0x22, 0x3a, 0x0a, 0x0a, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, + 0x63, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x22, 0x40, 0x0a, 0x23, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, + 0x65, 0x79, 0x22, 0x87, 0x01, 0x0a, 0x20, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, + 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x53, 0x79, 0x6e, 0x63, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, + 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x12, 0x34, 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, + 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0c, + 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x22, 0xa0, 0x01, 0x0a, + 0x23, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, + 0x12, 0x0a, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, + 0x61, 0x6e, 0x67, 0x12, 0x34, 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, + 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0c, 0x64, 0x65, 0x70, + 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x72, 0x6f, + 0x78, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x22, + 0x8c, 0x01, 0x0a, 0x25, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, + 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, + 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, + 0x65, 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x12, 0x34, 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, + 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, + 0x52, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x42, 0x08, + 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_dependencies_service_v2_request_proto_rawDescOnce sync.Once + file_entity_dependencies_service_v2_request_proto_rawDescData = file_entity_dependencies_service_v2_request_proto_rawDesc +) + +func file_entity_dependencies_service_v2_request_proto_rawDescGZIP() []byte { + file_entity_dependencies_service_v2_request_proto_rawDescOnce.Do(func() { + file_entity_dependencies_service_v2_request_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_dependencies_service_v2_request_proto_rawDescData) + }) + return file_entity_dependencies_service_v2_request_proto_rawDescData +} + +var file_entity_dependencies_service_v2_request_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_entity_dependencies_service_v2_request_proto_goTypes = []interface{}{ + (*Dependency)(nil), // 0: grpc.Dependency + (*DependenciesServiceV2ConnectRequest)(nil), // 1: grpc.DependenciesServiceV2ConnectRequest + (*DependenciesServiceV2SyncRequest)(nil), // 2: grpc.DependenciesServiceV2SyncRequest + (*DependenciesServiceV2InstallRequest)(nil), // 3: grpc.DependenciesServiceV2InstallRequest + (*DependenciesServiceV2UninstallRequest)(nil), // 4: grpc.DependenciesServiceV2UninstallRequest +} +var file_entity_dependencies_service_v2_request_proto_depIdxs = []int32{ + 0, // 0: grpc.DependenciesServiceV2SyncRequest.dependencies:type_name -> grpc.Dependency + 0, // 1: grpc.DependenciesServiceV2InstallRequest.dependencies:type_name -> grpc.Dependency + 0, // 2: grpc.DependenciesServiceV2UninstallRequest.dependencies:type_name -> grpc.Dependency + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_entity_dependencies_service_v2_request_proto_init() } +func file_entity_dependencies_service_v2_request_proto_init() { + if File_entity_dependencies_service_v2_request_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_entity_dependencies_service_v2_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Dependency); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_dependencies_service_v2_request_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DependenciesServiceV2ConnectRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_dependencies_service_v2_request_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DependenciesServiceV2SyncRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_dependencies_service_v2_request_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DependenciesServiceV2InstallRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_dependencies_service_v2_request_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DependenciesServiceV2UninstallRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_dependencies_service_v2_request_proto_rawDesc, + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_dependencies_service_v2_request_proto_goTypes, + DependencyIndexes: file_entity_dependencies_service_v2_request_proto_depIdxs, + MessageInfos: file_entity_dependencies_service_v2_request_proto_msgTypes, + }.Build() + File_entity_dependencies_service_v2_request_proto = out.File + file_entity_dependencies_service_v2_request_proto_rawDesc = nil + file_entity_dependencies_service_v2_request_proto_goTypes = nil + file_entity_dependencies_service_v2_request_proto_depIdxs = nil +} diff --git a/grpc/dependency_service_v2.pb.go b/grpc/dependency_service_v2.pb.go new file mode 100644 index 000000000..7057a9cba --- /dev/null +++ b/grpc/dependency_service_v2.pb.go @@ -0,0 +1,108 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/dependency_service_v2.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_dependency_service_v2_proto protoreflect.FileDescriptor + +var file_services_dependency_service_v2_proto_rawDesc = []byte{ + 0x0a, 0x24, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x64, 0x65, 0x70, 0x65, 0x6e, + 0x64, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x2c, 0x65, 0x6e, + 0x74, 0x69, 0x74, 0x79, 0x2f, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, + 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, 0x5f, 0x72, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, + 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x32, 0xc9, 0x02, 0x0a, 0x13, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x12, 0x48, 0x0a, 0x07, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x12, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, + 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, + 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x28, 0x01, 0x12, 0x40, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x26, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4a, 0x0a, 0x07, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, + 0x12, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, + 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x49, 0x6e, 0x73, + 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, + 0x01, 0x12, 0x5a, 0x0a, 0x15, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x44, 0x65, + 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x12, 0x2b, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x08, 0x5a, + 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_services_dependency_service_v2_proto_goTypes = []interface{}{ + (*DependenciesServiceV2ConnectRequest)(nil), // 0: grpc.DependenciesServiceV2ConnectRequest + (*DependenciesServiceV2SyncRequest)(nil), // 1: grpc.DependenciesServiceV2SyncRequest + (*DependenciesServiceV2InstallRequest)(nil), // 2: grpc.DependenciesServiceV2InstallRequest + (*DependenciesServiceV2UninstallRequest)(nil), // 3: grpc.DependenciesServiceV2UninstallRequest + (*Response)(nil), // 4: grpc.Response +} +var file_services_dependency_service_v2_proto_depIdxs = []int32{ + 0, // 0: grpc.DependencyServiceV2.Connect:input_type -> grpc.DependenciesServiceV2ConnectRequest + 1, // 1: grpc.DependencyServiceV2.Sync:input_type -> grpc.DependenciesServiceV2SyncRequest + 2, // 2: grpc.DependencyServiceV2.Install:input_type -> grpc.DependenciesServiceV2InstallRequest + 3, // 3: grpc.DependencyServiceV2.UninstallDependencies:input_type -> grpc.DependenciesServiceV2UninstallRequest + 4, // 4: grpc.DependencyServiceV2.Connect:output_type -> grpc.Response + 4, // 5: grpc.DependencyServiceV2.Sync:output_type -> grpc.Response + 4, // 6: grpc.DependencyServiceV2.Install:output_type -> grpc.Response + 4, // 7: grpc.DependencyServiceV2.UninstallDependencies:output_type -> grpc.Response + 4, // [4:8] is the sub-list for method output_type + 0, // [0:4] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_dependency_service_v2_proto_init() } +func file_services_dependency_service_v2_proto_init() { + if File_services_dependency_service_v2_proto != nil { + return + } + file_entity_dependencies_service_v2_request_proto_init() + file_entity_response_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_dependency_service_v2_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_dependency_service_v2_proto_goTypes, + DependencyIndexes: file_services_dependency_service_v2_proto_depIdxs, + }.Build() + File_services_dependency_service_v2_proto = out.File + file_services_dependency_service_v2_proto_rawDesc = nil + file_services_dependency_service_v2_proto_goTypes = nil + file_services_dependency_service_v2_proto_depIdxs = nil +} diff --git a/grpc/dependency_service_v2_grpc.pb.go b/grpc/dependency_service_v2_grpc.pb.go new file mode 100644 index 000000000..3d0a9eb96 --- /dev/null +++ b/grpc/dependency_service_v2_grpc.pb.go @@ -0,0 +1,312 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/dependency_service_v2.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// DependencyServiceV2Client is the client API for DependencyServiceV2 service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type DependencyServiceV2Client interface { + Connect(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_ConnectClient, error) + Sync(ctx context.Context, in *DependenciesServiceV2SyncRequest, opts ...grpc.CallOption) (*Response, error) + Install(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_InstallClient, error) + UninstallDependencies(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_UninstallDependenciesClient, error) +} + +type dependencyServiceV2Client struct { + cc grpc.ClientConnInterface +} + +func NewDependencyServiceV2Client(cc grpc.ClientConnInterface) DependencyServiceV2Client { + return &dependencyServiceV2Client{cc} +} + +func (c *dependencyServiceV2Client) Connect(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_ConnectClient, error) { + stream, err := c.cc.NewStream(ctx, &DependencyServiceV2_ServiceDesc.Streams[0], "/grpc.DependencyServiceV2/Connect", opts...) + if err != nil { + return nil, err + } + x := &dependencyServiceV2ConnectClient{stream} + return x, nil +} + +type DependencyServiceV2_ConnectClient interface { + Send(*DependenciesServiceV2ConnectRequest) error + CloseAndRecv() (*Response, error) + grpc.ClientStream +} + +type dependencyServiceV2ConnectClient struct { + grpc.ClientStream +} + +func (x *dependencyServiceV2ConnectClient) Send(m *DependenciesServiceV2ConnectRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *dependencyServiceV2ConnectClient) CloseAndRecv() (*Response, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(Response) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *dependencyServiceV2Client) Sync(ctx context.Context, in *DependenciesServiceV2SyncRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.DependencyServiceV2/Sync", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dependencyServiceV2Client) Install(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_InstallClient, error) { + stream, err := c.cc.NewStream(ctx, &DependencyServiceV2_ServiceDesc.Streams[1], "/grpc.DependencyServiceV2/Install", opts...) + if err != nil { + return nil, err + } + x := &dependencyServiceV2InstallClient{stream} + return x, nil +} + +type DependencyServiceV2_InstallClient interface { + Send(*DependenciesServiceV2InstallRequest) error + Recv() (*Response, error) + grpc.ClientStream +} + +type dependencyServiceV2InstallClient struct { + grpc.ClientStream +} + +func (x *dependencyServiceV2InstallClient) Send(m *DependenciesServiceV2InstallRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *dependencyServiceV2InstallClient) Recv() (*Response, error) { + m := new(Response) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *dependencyServiceV2Client) UninstallDependencies(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_UninstallDependenciesClient, error) { + stream, err := c.cc.NewStream(ctx, &DependencyServiceV2_ServiceDesc.Streams[2], "/grpc.DependencyServiceV2/UninstallDependencies", opts...) + if err != nil { + return nil, err + } + x := &dependencyServiceV2UninstallDependenciesClient{stream} + return x, nil +} + +type DependencyServiceV2_UninstallDependenciesClient interface { + Send(*DependenciesServiceV2UninstallRequest) error + Recv() (*Response, error) + grpc.ClientStream +} + +type dependencyServiceV2UninstallDependenciesClient struct { + grpc.ClientStream +} + +func (x *dependencyServiceV2UninstallDependenciesClient) Send(m *DependenciesServiceV2UninstallRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *dependencyServiceV2UninstallDependenciesClient) Recv() (*Response, error) { + m := new(Response) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// DependencyServiceV2Server is the server API for DependencyServiceV2 service. +// All implementations must embed UnimplementedDependencyServiceV2Server +// for forward compatibility +type DependencyServiceV2Server interface { + Connect(DependencyServiceV2_ConnectServer) error + Sync(context.Context, *DependenciesServiceV2SyncRequest) (*Response, error) + Install(DependencyServiceV2_InstallServer) error + UninstallDependencies(DependencyServiceV2_UninstallDependenciesServer) error + mustEmbedUnimplementedDependencyServiceV2Server() +} + +// UnimplementedDependencyServiceV2Server must be embedded to have forward compatible implementations. +type UnimplementedDependencyServiceV2Server struct { +} + +func (UnimplementedDependencyServiceV2Server) Connect(DependencyServiceV2_ConnectServer) error { + return status.Errorf(codes.Unimplemented, "method Connect not implemented") +} +func (UnimplementedDependencyServiceV2Server) Sync(context.Context, *DependenciesServiceV2SyncRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Sync not implemented") +} +func (UnimplementedDependencyServiceV2Server) Install(DependencyServiceV2_InstallServer) error { + return status.Errorf(codes.Unimplemented, "method Install not implemented") +} +func (UnimplementedDependencyServiceV2Server) UninstallDependencies(DependencyServiceV2_UninstallDependenciesServer) error { + return status.Errorf(codes.Unimplemented, "method UninstallDependencies not implemented") +} +func (UnimplementedDependencyServiceV2Server) mustEmbedUnimplementedDependencyServiceV2Server() {} + +// UnsafeDependencyServiceV2Server may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to DependencyServiceV2Server will +// result in compilation errors. +type UnsafeDependencyServiceV2Server interface { + mustEmbedUnimplementedDependencyServiceV2Server() +} + +func RegisterDependencyServiceV2Server(s grpc.ServiceRegistrar, srv DependencyServiceV2Server) { + s.RegisterService(&DependencyServiceV2_ServiceDesc, srv) +} + +func _DependencyServiceV2_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(DependencyServiceV2Server).Connect(&dependencyServiceV2ConnectServer{stream}) +} + +type DependencyServiceV2_ConnectServer interface { + SendAndClose(*Response) error + Recv() (*DependenciesServiceV2ConnectRequest, error) + grpc.ServerStream +} + +type dependencyServiceV2ConnectServer struct { + grpc.ServerStream +} + +func (x *dependencyServiceV2ConnectServer) SendAndClose(m *Response) error { + return x.ServerStream.SendMsg(m) +} + +func (x *dependencyServiceV2ConnectServer) Recv() (*DependenciesServiceV2ConnectRequest, error) { + m := new(DependenciesServiceV2ConnectRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _DependencyServiceV2_Sync_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DependenciesServiceV2SyncRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DependencyServiceV2Server).Sync(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.DependencyServiceV2/Sync", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DependencyServiceV2Server).Sync(ctx, req.(*DependenciesServiceV2SyncRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _DependencyServiceV2_Install_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(DependencyServiceV2Server).Install(&dependencyServiceV2InstallServer{stream}) +} + +type DependencyServiceV2_InstallServer interface { + Send(*Response) error + Recv() (*DependenciesServiceV2InstallRequest, error) + grpc.ServerStream +} + +type dependencyServiceV2InstallServer struct { + grpc.ServerStream +} + +func (x *dependencyServiceV2InstallServer) Send(m *Response) error { + return x.ServerStream.SendMsg(m) +} + +func (x *dependencyServiceV2InstallServer) Recv() (*DependenciesServiceV2InstallRequest, error) { + m := new(DependenciesServiceV2InstallRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _DependencyServiceV2_UninstallDependencies_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(DependencyServiceV2Server).UninstallDependencies(&dependencyServiceV2UninstallDependenciesServer{stream}) +} + +type DependencyServiceV2_UninstallDependenciesServer interface { + Send(*Response) error + Recv() (*DependenciesServiceV2UninstallRequest, error) + grpc.ServerStream +} + +type dependencyServiceV2UninstallDependenciesServer struct { + grpc.ServerStream +} + +func (x *dependencyServiceV2UninstallDependenciesServer) Send(m *Response) error { + return x.ServerStream.SendMsg(m) +} + +func (x *dependencyServiceV2UninstallDependenciesServer) Recv() (*DependenciesServiceV2UninstallRequest, error) { + m := new(DependenciesServiceV2UninstallRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// DependencyServiceV2_ServiceDesc is the grpc.ServiceDesc for DependencyServiceV2 service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var DependencyServiceV2_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.DependencyServiceV2", + HandlerType: (*DependencyServiceV2Server)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Sync", + Handler: _DependencyServiceV2_Sync_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Connect", + Handler: _DependencyServiceV2_Connect_Handler, + ClientStreams: true, + }, + { + StreamName: "Install", + Handler: _DependencyServiceV2_Install_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "UninstallDependencies", + Handler: _DependencyServiceV2_UninstallDependencies_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "services/dependency_service_v2.proto", +} diff --git a/grpc/go.mod b/grpc/go.mod new file mode 100644 index 000000000..bbc0968c8 --- /dev/null +++ b/grpc/go.mod @@ -0,0 +1,16 @@ +module github.com/crawlab-team/crawlab/grpc + +go 1.22 + +require ( + github.com/golang/protobuf v1.5.4 + google.golang.org/grpc v1.64.0 + google.golang.org/protobuf v1.34.1 +) + +require ( + golang.org/x/net v0.22.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect +) diff --git a/grpc/go.sum b/grpc/go.sum new file mode 100644 index 000000000..d41d2dc22 --- /dev/null +++ b/grpc/go.sum @@ -0,0 +1,16 @@ +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= diff --git a/grpc/message_service.pb.go b/grpc/message_service.pb.go new file mode 100644 index 000000000..fca04128e --- /dev/null +++ b/grpc/message_service.pb.go @@ -0,0 +1,79 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/message_service.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_message_service_proto protoreflect.FileDescriptor + +var file_services_message_service_proto_rawDesc = []byte{ + 0x0a, 0x1e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x32, 0x4b, 0x0a, 0x0e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x39, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, + 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +} + +var file_services_message_service_proto_goTypes = []interface{}{ + (*StreamMessage)(nil), // 0: grpc.StreamMessage +} +var file_services_message_service_proto_depIdxs = []int32{ + 0, // 0: grpc.MessageService.Connect:input_type -> grpc.StreamMessage + 0, // 1: grpc.MessageService.Connect:output_type -> grpc.StreamMessage + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_message_service_proto_init() } +func file_services_message_service_proto_init() { + if File_services_message_service_proto != nil { + return + } + file_entity_stream_message_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_message_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_message_service_proto_goTypes, + DependencyIndexes: file_services_message_service_proto_depIdxs, + }.Build() + File_services_message_service_proto = out.File + file_services_message_service_proto_rawDesc = nil + file_services_message_service_proto_goTypes = nil + file_services_message_service_proto_depIdxs = nil +} diff --git a/grpc/message_service_grpc.pb.go b/grpc/message_service_grpc.pb.go new file mode 100644 index 000000000..c1cc17b0a --- /dev/null +++ b/grpc/message_service_grpc.pb.go @@ -0,0 +1,137 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/message_service.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// MessageServiceClient is the client API for MessageService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type MessageServiceClient interface { + Connect(ctx context.Context, opts ...grpc.CallOption) (MessageService_ConnectClient, error) +} + +type messageServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewMessageServiceClient(cc grpc.ClientConnInterface) MessageServiceClient { + return &messageServiceClient{cc} +} + +func (c *messageServiceClient) Connect(ctx context.Context, opts ...grpc.CallOption) (MessageService_ConnectClient, error) { + stream, err := c.cc.NewStream(ctx, &MessageService_ServiceDesc.Streams[0], "/grpc.MessageService/Connect", opts...) + if err != nil { + return nil, err + } + x := &messageServiceConnectClient{stream} + return x, nil +} + +type MessageService_ConnectClient interface { + Send(*StreamMessage) error + Recv() (*StreamMessage, error) + grpc.ClientStream +} + +type messageServiceConnectClient struct { + grpc.ClientStream +} + +func (x *messageServiceConnectClient) Send(m *StreamMessage) error { + return x.ClientStream.SendMsg(m) +} + +func (x *messageServiceConnectClient) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// MessageServiceServer is the server API for MessageService service. +// All implementations must embed UnimplementedMessageServiceServer +// for forward compatibility +type MessageServiceServer interface { + Connect(MessageService_ConnectServer) error + mustEmbedUnimplementedMessageServiceServer() +} + +// UnimplementedMessageServiceServer must be embedded to have forward compatible implementations. +type UnimplementedMessageServiceServer struct { +} + +func (UnimplementedMessageServiceServer) Connect(MessageService_ConnectServer) error { + return status.Errorf(codes.Unimplemented, "method Connect not implemented") +} +func (UnimplementedMessageServiceServer) mustEmbedUnimplementedMessageServiceServer() {} + +// UnsafeMessageServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to MessageServiceServer will +// result in compilation errors. +type UnsafeMessageServiceServer interface { + mustEmbedUnimplementedMessageServiceServer() +} + +func RegisterMessageServiceServer(s grpc.ServiceRegistrar, srv MessageServiceServer) { + s.RegisterService(&MessageService_ServiceDesc, srv) +} + +func _MessageService_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(MessageServiceServer).Connect(&messageServiceConnectServer{stream}) +} + +type MessageService_ConnectServer interface { + Send(*StreamMessage) error + Recv() (*StreamMessage, error) + grpc.ServerStream +} + +type messageServiceConnectServer struct { + grpc.ServerStream +} + +func (x *messageServiceConnectServer) Send(m *StreamMessage) error { + return x.ServerStream.SendMsg(m) +} + +func (x *messageServiceConnectServer) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// MessageService_ServiceDesc is the grpc.ServiceDesc for MessageService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var MessageService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.MessageService", + HandlerType: (*MessageServiceServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "Connect", + Handler: _MessageService_Connect_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "services/message_service.proto", +} diff --git a/grpc/model_base_service.pb.go b/grpc/model_base_service.pb.go new file mode 100644 index 000000000..0f9af6959 --- /dev/null +++ b/grpc/model_base_service.pb.go @@ -0,0 +1,134 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/model_base_service.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_model_base_service_proto protoreflect.FileDescriptor + +var file_services_model_base_service_proto_rawDesc = []byte{ + 0x0a, 0x21, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x5f, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x14, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x2f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, + 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xac, 0x04, 0x0a, 0x10, 0x4d, 0x6f, 0x64, 0x65, 0x6c, + 0x42, 0x61, 0x73, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x2a, 0x0a, 0x07, 0x47, + 0x65, 0x74, 0x42, 0x79, 0x49, 0x64, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x26, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x0d, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x2a, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x0a, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x79, 0x49, 0x64, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x29, 0x0a, 0x06, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x0a, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4c, + 0x69, 0x73, 0x74, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x32, 0x0a, 0x0f, 0x46, 0x6f, 0x72, 0x63, 0x65, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x0a, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x42, 0x79, 0x49, 0x64, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x29, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x2c, 0x0a, 0x09, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x63, 0x12, + 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x29, 0x0a, 0x06, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x28, 0x0a, 0x05, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_services_model_base_service_proto_goTypes = []interface{}{ + (*Request)(nil), // 0: grpc.Request + (*Response)(nil), // 1: grpc.Response +} +var file_services_model_base_service_proto_depIdxs = []int32{ + 0, // 0: grpc.ModelBaseService.GetById:input_type -> grpc.Request + 0, // 1: grpc.ModelBaseService.Get:input_type -> grpc.Request + 0, // 2: grpc.ModelBaseService.GetList:input_type -> grpc.Request + 0, // 3: grpc.ModelBaseService.DeleteById:input_type -> grpc.Request + 0, // 4: grpc.ModelBaseService.Delete:input_type -> grpc.Request + 0, // 5: grpc.ModelBaseService.DeleteList:input_type -> grpc.Request + 0, // 6: grpc.ModelBaseService.ForceDeleteList:input_type -> grpc.Request + 0, // 7: grpc.ModelBaseService.UpdateById:input_type -> grpc.Request + 0, // 8: grpc.ModelBaseService.Update:input_type -> grpc.Request + 0, // 9: grpc.ModelBaseService.UpdateDoc:input_type -> grpc.Request + 0, // 10: grpc.ModelBaseService.Insert:input_type -> grpc.Request + 0, // 11: grpc.ModelBaseService.Count:input_type -> grpc.Request + 1, // 12: grpc.ModelBaseService.GetById:output_type -> grpc.Response + 1, // 13: grpc.ModelBaseService.Get:output_type -> grpc.Response + 1, // 14: grpc.ModelBaseService.GetList:output_type -> grpc.Response + 1, // 15: grpc.ModelBaseService.DeleteById:output_type -> grpc.Response + 1, // 16: grpc.ModelBaseService.Delete:output_type -> grpc.Response + 1, // 17: grpc.ModelBaseService.DeleteList:output_type -> grpc.Response + 1, // 18: grpc.ModelBaseService.ForceDeleteList:output_type -> grpc.Response + 1, // 19: grpc.ModelBaseService.UpdateById:output_type -> grpc.Response + 1, // 20: grpc.ModelBaseService.Update:output_type -> grpc.Response + 1, // 21: grpc.ModelBaseService.UpdateDoc:output_type -> grpc.Response + 1, // 22: grpc.ModelBaseService.Insert:output_type -> grpc.Response + 1, // 23: grpc.ModelBaseService.Count:output_type -> grpc.Response + 12, // [12:24] is the sub-list for method output_type + 0, // [0:12] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_model_base_service_proto_init() } +func file_services_model_base_service_proto_init() { + if File_services_model_base_service_proto != nil { + return + } + file_entity_request_proto_init() + file_entity_response_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_model_base_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_model_base_service_proto_goTypes, + DependencyIndexes: file_services_model_base_service_proto_depIdxs, + }.Build() + File_services_model_base_service_proto = out.File + file_services_model_base_service_proto_rawDesc = nil + file_services_model_base_service_proto_goTypes = nil + file_services_model_base_service_proto_depIdxs = nil +} diff --git a/grpc/model_base_service_grpc.pb.go b/grpc/model_base_service_grpc.pb.go new file mode 100644 index 000000000..465355ec7 --- /dev/null +++ b/grpc/model_base_service_grpc.pb.go @@ -0,0 +1,501 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/model_base_service.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// ModelBaseServiceClient is the client API for ModelBaseService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ModelBaseServiceClient interface { + GetById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Get(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + GetList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + DeleteById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Delete(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + DeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + ForceDeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + UpdateById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Update(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + UpdateDoc(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Insert(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Count(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) +} + +type modelBaseServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewModelBaseServiceClient(cc grpc.ClientConnInterface) ModelBaseServiceClient { + return &modelBaseServiceClient{cc} +} + +func (c *modelBaseServiceClient) GetById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/GetById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) Get(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Get", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) GetList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/GetList", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) DeleteById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/DeleteById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) Delete(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Delete", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) DeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/DeleteList", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) ForceDeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/ForceDeleteList", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) UpdateById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/UpdateById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) Update(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Update", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) UpdateDoc(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/UpdateDoc", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) Insert(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Insert", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) Count(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Count", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ModelBaseServiceServer is the server API for ModelBaseService service. +// All implementations must embed UnimplementedModelBaseServiceServer +// for forward compatibility +type ModelBaseServiceServer interface { + GetById(context.Context, *Request) (*Response, error) + Get(context.Context, *Request) (*Response, error) + GetList(context.Context, *Request) (*Response, error) + DeleteById(context.Context, *Request) (*Response, error) + Delete(context.Context, *Request) (*Response, error) + DeleteList(context.Context, *Request) (*Response, error) + ForceDeleteList(context.Context, *Request) (*Response, error) + UpdateById(context.Context, *Request) (*Response, error) + Update(context.Context, *Request) (*Response, error) + UpdateDoc(context.Context, *Request) (*Response, error) + Insert(context.Context, *Request) (*Response, error) + Count(context.Context, *Request) (*Response, error) + mustEmbedUnimplementedModelBaseServiceServer() +} + +// UnimplementedModelBaseServiceServer must be embedded to have forward compatible implementations. +type UnimplementedModelBaseServiceServer struct { +} + +func (UnimplementedModelBaseServiceServer) GetById(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetById not implemented") +} +func (UnimplementedModelBaseServiceServer) Get(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Get not implemented") +} +func (UnimplementedModelBaseServiceServer) GetList(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetList not implemented") +} +func (UnimplementedModelBaseServiceServer) DeleteById(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteById not implemented") +} +func (UnimplementedModelBaseServiceServer) Delete(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Delete not implemented") +} +func (UnimplementedModelBaseServiceServer) DeleteList(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteList not implemented") +} +func (UnimplementedModelBaseServiceServer) ForceDeleteList(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ForceDeleteList not implemented") +} +func (UnimplementedModelBaseServiceServer) UpdateById(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateById not implemented") +} +func (UnimplementedModelBaseServiceServer) Update(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Update not implemented") +} +func (UnimplementedModelBaseServiceServer) UpdateDoc(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateDoc not implemented") +} +func (UnimplementedModelBaseServiceServer) Insert(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Insert not implemented") +} +func (UnimplementedModelBaseServiceServer) Count(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Count not implemented") +} +func (UnimplementedModelBaseServiceServer) mustEmbedUnimplementedModelBaseServiceServer() {} + +// UnsafeModelBaseServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ModelBaseServiceServer will +// result in compilation errors. +type UnsafeModelBaseServiceServer interface { + mustEmbedUnimplementedModelBaseServiceServer() +} + +func RegisterModelBaseServiceServer(s grpc.ServiceRegistrar, srv ModelBaseServiceServer) { + s.RegisterService(&ModelBaseService_ServiceDesc, srv) +} + +func _ModelBaseService_GetById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).GetById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/GetById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).GetById(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_Get_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).Get(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/Get", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).Get(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_GetList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).GetList(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/GetList", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).GetList(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_DeleteById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).DeleteById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/DeleteById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).DeleteById(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).Delete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/Delete", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).Delete(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_DeleteList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).DeleteList(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/DeleteList", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).DeleteList(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_ForceDeleteList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).ForceDeleteList(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/ForceDeleteList", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).ForceDeleteList(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_UpdateById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).UpdateById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/UpdateById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).UpdateById(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_Update_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).Update(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/Update", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).Update(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_UpdateDoc_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).UpdateDoc(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/UpdateDoc", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).UpdateDoc(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_Insert_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).Insert(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/Insert", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).Insert(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_Count_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).Count(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/Count", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).Count(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +// ModelBaseService_ServiceDesc is the grpc.ServiceDesc for ModelBaseService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ModelBaseService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.ModelBaseService", + HandlerType: (*ModelBaseServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetById", + Handler: _ModelBaseService_GetById_Handler, + }, + { + MethodName: "Get", + Handler: _ModelBaseService_Get_Handler, + }, + { + MethodName: "GetList", + Handler: _ModelBaseService_GetList_Handler, + }, + { + MethodName: "DeleteById", + Handler: _ModelBaseService_DeleteById_Handler, + }, + { + MethodName: "Delete", + Handler: _ModelBaseService_Delete_Handler, + }, + { + MethodName: "DeleteList", + Handler: _ModelBaseService_DeleteList_Handler, + }, + { + MethodName: "ForceDeleteList", + Handler: _ModelBaseService_ForceDeleteList_Handler, + }, + { + MethodName: "UpdateById", + Handler: _ModelBaseService_UpdateById_Handler, + }, + { + MethodName: "Update", + Handler: _ModelBaseService_Update_Handler, + }, + { + MethodName: "UpdateDoc", + Handler: _ModelBaseService_UpdateDoc_Handler, + }, + { + MethodName: "Insert", + Handler: _ModelBaseService_Insert_Handler, + }, + { + MethodName: "Count", + Handler: _ModelBaseService_Count_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "services/model_base_service.proto", +} diff --git a/grpc/model_base_service_v2.pb.go b/grpc/model_base_service_v2.pb.go new file mode 100644 index 000000000..22377ca9e --- /dev/null +++ b/grpc/model_base_service_v2.pb.go @@ -0,0 +1,179 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/model_base_service_v2.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_model_base_service_v2_proto protoreflect.FileDescriptor + +var file_services_model_base_service_v2_proto_rawDesc = []byte{ + 0x0a, 0x24, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x5f, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x25, 0x65, 0x6e, + 0x74, 0x69, 0x74, 0x79, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x5f, 0x76, 0x32, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xd4, 0x07, 0x0a, 0x12, 0x4d, + 0x6f, 0x64, 0x65, 0x6c, 0x42, 0x61, 0x73, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, + 0x32, 0x12, 0x3f, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x42, 0x79, 0x49, 0x64, 0x12, 0x22, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x56, 0x32, 0x47, 0x65, 0x74, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x65, 0x12, 0x21, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x56, 0x32, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x3f, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x6e, 0x79, 0x12, 0x22, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x56, 0x32, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x79, 0x49, 0x64, + 0x12, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x79, 0x49, 0x64, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x09, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x4f, 0x6e, 0x65, 0x12, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, + 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x44, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x4f, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, + 0x0a, 0x0a, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x79, 0x12, 0x25, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x56, 0x32, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, + 0x79, 0x49, 0x64, 0x12, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, + 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x09, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4f, 0x6e, 0x65, 0x12, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x4f, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x79, 0x12, + 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x79, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x47, 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6c, + 0x61, 0x63, 0x65, 0x42, 0x79, 0x49, 0x64, 0x12, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, + 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x52, 0x65, 0x70, + 0x6c, 0x61, 0x63, 0x65, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4f, 0x6e, 0x65, 0x12, + 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x56, 0x32, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4f, 0x6e, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x09, 0x49, 0x6e, 0x73, 0x65, + 0x72, 0x74, 0x4f, 0x6e, 0x65, 0x12, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, + 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x49, 0x6e, 0x73, 0x65, 0x72, + 0x74, 0x4f, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, 0x0a, + 0x0a, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x4d, 0x61, 0x6e, 0x79, 0x12, 0x25, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, + 0x32, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x3b, 0x0a, 0x05, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x20, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x56, 0x32, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var file_services_model_base_service_v2_proto_goTypes = []interface{}{ + (*ModelServiceV2GetByIdRequest)(nil), // 0: grpc.ModelServiceV2GetByIdRequest + (*ModelServiceV2GetOneRequest)(nil), // 1: grpc.ModelServiceV2GetOneRequest + (*ModelServiceV2GetManyRequest)(nil), // 2: grpc.ModelServiceV2GetManyRequest + (*ModelServiceV2DeleteByIdRequest)(nil), // 3: grpc.ModelServiceV2DeleteByIdRequest + (*ModelServiceV2DeleteOneRequest)(nil), // 4: grpc.ModelServiceV2DeleteOneRequest + (*ModelServiceV2DeleteManyRequest)(nil), // 5: grpc.ModelServiceV2DeleteManyRequest + (*ModelServiceV2UpdateByIdRequest)(nil), // 6: grpc.ModelServiceV2UpdateByIdRequest + (*ModelServiceV2UpdateOneRequest)(nil), // 7: grpc.ModelServiceV2UpdateOneRequest + (*ModelServiceV2UpdateManyRequest)(nil), // 8: grpc.ModelServiceV2UpdateManyRequest + (*ModelServiceV2ReplaceByIdRequest)(nil), // 9: grpc.ModelServiceV2ReplaceByIdRequest + (*ModelServiceV2ReplaceOneRequest)(nil), // 10: grpc.ModelServiceV2ReplaceOneRequest + (*ModelServiceV2InsertOneRequest)(nil), // 11: grpc.ModelServiceV2InsertOneRequest + (*ModelServiceV2InsertManyRequest)(nil), // 12: grpc.ModelServiceV2InsertManyRequest + (*ModelServiceV2CountRequest)(nil), // 13: grpc.ModelServiceV2CountRequest + (*Response)(nil), // 14: grpc.Response +} +var file_services_model_base_service_v2_proto_depIdxs = []int32{ + 0, // 0: grpc.ModelBaseServiceV2.GetById:input_type -> grpc.ModelServiceV2GetByIdRequest + 1, // 1: grpc.ModelBaseServiceV2.GetOne:input_type -> grpc.ModelServiceV2GetOneRequest + 2, // 2: grpc.ModelBaseServiceV2.GetMany:input_type -> grpc.ModelServiceV2GetManyRequest + 3, // 3: grpc.ModelBaseServiceV2.DeleteById:input_type -> grpc.ModelServiceV2DeleteByIdRequest + 4, // 4: grpc.ModelBaseServiceV2.DeleteOne:input_type -> grpc.ModelServiceV2DeleteOneRequest + 5, // 5: grpc.ModelBaseServiceV2.DeleteMany:input_type -> grpc.ModelServiceV2DeleteManyRequest + 6, // 6: grpc.ModelBaseServiceV2.UpdateById:input_type -> grpc.ModelServiceV2UpdateByIdRequest + 7, // 7: grpc.ModelBaseServiceV2.UpdateOne:input_type -> grpc.ModelServiceV2UpdateOneRequest + 8, // 8: grpc.ModelBaseServiceV2.UpdateMany:input_type -> grpc.ModelServiceV2UpdateManyRequest + 9, // 9: grpc.ModelBaseServiceV2.ReplaceById:input_type -> grpc.ModelServiceV2ReplaceByIdRequest + 10, // 10: grpc.ModelBaseServiceV2.ReplaceOne:input_type -> grpc.ModelServiceV2ReplaceOneRequest + 11, // 11: grpc.ModelBaseServiceV2.InsertOne:input_type -> grpc.ModelServiceV2InsertOneRequest + 12, // 12: grpc.ModelBaseServiceV2.InsertMany:input_type -> grpc.ModelServiceV2InsertManyRequest + 13, // 13: grpc.ModelBaseServiceV2.Count:input_type -> grpc.ModelServiceV2CountRequest + 14, // 14: grpc.ModelBaseServiceV2.GetById:output_type -> grpc.Response + 14, // 15: grpc.ModelBaseServiceV2.GetOne:output_type -> grpc.Response + 14, // 16: grpc.ModelBaseServiceV2.GetMany:output_type -> grpc.Response + 14, // 17: grpc.ModelBaseServiceV2.DeleteById:output_type -> grpc.Response + 14, // 18: grpc.ModelBaseServiceV2.DeleteOne:output_type -> grpc.Response + 14, // 19: grpc.ModelBaseServiceV2.DeleteMany:output_type -> grpc.Response + 14, // 20: grpc.ModelBaseServiceV2.UpdateById:output_type -> grpc.Response + 14, // 21: grpc.ModelBaseServiceV2.UpdateOne:output_type -> grpc.Response + 14, // 22: grpc.ModelBaseServiceV2.UpdateMany:output_type -> grpc.Response + 14, // 23: grpc.ModelBaseServiceV2.ReplaceById:output_type -> grpc.Response + 14, // 24: grpc.ModelBaseServiceV2.ReplaceOne:output_type -> grpc.Response + 14, // 25: grpc.ModelBaseServiceV2.InsertOne:output_type -> grpc.Response + 14, // 26: grpc.ModelBaseServiceV2.InsertMany:output_type -> grpc.Response + 14, // 27: grpc.ModelBaseServiceV2.Count:output_type -> grpc.Response + 14, // [14:28] is the sub-list for method output_type + 0, // [0:14] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_model_base_service_v2_proto_init() } +func file_services_model_base_service_v2_proto_init() { + if File_services_model_base_service_v2_proto != nil { + return + } + file_entity_model_service_v2_request_proto_init() + file_entity_response_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_model_base_service_v2_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_model_base_service_v2_proto_goTypes, + DependencyIndexes: file_services_model_base_service_v2_proto_depIdxs, + }.Build() + File_services_model_base_service_v2_proto = out.File + file_services_model_base_service_v2_proto_rawDesc = nil + file_services_model_base_service_v2_proto_goTypes = nil + file_services_model_base_service_v2_proto_depIdxs = nil +} diff --git a/grpc/model_base_service_v2_grpc.pb.go b/grpc/model_base_service_v2_grpc.pb.go new file mode 100644 index 000000000..6f5fac73f --- /dev/null +++ b/grpc/model_base_service_v2_grpc.pb.go @@ -0,0 +1,573 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/model_base_service_v2.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// ModelBaseServiceV2Client is the client API for ModelBaseServiceV2 service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ModelBaseServiceV2Client interface { + GetById(ctx context.Context, in *ModelServiceV2GetByIdRequest, opts ...grpc.CallOption) (*Response, error) + GetOne(ctx context.Context, in *ModelServiceV2GetOneRequest, opts ...grpc.CallOption) (*Response, error) + GetMany(ctx context.Context, in *ModelServiceV2GetManyRequest, opts ...grpc.CallOption) (*Response, error) + DeleteById(ctx context.Context, in *ModelServiceV2DeleteByIdRequest, opts ...grpc.CallOption) (*Response, error) + DeleteOne(ctx context.Context, in *ModelServiceV2DeleteOneRequest, opts ...grpc.CallOption) (*Response, error) + DeleteMany(ctx context.Context, in *ModelServiceV2DeleteManyRequest, opts ...grpc.CallOption) (*Response, error) + UpdateById(ctx context.Context, in *ModelServiceV2UpdateByIdRequest, opts ...grpc.CallOption) (*Response, error) + UpdateOne(ctx context.Context, in *ModelServiceV2UpdateOneRequest, opts ...grpc.CallOption) (*Response, error) + UpdateMany(ctx context.Context, in *ModelServiceV2UpdateManyRequest, opts ...grpc.CallOption) (*Response, error) + ReplaceById(ctx context.Context, in *ModelServiceV2ReplaceByIdRequest, opts ...grpc.CallOption) (*Response, error) + ReplaceOne(ctx context.Context, in *ModelServiceV2ReplaceOneRequest, opts ...grpc.CallOption) (*Response, error) + InsertOne(ctx context.Context, in *ModelServiceV2InsertOneRequest, opts ...grpc.CallOption) (*Response, error) + InsertMany(ctx context.Context, in *ModelServiceV2InsertManyRequest, opts ...grpc.CallOption) (*Response, error) + Count(ctx context.Context, in *ModelServiceV2CountRequest, opts ...grpc.CallOption) (*Response, error) +} + +type modelBaseServiceV2Client struct { + cc grpc.ClientConnInterface +} + +func NewModelBaseServiceV2Client(cc grpc.ClientConnInterface) ModelBaseServiceV2Client { + return &modelBaseServiceV2Client{cc} +} + +func (c *modelBaseServiceV2Client) GetById(ctx context.Context, in *ModelServiceV2GetByIdRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) GetOne(ctx context.Context, in *ModelServiceV2GetOneRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetOne", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) GetMany(ctx context.Context, in *ModelServiceV2GetManyRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetMany", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) DeleteById(ctx context.Context, in *ModelServiceV2DeleteByIdRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) DeleteOne(ctx context.Context, in *ModelServiceV2DeleteOneRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteOne", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) DeleteMany(ctx context.Context, in *ModelServiceV2DeleteManyRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteMany", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) UpdateById(ctx context.Context, in *ModelServiceV2UpdateByIdRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) UpdateOne(ctx context.Context, in *ModelServiceV2UpdateOneRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateOne", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) UpdateMany(ctx context.Context, in *ModelServiceV2UpdateManyRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateMany", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) ReplaceById(ctx context.Context, in *ModelServiceV2ReplaceByIdRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/ReplaceById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) ReplaceOne(ctx context.Context, in *ModelServiceV2ReplaceOneRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/ReplaceOne", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) InsertOne(ctx context.Context, in *ModelServiceV2InsertOneRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/InsertOne", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) InsertMany(ctx context.Context, in *ModelServiceV2InsertManyRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/InsertMany", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) Count(ctx context.Context, in *ModelServiceV2CountRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/Count", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ModelBaseServiceV2Server is the server API for ModelBaseServiceV2 service. +// All implementations must embed UnimplementedModelBaseServiceV2Server +// for forward compatibility +type ModelBaseServiceV2Server interface { + GetById(context.Context, *ModelServiceV2GetByIdRequest) (*Response, error) + GetOne(context.Context, *ModelServiceV2GetOneRequest) (*Response, error) + GetMany(context.Context, *ModelServiceV2GetManyRequest) (*Response, error) + DeleteById(context.Context, *ModelServiceV2DeleteByIdRequest) (*Response, error) + DeleteOne(context.Context, *ModelServiceV2DeleteOneRequest) (*Response, error) + DeleteMany(context.Context, *ModelServiceV2DeleteManyRequest) (*Response, error) + UpdateById(context.Context, *ModelServiceV2UpdateByIdRequest) (*Response, error) + UpdateOne(context.Context, *ModelServiceV2UpdateOneRequest) (*Response, error) + UpdateMany(context.Context, *ModelServiceV2UpdateManyRequest) (*Response, error) + ReplaceById(context.Context, *ModelServiceV2ReplaceByIdRequest) (*Response, error) + ReplaceOne(context.Context, *ModelServiceV2ReplaceOneRequest) (*Response, error) + InsertOne(context.Context, *ModelServiceV2InsertOneRequest) (*Response, error) + InsertMany(context.Context, *ModelServiceV2InsertManyRequest) (*Response, error) + Count(context.Context, *ModelServiceV2CountRequest) (*Response, error) + mustEmbedUnimplementedModelBaseServiceV2Server() +} + +// UnimplementedModelBaseServiceV2Server must be embedded to have forward compatible implementations. +type UnimplementedModelBaseServiceV2Server struct { +} + +func (UnimplementedModelBaseServiceV2Server) GetById(context.Context, *ModelServiceV2GetByIdRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetById not implemented") +} +func (UnimplementedModelBaseServiceV2Server) GetOne(context.Context, *ModelServiceV2GetOneRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetOne not implemented") +} +func (UnimplementedModelBaseServiceV2Server) GetMany(context.Context, *ModelServiceV2GetManyRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetMany not implemented") +} +func (UnimplementedModelBaseServiceV2Server) DeleteById(context.Context, *ModelServiceV2DeleteByIdRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteById not implemented") +} +func (UnimplementedModelBaseServiceV2Server) DeleteOne(context.Context, *ModelServiceV2DeleteOneRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteOne not implemented") +} +func (UnimplementedModelBaseServiceV2Server) DeleteMany(context.Context, *ModelServiceV2DeleteManyRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteMany not implemented") +} +func (UnimplementedModelBaseServiceV2Server) UpdateById(context.Context, *ModelServiceV2UpdateByIdRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateById not implemented") +} +func (UnimplementedModelBaseServiceV2Server) UpdateOne(context.Context, *ModelServiceV2UpdateOneRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateOne not implemented") +} +func (UnimplementedModelBaseServiceV2Server) UpdateMany(context.Context, *ModelServiceV2UpdateManyRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateMany not implemented") +} +func (UnimplementedModelBaseServiceV2Server) ReplaceById(context.Context, *ModelServiceV2ReplaceByIdRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReplaceById not implemented") +} +func (UnimplementedModelBaseServiceV2Server) ReplaceOne(context.Context, *ModelServiceV2ReplaceOneRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReplaceOne not implemented") +} +func (UnimplementedModelBaseServiceV2Server) InsertOne(context.Context, *ModelServiceV2InsertOneRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method InsertOne not implemented") +} +func (UnimplementedModelBaseServiceV2Server) InsertMany(context.Context, *ModelServiceV2InsertManyRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method InsertMany not implemented") +} +func (UnimplementedModelBaseServiceV2Server) Count(context.Context, *ModelServiceV2CountRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Count not implemented") +} +func (UnimplementedModelBaseServiceV2Server) mustEmbedUnimplementedModelBaseServiceV2Server() {} + +// UnsafeModelBaseServiceV2Server may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ModelBaseServiceV2Server will +// result in compilation errors. +type UnsafeModelBaseServiceV2Server interface { + mustEmbedUnimplementedModelBaseServiceV2Server() +} + +func RegisterModelBaseServiceV2Server(s grpc.ServiceRegistrar, srv ModelBaseServiceV2Server) { + s.RegisterService(&ModelBaseServiceV2_ServiceDesc, srv) +} + +func _ModelBaseServiceV2_GetById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2GetByIdRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).GetById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/GetById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).GetById(ctx, req.(*ModelServiceV2GetByIdRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_GetOne_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2GetOneRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).GetOne(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/GetOne", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).GetOne(ctx, req.(*ModelServiceV2GetOneRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_GetMany_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2GetManyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).GetMany(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/GetMany", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).GetMany(ctx, req.(*ModelServiceV2GetManyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_DeleteById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2DeleteByIdRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).DeleteById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/DeleteById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).DeleteById(ctx, req.(*ModelServiceV2DeleteByIdRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_DeleteOne_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2DeleteOneRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).DeleteOne(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/DeleteOne", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).DeleteOne(ctx, req.(*ModelServiceV2DeleteOneRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_DeleteMany_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2DeleteManyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).DeleteMany(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/DeleteMany", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).DeleteMany(ctx, req.(*ModelServiceV2DeleteManyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_UpdateById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2UpdateByIdRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).UpdateById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/UpdateById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).UpdateById(ctx, req.(*ModelServiceV2UpdateByIdRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_UpdateOne_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2UpdateOneRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).UpdateOne(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/UpdateOne", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).UpdateOne(ctx, req.(*ModelServiceV2UpdateOneRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_UpdateMany_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2UpdateManyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).UpdateMany(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/UpdateMany", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).UpdateMany(ctx, req.(*ModelServiceV2UpdateManyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_ReplaceById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2ReplaceByIdRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).ReplaceById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/ReplaceById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).ReplaceById(ctx, req.(*ModelServiceV2ReplaceByIdRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_ReplaceOne_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2ReplaceOneRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).ReplaceOne(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/ReplaceOne", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).ReplaceOne(ctx, req.(*ModelServiceV2ReplaceOneRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_InsertOne_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2InsertOneRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).InsertOne(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/InsertOne", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).InsertOne(ctx, req.(*ModelServiceV2InsertOneRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_InsertMany_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2InsertManyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).InsertMany(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/InsertMany", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).InsertMany(ctx, req.(*ModelServiceV2InsertManyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_Count_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2CountRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).Count(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/Count", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).Count(ctx, req.(*ModelServiceV2CountRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// ModelBaseServiceV2_ServiceDesc is the grpc.ServiceDesc for ModelBaseServiceV2 service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ModelBaseServiceV2_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.ModelBaseServiceV2", + HandlerType: (*ModelBaseServiceV2Server)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetById", + Handler: _ModelBaseServiceV2_GetById_Handler, + }, + { + MethodName: "GetOne", + Handler: _ModelBaseServiceV2_GetOne_Handler, + }, + { + MethodName: "GetMany", + Handler: _ModelBaseServiceV2_GetMany_Handler, + }, + { + MethodName: "DeleteById", + Handler: _ModelBaseServiceV2_DeleteById_Handler, + }, + { + MethodName: "DeleteOne", + Handler: _ModelBaseServiceV2_DeleteOne_Handler, + }, + { + MethodName: "DeleteMany", + Handler: _ModelBaseServiceV2_DeleteMany_Handler, + }, + { + MethodName: "UpdateById", + Handler: _ModelBaseServiceV2_UpdateById_Handler, + }, + { + MethodName: "UpdateOne", + Handler: _ModelBaseServiceV2_UpdateOne_Handler, + }, + { + MethodName: "UpdateMany", + Handler: _ModelBaseServiceV2_UpdateMany_Handler, + }, + { + MethodName: "ReplaceById", + Handler: _ModelBaseServiceV2_ReplaceById_Handler, + }, + { + MethodName: "ReplaceOne", + Handler: _ModelBaseServiceV2_ReplaceOne_Handler, + }, + { + MethodName: "InsertOne", + Handler: _ModelBaseServiceV2_InsertOne_Handler, + }, + { + MethodName: "InsertMany", + Handler: _ModelBaseServiceV2_InsertMany_Handler, + }, + { + MethodName: "Count", + Handler: _ModelBaseServiceV2_Count_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "services/model_base_service_v2.proto", +} diff --git a/grpc/model_delegate.pb.go b/grpc/model_delegate.pb.go new file mode 100644 index 000000000..76dfa4897 --- /dev/null +++ b/grpc/model_delegate.pb.go @@ -0,0 +1,80 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/model_delegate.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_model_delegate_proto protoreflect.FileDescriptor + +var file_services_model_delegate_proto_rawDesc = []byte{ + 0x0a, 0x1d, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x67, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x14, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, + 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x32, 0x36, 0x0a, 0x0d, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x44, 0x65, 0x6c, 0x65, 0x67, + 0x61, 0x74, 0x65, 0x12, 0x25, 0x0a, 0x02, 0x44, 0x6f, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, + 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_services_model_delegate_proto_goTypes = []interface{}{ + (*Request)(nil), // 0: grpc.Request + (*Response)(nil), // 1: grpc.Response +} +var file_services_model_delegate_proto_depIdxs = []int32{ + 0, // 0: grpc.ModelDelegate.Do:input_type -> grpc.Request + 1, // 1: grpc.ModelDelegate.Do:output_type -> grpc.Response + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_model_delegate_proto_init() } +func file_services_model_delegate_proto_init() { + if File_services_model_delegate_proto != nil { + return + } + file_entity_request_proto_init() + file_entity_response_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_model_delegate_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_model_delegate_proto_goTypes, + DependencyIndexes: file_services_model_delegate_proto_depIdxs, + }.Build() + File_services_model_delegate_proto = out.File + file_services_model_delegate_proto_rawDesc = nil + file_services_model_delegate_proto_goTypes = nil + file_services_model_delegate_proto_depIdxs = nil +} diff --git a/grpc/model_delegate_grpc.pb.go b/grpc/model_delegate_grpc.pb.go new file mode 100644 index 000000000..3ab1efa50 --- /dev/null +++ b/grpc/model_delegate_grpc.pb.go @@ -0,0 +1,105 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/model_delegate.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// ModelDelegateClient is the client API for ModelDelegate service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ModelDelegateClient interface { + Do(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) +} + +type modelDelegateClient struct { + cc grpc.ClientConnInterface +} + +func NewModelDelegateClient(cc grpc.ClientConnInterface) ModelDelegateClient { + return &modelDelegateClient{cc} +} + +func (c *modelDelegateClient) Do(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelDelegate/Do", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ModelDelegateServer is the server API for ModelDelegate service. +// All implementations must embed UnimplementedModelDelegateServer +// for forward compatibility +type ModelDelegateServer interface { + Do(context.Context, *Request) (*Response, error) + mustEmbedUnimplementedModelDelegateServer() +} + +// UnimplementedModelDelegateServer must be embedded to have forward compatible implementations. +type UnimplementedModelDelegateServer struct { +} + +func (UnimplementedModelDelegateServer) Do(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Do not implemented") +} +func (UnimplementedModelDelegateServer) mustEmbedUnimplementedModelDelegateServer() {} + +// UnsafeModelDelegateServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ModelDelegateServer will +// result in compilation errors. +type UnsafeModelDelegateServer interface { + mustEmbedUnimplementedModelDelegateServer() +} + +func RegisterModelDelegateServer(s grpc.ServiceRegistrar, srv ModelDelegateServer) { + s.RegisterService(&ModelDelegate_ServiceDesc, srv) +} + +func _ModelDelegate_Do_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelDelegateServer).Do(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelDelegate/Do", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelDelegateServer).Do(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +// ModelDelegate_ServiceDesc is the grpc.ServiceDesc for ModelDelegate service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ModelDelegate_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.ModelDelegate", + HandlerType: (*ModelDelegateServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Do", + Handler: _ModelDelegate_Do_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "services/model_delegate.proto", +} diff --git a/grpc/model_service_v2_request.pb.go b/grpc/model_service_v2_request.pb.go new file mode 100644 index 000000000..b8958a9c6 --- /dev/null +++ b/grpc/model_service_v2_request.pb.go @@ -0,0 +1,1316 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/model_service_v2_request.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type ModelServiceV2GetByIdRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *ModelServiceV2GetByIdRequest) Reset() { + *x = ModelServiceV2GetByIdRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2GetByIdRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2GetByIdRequest) ProtoMessage() {} + +func (x *ModelServiceV2GetByIdRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2GetByIdRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2GetByIdRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{0} +} + +func (x *ModelServiceV2GetByIdRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2GetByIdRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2GetByIdRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type ModelServiceV2GetOneRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` + FindOptions []byte `protobuf:"bytes,4,opt,name=find_options,json=findOptions,proto3" json:"find_options,omitempty"` +} + +func (x *ModelServiceV2GetOneRequest) Reset() { + *x = ModelServiceV2GetOneRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2GetOneRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2GetOneRequest) ProtoMessage() {} + +func (x *ModelServiceV2GetOneRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2GetOneRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2GetOneRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{1} +} + +func (x *ModelServiceV2GetOneRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2GetOneRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2GetOneRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +func (x *ModelServiceV2GetOneRequest) GetFindOptions() []byte { + if x != nil { + return x.FindOptions + } + return nil +} + +type ModelServiceV2GetManyRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` + FindOptions []byte `protobuf:"bytes,4,opt,name=find_options,json=findOptions,proto3" json:"find_options,omitempty"` +} + +func (x *ModelServiceV2GetManyRequest) Reset() { + *x = ModelServiceV2GetManyRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2GetManyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2GetManyRequest) ProtoMessage() {} + +func (x *ModelServiceV2GetManyRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2GetManyRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2GetManyRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{2} +} + +func (x *ModelServiceV2GetManyRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2GetManyRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2GetManyRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +func (x *ModelServiceV2GetManyRequest) GetFindOptions() []byte { + if x != nil { + return x.FindOptions + } + return nil +} + +type ModelServiceV2DeleteByIdRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *ModelServiceV2DeleteByIdRequest) Reset() { + *x = ModelServiceV2DeleteByIdRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2DeleteByIdRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2DeleteByIdRequest) ProtoMessage() {} + +func (x *ModelServiceV2DeleteByIdRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2DeleteByIdRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2DeleteByIdRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{3} +} + +func (x *ModelServiceV2DeleteByIdRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2DeleteByIdRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2DeleteByIdRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type ModelServiceV2DeleteOneRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` +} + +func (x *ModelServiceV2DeleteOneRequest) Reset() { + *x = ModelServiceV2DeleteOneRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2DeleteOneRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2DeleteOneRequest) ProtoMessage() {} + +func (x *ModelServiceV2DeleteOneRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2DeleteOneRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2DeleteOneRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{4} +} + +func (x *ModelServiceV2DeleteOneRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2DeleteOneRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2DeleteOneRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +type ModelServiceV2DeleteManyRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` +} + +func (x *ModelServiceV2DeleteManyRequest) Reset() { + *x = ModelServiceV2DeleteManyRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2DeleteManyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2DeleteManyRequest) ProtoMessage() {} + +func (x *ModelServiceV2DeleteManyRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2DeleteManyRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2DeleteManyRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{5} +} + +func (x *ModelServiceV2DeleteManyRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2DeleteManyRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2DeleteManyRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +type ModelServiceV2UpdateByIdRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` + Update []byte `protobuf:"bytes,4,opt,name=update,proto3" json:"update,omitempty"` +} + +func (x *ModelServiceV2UpdateByIdRequest) Reset() { + *x = ModelServiceV2UpdateByIdRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2UpdateByIdRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2UpdateByIdRequest) ProtoMessage() {} + +func (x *ModelServiceV2UpdateByIdRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2UpdateByIdRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2UpdateByIdRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{6} +} + +func (x *ModelServiceV2UpdateByIdRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2UpdateByIdRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2UpdateByIdRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *ModelServiceV2UpdateByIdRequest) GetUpdate() []byte { + if x != nil { + return x.Update + } + return nil +} + +type ModelServiceV2UpdateOneRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` + Update []byte `protobuf:"bytes,4,opt,name=update,proto3" json:"update,omitempty"` +} + +func (x *ModelServiceV2UpdateOneRequest) Reset() { + *x = ModelServiceV2UpdateOneRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2UpdateOneRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2UpdateOneRequest) ProtoMessage() {} + +func (x *ModelServiceV2UpdateOneRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2UpdateOneRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2UpdateOneRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{7} +} + +func (x *ModelServiceV2UpdateOneRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2UpdateOneRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2UpdateOneRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +func (x *ModelServiceV2UpdateOneRequest) GetUpdate() []byte { + if x != nil { + return x.Update + } + return nil +} + +type ModelServiceV2UpdateManyRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` + Update []byte `protobuf:"bytes,4,opt,name=update,proto3" json:"update,omitempty"` +} + +func (x *ModelServiceV2UpdateManyRequest) Reset() { + *x = ModelServiceV2UpdateManyRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2UpdateManyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2UpdateManyRequest) ProtoMessage() {} + +func (x *ModelServiceV2UpdateManyRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2UpdateManyRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2UpdateManyRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{8} +} + +func (x *ModelServiceV2UpdateManyRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2UpdateManyRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2UpdateManyRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +func (x *ModelServiceV2UpdateManyRequest) GetUpdate() []byte { + if x != nil { + return x.Update + } + return nil +} + +type ModelServiceV2ReplaceByIdRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` + Model []byte `protobuf:"bytes,4,opt,name=model,proto3" json:"model,omitempty"` +} + +func (x *ModelServiceV2ReplaceByIdRequest) Reset() { + *x = ModelServiceV2ReplaceByIdRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2ReplaceByIdRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2ReplaceByIdRequest) ProtoMessage() {} + +func (x *ModelServiceV2ReplaceByIdRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2ReplaceByIdRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2ReplaceByIdRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{9} +} + +func (x *ModelServiceV2ReplaceByIdRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2ReplaceByIdRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2ReplaceByIdRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *ModelServiceV2ReplaceByIdRequest) GetModel() []byte { + if x != nil { + return x.Model + } + return nil +} + +type ModelServiceV2ReplaceOneRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` + Model []byte `protobuf:"bytes,4,opt,name=model,proto3" json:"model,omitempty"` +} + +func (x *ModelServiceV2ReplaceOneRequest) Reset() { + *x = ModelServiceV2ReplaceOneRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2ReplaceOneRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2ReplaceOneRequest) ProtoMessage() {} + +func (x *ModelServiceV2ReplaceOneRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2ReplaceOneRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2ReplaceOneRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{10} +} + +func (x *ModelServiceV2ReplaceOneRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2ReplaceOneRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2ReplaceOneRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +func (x *ModelServiceV2ReplaceOneRequest) GetModel() []byte { + if x != nil { + return x.Model + } + return nil +} + +type ModelServiceV2InsertOneRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Model []byte `protobuf:"bytes,3,opt,name=model,proto3" json:"model,omitempty"` +} + +func (x *ModelServiceV2InsertOneRequest) Reset() { + *x = ModelServiceV2InsertOneRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2InsertOneRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2InsertOneRequest) ProtoMessage() {} + +func (x *ModelServiceV2InsertOneRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2InsertOneRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2InsertOneRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{11} +} + +func (x *ModelServiceV2InsertOneRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2InsertOneRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2InsertOneRequest) GetModel() []byte { + if x != nil { + return x.Model + } + return nil +} + +type ModelServiceV2InsertManyRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Models []byte `protobuf:"bytes,3,opt,name=models,proto3" json:"models,omitempty"` +} + +func (x *ModelServiceV2InsertManyRequest) Reset() { + *x = ModelServiceV2InsertManyRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2InsertManyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2InsertManyRequest) ProtoMessage() {} + +func (x *ModelServiceV2InsertManyRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2InsertManyRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2InsertManyRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{12} +} + +func (x *ModelServiceV2InsertManyRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2InsertManyRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2InsertManyRequest) GetModels() []byte { + if x != nil { + return x.Models + } + return nil +} + +type ModelServiceV2CountRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` +} + +func (x *ModelServiceV2CountRequest) Reset() { + *x = ModelServiceV2CountRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2CountRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2CountRequest) ProtoMessage() {} + +func (x *ModelServiceV2CountRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2CountRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2CountRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{13} +} + +func (x *ModelServiceV2CountRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2CountRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2CountRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +var File_entity_model_service_v2_request_proto protoreflect.FileDescriptor + +var file_entity_model_service_v2_request_proto_rawDesc = []byte{ + 0x0a, 0x25, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0x68, 0x0a, + 0x1c, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x47, + 0x65, 0x74, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, + 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, + 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x90, 0x01, 0x0a, 0x1b, 0x4d, 0x6f, 0x64, 0x65, + 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, + 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x69, 0x6e, 0x64, 0x5f, + 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x66, + 0x69, 0x6e, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x91, 0x01, 0x0a, 0x1c, 0x4d, + 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x47, 0x65, 0x74, + 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, + 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, + 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x66, + 0x69, 0x6e, 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x0b, 0x66, 0x69, 0x6e, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x6b, + 0x0a, 0x1f, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x70, 0x0a, 0x1e, 0x4d, + 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x4f, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, + 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, + 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x22, 0x71, 0x0a, + 0x1f, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, + 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x22, 0x83, 0x01, 0x0a, 0x1f, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, + 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, + 0x0a, 0x06, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, + 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x22, 0x88, 0x01, 0x0a, 0x1e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4f, + 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, + 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, + 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x22, 0x89, 0x01, 0x0a, 0x1f, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, + 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, + 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x22, 0x82, 0x01, + 0x0a, 0x20, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, + 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, + 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x22, 0x87, 0x01, 0x0a, 0x1f, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x56, 0x32, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4f, 0x6e, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, + 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x22, 0x70, 0x0a, 0x1e, + 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x49, 0x6e, + 0x73, 0x65, 0x72, 0x74, 0x4f, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, + 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x22, 0x73, + 0x0a, 0x1f, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, + 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x73, 0x22, 0x6c, 0x0a, 0x1a, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, + 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, + 0x79, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var ( + file_entity_model_service_v2_request_proto_rawDescOnce sync.Once + file_entity_model_service_v2_request_proto_rawDescData = file_entity_model_service_v2_request_proto_rawDesc +) + +func file_entity_model_service_v2_request_proto_rawDescGZIP() []byte { + file_entity_model_service_v2_request_proto_rawDescOnce.Do(func() { + file_entity_model_service_v2_request_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_model_service_v2_request_proto_rawDescData) + }) + return file_entity_model_service_v2_request_proto_rawDescData +} + +var file_entity_model_service_v2_request_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_entity_model_service_v2_request_proto_goTypes = []interface{}{ + (*ModelServiceV2GetByIdRequest)(nil), // 0: grpc.ModelServiceV2GetByIdRequest + (*ModelServiceV2GetOneRequest)(nil), // 1: grpc.ModelServiceV2GetOneRequest + (*ModelServiceV2GetManyRequest)(nil), // 2: grpc.ModelServiceV2GetManyRequest + (*ModelServiceV2DeleteByIdRequest)(nil), // 3: grpc.ModelServiceV2DeleteByIdRequest + (*ModelServiceV2DeleteOneRequest)(nil), // 4: grpc.ModelServiceV2DeleteOneRequest + (*ModelServiceV2DeleteManyRequest)(nil), // 5: grpc.ModelServiceV2DeleteManyRequest + (*ModelServiceV2UpdateByIdRequest)(nil), // 6: grpc.ModelServiceV2UpdateByIdRequest + (*ModelServiceV2UpdateOneRequest)(nil), // 7: grpc.ModelServiceV2UpdateOneRequest + (*ModelServiceV2UpdateManyRequest)(nil), // 8: grpc.ModelServiceV2UpdateManyRequest + (*ModelServiceV2ReplaceByIdRequest)(nil), // 9: grpc.ModelServiceV2ReplaceByIdRequest + (*ModelServiceV2ReplaceOneRequest)(nil), // 10: grpc.ModelServiceV2ReplaceOneRequest + (*ModelServiceV2InsertOneRequest)(nil), // 11: grpc.ModelServiceV2InsertOneRequest + (*ModelServiceV2InsertManyRequest)(nil), // 12: grpc.ModelServiceV2InsertManyRequest + (*ModelServiceV2CountRequest)(nil), // 13: grpc.ModelServiceV2CountRequest +} +var file_entity_model_service_v2_request_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_model_service_v2_request_proto_init() } +func file_entity_model_service_v2_request_proto_init() { + if File_entity_model_service_v2_request_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_entity_model_service_v2_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2GetByIdRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2GetOneRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2GetManyRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2DeleteByIdRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2DeleteOneRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2DeleteManyRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2UpdateByIdRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2UpdateOneRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2UpdateManyRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2ReplaceByIdRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2ReplaceOneRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2InsertOneRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2InsertManyRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2CountRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_model_service_v2_request_proto_rawDesc, + NumEnums: 0, + NumMessages: 14, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_model_service_v2_request_proto_goTypes, + DependencyIndexes: file_entity_model_service_v2_request_proto_depIdxs, + MessageInfos: file_entity_model_service_v2_request_proto_msgTypes, + }.Build() + File_entity_model_service_v2_request_proto = out.File + file_entity_model_service_v2_request_proto_rawDesc = nil + file_entity_model_service_v2_request_proto_goTypes = nil + file_entity_model_service_v2_request_proto_depIdxs = nil +} diff --git a/grpc/node.pb.go b/grpc/node.pb.go new file mode 100644 index 000000000..2e32c4002 --- /dev/null +++ b/grpc/node.pb.go @@ -0,0 +1,251 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: models/node.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type Node struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + XId string `protobuf:"bytes,1,opt,name=_id,json=Id,proto3" json:"_id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Ip string `protobuf:"bytes,3,opt,name=ip,proto3" json:"ip,omitempty"` + Port string `protobuf:"bytes,5,opt,name=port,proto3" json:"port,omitempty"` + Mac string `protobuf:"bytes,6,opt,name=mac,proto3" json:"mac,omitempty"` + Hostname string `protobuf:"bytes,7,opt,name=hostname,proto3" json:"hostname,omitempty"` + Description string `protobuf:"bytes,8,opt,name=description,proto3" json:"description,omitempty"` + Key string `protobuf:"bytes,9,opt,name=key,proto3" json:"key,omitempty"` + IsMaster bool `protobuf:"varint,11,opt,name=is_master,json=isMaster,proto3" json:"is_master,omitempty"` + UpdateTs string `protobuf:"bytes,12,opt,name=update_ts,json=updateTs,proto3" json:"update_ts,omitempty"` + CreateTs string `protobuf:"bytes,13,opt,name=create_ts,json=createTs,proto3" json:"create_ts,omitempty"` + UpdateTsUnix int64 `protobuf:"varint,14,opt,name=update_ts_unix,json=updateTsUnix,proto3" json:"update_ts_unix,omitempty"` +} + +func (x *Node) Reset() { + *x = Node{} + if protoimpl.UnsafeEnabled { + mi := &file_models_node_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Node) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Node) ProtoMessage() {} + +func (x *Node) ProtoReflect() protoreflect.Message { + mi := &file_models_node_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Node.ProtoReflect.Descriptor instead. +func (*Node) Descriptor() ([]byte, []int) { + return file_models_node_proto_rawDescGZIP(), []int{0} +} + +func (x *Node) GetXId() string { + if x != nil { + return x.XId + } + return "" +} + +func (x *Node) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Node) GetIp() string { + if x != nil { + return x.Ip + } + return "" +} + +func (x *Node) GetPort() string { + if x != nil { + return x.Port + } + return "" +} + +func (x *Node) GetMac() string { + if x != nil { + return x.Mac + } + return "" +} + +func (x *Node) GetHostname() string { + if x != nil { + return x.Hostname + } + return "" +} + +func (x *Node) GetDescription() string { + if x != nil { + return x.Description + } + return "" +} + +func (x *Node) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *Node) GetIsMaster() bool { + if x != nil { + return x.IsMaster + } + return false +} + +func (x *Node) GetUpdateTs() string { + if x != nil { + return x.UpdateTs + } + return "" +} + +func (x *Node) GetCreateTs() string { + if x != nil { + return x.CreateTs + } + return "" +} + +func (x *Node) GetUpdateTsUnix() int64 { + if x != nil { + return x.UpdateTsUnix + } + return 0 +} + +var File_models_node_proto protoreflect.FileDescriptor + +var file_models_node_proto_rawDesc = []byte{ + 0x0a, 0x11, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x73, 0x2f, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0xae, 0x02, 0x0a, 0x04, 0x4e, 0x6f, + 0x64, 0x65, 0x12, 0x0f, 0x0a, 0x03, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x02, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6d, + 0x61, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x61, 0x63, 0x12, 0x1a, 0x0a, + 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x1b, 0x0a, + 0x09, 0x69, 0x73, 0x5f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x08, 0x69, 0x73, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x75, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x5f, 0x74, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x54, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, + 0x73, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x75, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x54, 0x73, 0x55, 0x6e, 0x69, 0x78, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, + 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_models_node_proto_rawDescOnce sync.Once + file_models_node_proto_rawDescData = file_models_node_proto_rawDesc +) + +func file_models_node_proto_rawDescGZIP() []byte { + file_models_node_proto_rawDescOnce.Do(func() { + file_models_node_proto_rawDescData = protoimpl.X.CompressGZIP(file_models_node_proto_rawDescData) + }) + return file_models_node_proto_rawDescData +} + +var file_models_node_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_models_node_proto_goTypes = []interface{}{ + (*Node)(nil), // 0: grpc.Node +} +var file_models_node_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_models_node_proto_init() } +func file_models_node_proto_init() { + if File_models_node_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_models_node_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Node); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_models_node_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_models_node_proto_goTypes, + DependencyIndexes: file_models_node_proto_depIdxs, + MessageInfos: file_models_node_proto_msgTypes, + }.Build() + File_models_node_proto = out.File + file_models_node_proto_rawDesc = nil + file_models_node_proto_goTypes = nil + file_models_node_proto_depIdxs = nil +} diff --git a/grpc/node_info.pb.go b/grpc/node_info.pb.go new file mode 100644 index 000000000..135ddcbfb --- /dev/null +++ b/grpc/node_info.pb.go @@ -0,0 +1,156 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/node_info.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type NodeInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + IsMaster bool `protobuf:"varint,2,opt,name=is_master,json=isMaster,proto3" json:"is_master,omitempty"` +} + +func (x *NodeInfo) Reset() { + *x = NodeInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_node_info_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeInfo) ProtoMessage() {} + +func (x *NodeInfo) ProtoReflect() protoreflect.Message { + mi := &file_entity_node_info_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeInfo.ProtoReflect.Descriptor instead. +func (*NodeInfo) Descriptor() ([]byte, []int) { + return file_entity_node_info_proto_rawDescGZIP(), []int{0} +} + +func (x *NodeInfo) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *NodeInfo) GetIsMaster() bool { + if x != nil { + return x.IsMaster + } + return false +} + +var File_entity_node_info_proto protoreflect.FileDescriptor + +var file_entity_node_info_proto_rawDesc = []byte{ + 0x0a, 0x16, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x6e, + 0x66, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0x39, + 0x0a, 0x08, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x1b, 0x0a, 0x09, + 0x69, 0x73, 0x5f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x08, 0x69, 0x73, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, + 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_node_info_proto_rawDescOnce sync.Once + file_entity_node_info_proto_rawDescData = file_entity_node_info_proto_rawDesc +) + +func file_entity_node_info_proto_rawDescGZIP() []byte { + file_entity_node_info_proto_rawDescOnce.Do(func() { + file_entity_node_info_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_node_info_proto_rawDescData) + }) + return file_entity_node_info_proto_rawDescData +} + +var file_entity_node_info_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_entity_node_info_proto_goTypes = []interface{}{ + (*NodeInfo)(nil), // 0: grpc.NodeInfo +} +var file_entity_node_info_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_node_info_proto_init() } +func file_entity_node_info_proto_init() { + if File_entity_node_info_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_entity_node_info_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_node_info_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_node_info_proto_goTypes, + DependencyIndexes: file_entity_node_info_proto_depIdxs, + MessageInfos: file_entity_node_info_proto_msgTypes, + }.Build() + File_entity_node_info_proto = out.File + file_entity_node_info_proto_rawDesc = nil + file_entity_node_info_proto_goTypes = nil + file_entity_node_info_proto_depIdxs = nil +} diff --git a/grpc/node_service.pb.go b/grpc/node_service.pb.go new file mode 100644 index 000000000..ebed25592 --- /dev/null +++ b/grpc/node_service.pb.go @@ -0,0 +1,104 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/node_service.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_node_service_proto protoreflect.FileDescriptor + +var file_services_node_service_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, + 0x72, 0x70, 0x63, 0x1a, 0x14, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x1a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xfa, 0x01, + 0x0a, 0x0b, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x2b, 0x0a, + 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x30, 0x0a, 0x0d, 0x53, 0x65, + 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x0d, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x27, 0x0a, 0x04, + 0x50, 0x69, 0x6e, 0x67, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, + 0x62, 0x65, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x2e, 0x0a, 0x0b, 0x55, 0x6e, + 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, + 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_services_node_service_proto_goTypes = []interface{}{ + (*Request)(nil), // 0: grpc.Request + (*Response)(nil), // 1: grpc.Response + (*StreamMessage)(nil), // 2: grpc.StreamMessage +} +var file_services_node_service_proto_depIdxs = []int32{ + 0, // 0: grpc.NodeService.Register:input_type -> grpc.Request + 0, // 1: grpc.NodeService.SendHeartbeat:input_type -> grpc.Request + 0, // 2: grpc.NodeService.Ping:input_type -> grpc.Request + 0, // 3: grpc.NodeService.Subscribe:input_type -> grpc.Request + 0, // 4: grpc.NodeService.Unsubscribe:input_type -> grpc.Request + 1, // 5: grpc.NodeService.Register:output_type -> grpc.Response + 1, // 6: grpc.NodeService.SendHeartbeat:output_type -> grpc.Response + 1, // 7: grpc.NodeService.Ping:output_type -> grpc.Response + 2, // 8: grpc.NodeService.Subscribe:output_type -> grpc.StreamMessage + 1, // 9: grpc.NodeService.Unsubscribe:output_type -> grpc.Response + 5, // [5:10] is the sub-list for method output_type + 0, // [0:5] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_node_service_proto_init() } +func file_services_node_service_proto_init() { + if File_services_node_service_proto != nil { + return + } + file_entity_request_proto_init() + file_entity_response_proto_init() + file_entity_stream_message_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_node_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_node_service_proto_goTypes, + DependencyIndexes: file_services_node_service_proto_depIdxs, + }.Build() + File_services_node_service_proto = out.File + file_services_node_service_proto_rawDesc = nil + file_services_node_service_proto_goTypes = nil + file_services_node_service_proto_depIdxs = nil +} diff --git a/grpc/node_service_grpc.pb.go b/grpc/node_service_grpc.pb.go new file mode 100644 index 000000000..1e0372f10 --- /dev/null +++ b/grpc/node_service_grpc.pb.go @@ -0,0 +1,277 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/node_service.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// NodeServiceClient is the client API for NodeService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type NodeServiceClient interface { + Register(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + SendHeartbeat(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Ping(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Subscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (NodeService_SubscribeClient, error) + Unsubscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) +} + +type nodeServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewNodeServiceClient(cc grpc.ClientConnInterface) NodeServiceClient { + return &nodeServiceClient{cc} +} + +func (c *nodeServiceClient) Register(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.NodeService/Register", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *nodeServiceClient) SendHeartbeat(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.NodeService/SendHeartbeat", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *nodeServiceClient) Ping(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.NodeService/Ping", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *nodeServiceClient) Subscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (NodeService_SubscribeClient, error) { + stream, err := c.cc.NewStream(ctx, &NodeService_ServiceDesc.Streams[0], "/grpc.NodeService/Subscribe", opts...) + if err != nil { + return nil, err + } + x := &nodeServiceSubscribeClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type NodeService_SubscribeClient interface { + Recv() (*StreamMessage, error) + grpc.ClientStream +} + +type nodeServiceSubscribeClient struct { + grpc.ClientStream +} + +func (x *nodeServiceSubscribeClient) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *nodeServiceClient) Unsubscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.NodeService/Unsubscribe", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// NodeServiceServer is the server API for NodeService service. +// All implementations must embed UnimplementedNodeServiceServer +// for forward compatibility +type NodeServiceServer interface { + Register(context.Context, *Request) (*Response, error) + SendHeartbeat(context.Context, *Request) (*Response, error) + Ping(context.Context, *Request) (*Response, error) + Subscribe(*Request, NodeService_SubscribeServer) error + Unsubscribe(context.Context, *Request) (*Response, error) + mustEmbedUnimplementedNodeServiceServer() +} + +// UnimplementedNodeServiceServer must be embedded to have forward compatible implementations. +type UnimplementedNodeServiceServer struct { +} + +func (UnimplementedNodeServiceServer) Register(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Register not implemented") +} +func (UnimplementedNodeServiceServer) SendHeartbeat(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method SendHeartbeat not implemented") +} +func (UnimplementedNodeServiceServer) Ping(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented") +} +func (UnimplementedNodeServiceServer) Subscribe(*Request, NodeService_SubscribeServer) error { + return status.Errorf(codes.Unimplemented, "method Subscribe not implemented") +} +func (UnimplementedNodeServiceServer) Unsubscribe(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Unsubscribe not implemented") +} +func (UnimplementedNodeServiceServer) mustEmbedUnimplementedNodeServiceServer() {} + +// UnsafeNodeServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to NodeServiceServer will +// result in compilation errors. +type UnsafeNodeServiceServer interface { + mustEmbedUnimplementedNodeServiceServer() +} + +func RegisterNodeServiceServer(s grpc.ServiceRegistrar, srv NodeServiceServer) { + s.RegisterService(&NodeService_ServiceDesc, srv) +} + +func _NodeService_Register_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServiceServer).Register(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.NodeService/Register", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServiceServer).Register(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _NodeService_SendHeartbeat_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServiceServer).SendHeartbeat(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.NodeService/SendHeartbeat", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServiceServer).SendHeartbeat(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _NodeService_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServiceServer).Ping(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.NodeService/Ping", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServiceServer).Ping(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _NodeService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(Request) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(NodeServiceServer).Subscribe(m, &nodeServiceSubscribeServer{stream}) +} + +type NodeService_SubscribeServer interface { + Send(*StreamMessage) error + grpc.ServerStream +} + +type nodeServiceSubscribeServer struct { + grpc.ServerStream +} + +func (x *nodeServiceSubscribeServer) Send(m *StreamMessage) error { + return x.ServerStream.SendMsg(m) +} + +func _NodeService_Unsubscribe_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServiceServer).Unsubscribe(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.NodeService/Unsubscribe", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServiceServer).Unsubscribe(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +// NodeService_ServiceDesc is the grpc.ServiceDesc for NodeService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var NodeService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.NodeService", + HandlerType: (*NodeServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Register", + Handler: _NodeService_Register_Handler, + }, + { + MethodName: "SendHeartbeat", + Handler: _NodeService_SendHeartbeat_Handler, + }, + { + MethodName: "Ping", + Handler: _NodeService_Ping_Handler, + }, + { + MethodName: "Unsubscribe", + Handler: _NodeService_Unsubscribe_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Subscribe", + Handler: _NodeService_Subscribe_Handler, + ServerStreams: true, + }, + }, + Metadata: "services/node_service.proto", +} diff --git a/grpc/plugin_request.pb.go b/grpc/plugin_request.pb.go new file mode 100644 index 000000000..3db72ec14 --- /dev/null +++ b/grpc/plugin_request.pb.go @@ -0,0 +1,166 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/plugin_request.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type PluginRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + NodeKey string `protobuf:"bytes,2,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Data []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` +} + +func (x *PluginRequest) Reset() { + *x = PluginRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_plugin_request_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PluginRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PluginRequest) ProtoMessage() {} + +func (x *PluginRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_plugin_request_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PluginRequest.ProtoReflect.Descriptor instead. +func (*PluginRequest) Descriptor() ([]byte, []int) { + return file_entity_plugin_request_proto_rawDescGZIP(), []int{0} +} + +func (x *PluginRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *PluginRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *PluginRequest) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +var File_entity_plugin_request_proto protoreflect.FileDescriptor + +var file_entity_plugin_request_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x5f, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, + 0x72, 0x70, 0x63, 0x22, 0x52, 0x0a, 0x0d, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, + 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, + 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, + 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_plugin_request_proto_rawDescOnce sync.Once + file_entity_plugin_request_proto_rawDescData = file_entity_plugin_request_proto_rawDesc +) + +func file_entity_plugin_request_proto_rawDescGZIP() []byte { + file_entity_plugin_request_proto_rawDescOnce.Do(func() { + file_entity_plugin_request_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_plugin_request_proto_rawDescData) + }) + return file_entity_plugin_request_proto_rawDescData +} + +var file_entity_plugin_request_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_entity_plugin_request_proto_goTypes = []interface{}{ + (*PluginRequest)(nil), // 0: grpc.PluginRequest +} +var file_entity_plugin_request_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_plugin_request_proto_init() } +func file_entity_plugin_request_proto_init() { + if File_entity_plugin_request_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_entity_plugin_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PluginRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_plugin_request_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_plugin_request_proto_goTypes, + DependencyIndexes: file_entity_plugin_request_proto_depIdxs, + MessageInfos: file_entity_plugin_request_proto_msgTypes, + }.Build() + File_entity_plugin_request_proto = out.File + file_entity_plugin_request_proto_rawDesc = nil + file_entity_plugin_request_proto_goTypes = nil + file_entity_plugin_request_proto_depIdxs = nil +} diff --git a/grpc/plugin_service.pb.go b/grpc/plugin_service.pb.go new file mode 100644 index 000000000..d8974eefc --- /dev/null +++ b/grpc/plugin_service.pb.go @@ -0,0 +1,96 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/plugin_service.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_plugin_service_proto protoreflect.FileDescriptor + +var file_services_plugin_service_proto_rawDesc = []byte{ + 0x0a, 0x1d, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, + 0x6e, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x70, 0x6c, + 0x75, 0x67, 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xb5, 0x01, 0x0a, 0x0d, 0x50, 0x6c, 0x75, 0x67, 0x69, + 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x31, 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, + 0x73, 0x74, 0x65, 0x72, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x6c, 0x75, 0x67, + 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x09, 0x53, + 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x36, 0x0a, 0x04, 0x50, 0x6f, 0x6c, 0x6c, 0x12, 0x13, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x08, + 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_services_plugin_service_proto_goTypes = []interface{}{ + (*PluginRequest)(nil), // 0: grpc.PluginRequest + (*StreamMessage)(nil), // 1: grpc.StreamMessage + (*Response)(nil), // 2: grpc.Response +} +var file_services_plugin_service_proto_depIdxs = []int32{ + 0, // 0: grpc.PluginService.Register:input_type -> grpc.PluginRequest + 0, // 1: grpc.PluginService.Subscribe:input_type -> grpc.PluginRequest + 1, // 2: grpc.PluginService.Poll:input_type -> grpc.StreamMessage + 2, // 3: grpc.PluginService.Register:output_type -> grpc.Response + 1, // 4: grpc.PluginService.Subscribe:output_type -> grpc.StreamMessage + 1, // 5: grpc.PluginService.Poll:output_type -> grpc.StreamMessage + 3, // [3:6] is the sub-list for method output_type + 0, // [0:3] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_plugin_service_proto_init() } +func file_services_plugin_service_proto_init() { + if File_services_plugin_service_proto != nil { + return + } + file_entity_plugin_request_proto_init() + file_entity_response_proto_init() + file_entity_stream_message_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_plugin_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_plugin_service_proto_goTypes, + DependencyIndexes: file_services_plugin_service_proto_depIdxs, + }.Build() + File_services_plugin_service_proto = out.File + file_services_plugin_service_proto_rawDesc = nil + file_services_plugin_service_proto_goTypes = nil + file_services_plugin_service_proto_depIdxs = nil +} diff --git a/grpc/plugin_service_grpc.pb.go b/grpc/plugin_service_grpc.pb.go new file mode 100644 index 000000000..06b93262c --- /dev/null +++ b/grpc/plugin_service_grpc.pb.go @@ -0,0 +1,237 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/plugin_service.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// PluginServiceClient is the client API for PluginService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type PluginServiceClient interface { + Register(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (*Response, error) + Subscribe(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (PluginService_SubscribeClient, error) + Poll(ctx context.Context, opts ...grpc.CallOption) (PluginService_PollClient, error) +} + +type pluginServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewPluginServiceClient(cc grpc.ClientConnInterface) PluginServiceClient { + return &pluginServiceClient{cc} +} + +func (c *pluginServiceClient) Register(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.PluginService/Register", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *pluginServiceClient) Subscribe(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (PluginService_SubscribeClient, error) { + stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[0], "/grpc.PluginService/Subscribe", opts...) + if err != nil { + return nil, err + } + x := &pluginServiceSubscribeClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type PluginService_SubscribeClient interface { + Recv() (*StreamMessage, error) + grpc.ClientStream +} + +type pluginServiceSubscribeClient struct { + grpc.ClientStream +} + +func (x *pluginServiceSubscribeClient) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *pluginServiceClient) Poll(ctx context.Context, opts ...grpc.CallOption) (PluginService_PollClient, error) { + stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[1], "/grpc.PluginService/Poll", opts...) + if err != nil { + return nil, err + } + x := &pluginServicePollClient{stream} + return x, nil +} + +type PluginService_PollClient interface { + Send(*StreamMessage) error + Recv() (*StreamMessage, error) + grpc.ClientStream +} + +type pluginServicePollClient struct { + grpc.ClientStream +} + +func (x *pluginServicePollClient) Send(m *StreamMessage) error { + return x.ClientStream.SendMsg(m) +} + +func (x *pluginServicePollClient) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// PluginServiceServer is the server API for PluginService service. +// All implementations must embed UnimplementedPluginServiceServer +// for forward compatibility +type PluginServiceServer interface { + Register(context.Context, *PluginRequest) (*Response, error) + Subscribe(*PluginRequest, PluginService_SubscribeServer) error + Poll(PluginService_PollServer) error + mustEmbedUnimplementedPluginServiceServer() +} + +// UnimplementedPluginServiceServer must be embedded to have forward compatible implementations. +type UnimplementedPluginServiceServer struct { +} + +func (UnimplementedPluginServiceServer) Register(context.Context, *PluginRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Register not implemented") +} +func (UnimplementedPluginServiceServer) Subscribe(*PluginRequest, PluginService_SubscribeServer) error { + return status.Errorf(codes.Unimplemented, "method Subscribe not implemented") +} +func (UnimplementedPluginServiceServer) Poll(PluginService_PollServer) error { + return status.Errorf(codes.Unimplemented, "method Poll not implemented") +} +func (UnimplementedPluginServiceServer) mustEmbedUnimplementedPluginServiceServer() {} + +// UnsafePluginServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to PluginServiceServer will +// result in compilation errors. +type UnsafePluginServiceServer interface { + mustEmbedUnimplementedPluginServiceServer() +} + +func RegisterPluginServiceServer(s grpc.ServiceRegistrar, srv PluginServiceServer) { + s.RegisterService(&PluginService_ServiceDesc, srv) +} + +func _PluginService_Register_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PluginRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PluginServiceServer).Register(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.PluginService/Register", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PluginServiceServer).Register(ctx, req.(*PluginRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _PluginService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(PluginRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(PluginServiceServer).Subscribe(m, &pluginServiceSubscribeServer{stream}) +} + +type PluginService_SubscribeServer interface { + Send(*StreamMessage) error + grpc.ServerStream +} + +type pluginServiceSubscribeServer struct { + grpc.ServerStream +} + +func (x *pluginServiceSubscribeServer) Send(m *StreamMessage) error { + return x.ServerStream.SendMsg(m) +} + +func _PluginService_Poll_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(PluginServiceServer).Poll(&pluginServicePollServer{stream}) +} + +type PluginService_PollServer interface { + Send(*StreamMessage) error + Recv() (*StreamMessage, error) + grpc.ServerStream +} + +type pluginServicePollServer struct { + grpc.ServerStream +} + +func (x *pluginServicePollServer) Send(m *StreamMessage) error { + return x.ServerStream.SendMsg(m) +} + +func (x *pluginServicePollServer) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// PluginService_ServiceDesc is the grpc.ServiceDesc for PluginService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var PluginService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.PluginService", + HandlerType: (*PluginServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Register", + Handler: _PluginService_Register_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Subscribe", + Handler: _PluginService_Subscribe_Handler, + ServerStreams: true, + }, + { + StreamName: "Poll", + Handler: _PluginService_Poll_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "services/plugin_service.proto", +} diff --git a/grpc/proto/entity/dependencies_service_v2_request.proto b/grpc/proto/entity/dependencies_service_v2_request.proto new file mode 100644 index 000000000..c7cfa79d5 --- /dev/null +++ b/grpc/proto/entity/dependencies_service_v2_request.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message Dependency { + string name = 1; + string version = 2; +} + +message DependenciesServiceV2ConnectRequest { + string node_key = 1; +} + +message DependenciesServiceV2SyncRequest { + string node_key = 1; + string lang = 2; + repeated Dependency dependencies = 3; +} + +message DependenciesServiceV2InstallRequest { + string node_key = 1; + string lang = 2; + repeated Dependency dependencies = 3; + string proxy = 4; +} + +message DependenciesServiceV2UninstallRequest { + string node_key = 1; + string lang = 2; + repeated Dependency dependencies = 3; +} \ No newline at end of file diff --git a/grpc/proto/entity/model_service_v2_request.proto b/grpc/proto/entity/model_service_v2_request.proto new file mode 100644 index 000000000..f2933ca77 --- /dev/null +++ b/grpc/proto/entity/model_service_v2_request.proto @@ -0,0 +1,95 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message ModelServiceV2GetByIdRequest { + string node_key = 1; + string model_type = 2; + string id = 3; +} + +message ModelServiceV2GetOneRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; + bytes find_options = 4; +} + +message ModelServiceV2GetManyRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; + bytes find_options = 4; +} + +message ModelServiceV2DeleteByIdRequest { + string node_key = 1; + string model_type = 2; + string id = 3; +} + +message ModelServiceV2DeleteOneRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; +} + +message ModelServiceV2DeleteManyRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; +} + +message ModelServiceV2UpdateByIdRequest { + string node_key = 1; + string model_type = 2; + string id = 3; + bytes update = 4; +} + +message ModelServiceV2UpdateOneRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; + bytes update = 4; +} + +message ModelServiceV2UpdateManyRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; + bytes update = 4; +} + +message ModelServiceV2ReplaceByIdRequest { + string node_key = 1; + string model_type = 2; + string id = 3; + bytes model = 4; +} + +message ModelServiceV2ReplaceOneRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; + bytes model = 4; +} + +message ModelServiceV2InsertOneRequest { + string node_key = 1; + string model_type = 2; + bytes model = 3; +} + +message ModelServiceV2InsertManyRequest { + string node_key = 1; + string model_type = 2; + bytes models = 3; +} + +message ModelServiceV2CountRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; +} diff --git a/grpc/proto/entity/node_info.proto b/grpc/proto/entity/node_info.proto new file mode 100644 index 000000000..e289d29d3 --- /dev/null +++ b/grpc/proto/entity/node_info.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message NodeInfo { + string key = 1; + bool is_master = 2; +} diff --git a/grpc/proto/entity/plugin_request.proto b/grpc/proto/entity/plugin_request.proto new file mode 100644 index 000000000..9359028ee --- /dev/null +++ b/grpc/proto/entity/plugin_request.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message PluginRequest { + string name = 1; + string node_key = 2; + bytes data = 3; +} diff --git a/grpc/proto/entity/request.proto b/grpc/proto/entity/request.proto new file mode 100644 index 000000000..78b88e7d4 --- /dev/null +++ b/grpc/proto/entity/request.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message Request { + string node_key = 1; + bytes data = 2; +} diff --git a/grpc/proto/entity/response.proto b/grpc/proto/entity/response.proto new file mode 100644 index 000000000..9145a1b03 --- /dev/null +++ b/grpc/proto/entity/response.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +import "entity/response_code.proto"; + +package grpc; +option go_package = ".;grpc"; + +message Response { + ResponseCode code = 1; + string message = 2; + bytes data = 3; + string error = 4; + int64 total = 5; +} diff --git a/grpc/proto/entity/response_code.proto b/grpc/proto/entity/response_code.proto new file mode 100644 index 000000000..cb7398fce --- /dev/null +++ b/grpc/proto/entity/response_code.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +option go_package = ".;grpc"; + +enum ResponseCode { + OK = 0; + ERROR = 1; +} \ No newline at end of file diff --git a/grpc/proto/entity/stream_message.proto b/grpc/proto/entity/stream_message.proto new file mode 100644 index 000000000..44cd7af43 --- /dev/null +++ b/grpc/proto/entity/stream_message.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +import "entity/stream_message_code.proto"; + +package grpc; +option go_package = ".;grpc"; + +message StreamMessage { + StreamMessageCode code = 1; + string node_key = 2; + string key = 3; + string from = 4; + string to = 5; + bytes data = 6; + string error = 7; +} diff --git a/grpc/proto/entity/stream_message_code.proto b/grpc/proto/entity/stream_message_code.proto new file mode 100644 index 000000000..f2c418827 --- /dev/null +++ b/grpc/proto/entity/stream_message_code.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +enum StreamMessageCode { + // ping worker nodes to check their health + PING = 0; + // ask worker node(s) to run task with given id + RUN_TASK = 1; + // ask worker node(s) to cancel task with given id + CANCEL_TASK = 2; + // insert data + INSERT_DATA = 3; + // insert logs + INSERT_LOGS = 4; + // send event + SEND_EVENT = 5; + // install plugin + INSTALL_PLUGIN = 6; + // uninstall plugin + UNINSTALL_PLUGIN = 7; + // start plugin + START_PLUGIN = 8; + // stop plugin + STOP_PLUGIN = 9; + // connect + CONNECT = 10; + // disconnect + DISCONNECT = 11; + // send + SEND = 12; +} diff --git a/grpc/proto/entity/stream_message_data_task.proto b/grpc/proto/entity/stream_message_data_task.proto new file mode 100644 index 000000000..b6c082d94 --- /dev/null +++ b/grpc/proto/entity/stream_message_data_task.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message StreamMessageDataTask { + string task_id = 1; + string data = 2; +} diff --git a/grpc/proto/models/node.proto b/grpc/proto/models/node.proto new file mode 100644 index 000000000..aa5312cc7 --- /dev/null +++ b/grpc/proto/models/node.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message Node { + string _id = 1; + string name = 2; + string ip = 3; + string port = 5; + string mac = 6; + string hostname = 7; + string description = 8; + string key = 9; + bool is_master = 11; + string update_ts = 12; + string create_ts = 13; + int64 update_ts_unix = 14; +} diff --git a/grpc/proto/models/task.proto b/grpc/proto/models/task.proto new file mode 100644 index 000000000..52ea7056d --- /dev/null +++ b/grpc/proto/models/task.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message Task { + string _id = 1; + string spider_id = 2; + string status = 5; + string node_id = 6; + string cmd = 8; + string param = 9; + string error = 10; + int32 pid = 16; + string run_type = 17; + string schedule_id = 18; + string type = 19; +} \ No newline at end of file diff --git a/grpc/proto/services/dependency_service_v2.proto b/grpc/proto/services/dependency_service_v2.proto new file mode 100644 index 000000000..dea310d64 --- /dev/null +++ b/grpc/proto/services/dependency_service_v2.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +import "entity/dependencies_service_v2_request.proto"; +import "entity/response.proto"; + +package grpc; +option go_package = ".;grpc"; + +service DependencyServiceV2 { + rpc Connect(stream DependenciesServiceV2ConnectRequest) returns (Response){}; + rpc Sync(DependenciesServiceV2SyncRequest) returns (Response){}; + rpc Install(stream DependenciesServiceV2InstallRequest) returns (stream Response){}; + rpc UninstallDependencies(stream DependenciesServiceV2UninstallRequest) returns (stream Response){}; +} diff --git a/grpc/proto/services/message_service.proto b/grpc/proto/services/message_service.proto new file mode 100644 index 000000000..412074035 --- /dev/null +++ b/grpc/proto/services/message_service.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +import "entity/stream_message.proto"; + +package grpc; +option go_package = ".;grpc"; + +service MessageService { + rpc Connect(stream StreamMessage) returns (stream StreamMessage){}; +} diff --git a/grpc/proto/services/model_base_service.proto b/grpc/proto/services/model_base_service.proto new file mode 100644 index 000000000..0836659a9 --- /dev/null +++ b/grpc/proto/services/model_base_service.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +import "entity/request.proto"; +import "entity/response.proto"; + +package grpc; +option go_package = ".;grpc"; + +service ModelBaseService { + rpc GetById(Request) returns (Response){}; + rpc Get(Request) returns (Response){}; + rpc GetList(Request) returns (Response){}; + rpc DeleteById(Request) returns (Response){}; + rpc Delete(Request) returns (Response){}; + rpc DeleteList(Request) returns (Response){}; + rpc ForceDeleteList(Request) returns (Response){}; + rpc UpdateById(Request) returns (Response){}; + rpc Update(Request) returns (Response){}; + rpc UpdateDoc(Request) returns (Response){}; + rpc Insert(Request) returns (Response){}; + rpc Count(Request) returns (Response){}; +} diff --git a/grpc/proto/services/model_base_service_v2.proto b/grpc/proto/services/model_base_service_v2.proto new file mode 100644 index 000000000..e0b309f75 --- /dev/null +++ b/grpc/proto/services/model_base_service_v2.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +import "entity/model_service_v2_request.proto"; +import "entity/response.proto"; + +package grpc; +option go_package = ".;grpc"; + +service ModelBaseServiceV2 { + rpc GetById(ModelServiceV2GetByIdRequest) returns (Response){}; + rpc GetOne(ModelServiceV2GetOneRequest) returns (Response){}; + rpc GetMany(ModelServiceV2GetManyRequest) returns (Response){}; + rpc DeleteById(ModelServiceV2DeleteByIdRequest) returns (Response){}; + rpc DeleteOne(ModelServiceV2DeleteOneRequest) returns (Response){}; + rpc DeleteMany(ModelServiceV2DeleteManyRequest) returns (Response){}; + rpc UpdateById(ModelServiceV2UpdateByIdRequest) returns (Response){}; + rpc UpdateOne(ModelServiceV2UpdateOneRequest) returns (Response){}; + rpc UpdateMany(ModelServiceV2UpdateManyRequest) returns (Response){}; + rpc ReplaceById(ModelServiceV2ReplaceByIdRequest) returns (Response){}; + rpc ReplaceOne(ModelServiceV2ReplaceOneRequest) returns (Response){}; + rpc InsertOne(ModelServiceV2InsertOneRequest) returns (Response){}; + rpc InsertMany(ModelServiceV2InsertManyRequest) returns (Response){}; + rpc Count(ModelServiceV2CountRequest) returns (Response){}; +} diff --git a/grpc/proto/services/model_delegate.proto b/grpc/proto/services/model_delegate.proto new file mode 100644 index 000000000..1b60477c7 --- /dev/null +++ b/grpc/proto/services/model_delegate.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +import "entity/request.proto"; +import "entity/response.proto"; + +package grpc; +option go_package = ".;grpc"; + +service ModelDelegate { + rpc Do(Request) returns (Response){}; +} diff --git a/grpc/proto/services/node_service.proto b/grpc/proto/services/node_service.proto new file mode 100644 index 000000000..4b43d083b --- /dev/null +++ b/grpc/proto/services/node_service.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +import "entity/request.proto"; +import "entity/response.proto"; +import "entity/stream_message.proto"; + +package grpc; +option go_package = ".;grpc"; + +service NodeService { + rpc Register(Request) returns (Response){}; + rpc SendHeartbeat(Request) returns (Response){}; + rpc Ping(Request) returns (Response){}; + rpc Subscribe(Request) returns (stream StreamMessage){}; + rpc Unsubscribe(Request) returns (Response){}; +} diff --git a/grpc/proto/services/plugin_service.proto b/grpc/proto/services/plugin_service.proto new file mode 100644 index 000000000..5e41977e5 --- /dev/null +++ b/grpc/proto/services/plugin_service.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +import "entity/plugin_request.proto"; +import "entity/response.proto"; +import "entity/stream_message.proto"; + +package grpc; +option go_package = ".;grpc"; + +service PluginService { + rpc Register(PluginRequest) returns (Response){}; + rpc Subscribe(PluginRequest) returns (stream StreamMessage){}; + rpc Poll(stream StreamMessage) returns (stream StreamMessage){}; +} diff --git a/grpc/proto/services/task_service.proto b/grpc/proto/services/task_service.proto new file mode 100644 index 000000000..f004e5651 --- /dev/null +++ b/grpc/proto/services/task_service.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +import "entity/request.proto"; +import "entity/response.proto"; +import "entity/stream_message.proto"; + +package grpc; +option go_package = ".;grpc"; + +service TaskService { + rpc Subscribe(stream StreamMessage) returns (Response){}; + rpc Fetch(Request) returns (Response){}; + rpc SendNotification(Request) returns (Response){}; +} diff --git a/grpc/request.pb.go b/grpc/request.pb.go new file mode 100644 index 000000000..3b5415549 --- /dev/null +++ b/grpc/request.pb.go @@ -0,0 +1,156 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/request.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type Request struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` +} + +func (x *Request) Reset() { + *x = Request{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_request_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Request) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Request) ProtoMessage() {} + +func (x *Request) ProtoReflect() protoreflect.Message { + mi := &file_entity_request_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Request.ProtoReflect.Descriptor instead. +func (*Request) Descriptor() ([]byte, []int) { + return file_entity_request_proto_rawDescGZIP(), []int{0} +} + +func (x *Request) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *Request) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +var File_entity_request_proto protoreflect.FileDescriptor + +var file_entity_request_proto_rawDesc = []byte{ + 0x0a, 0x14, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0x38, 0x0a, 0x07, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, + 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_request_proto_rawDescOnce sync.Once + file_entity_request_proto_rawDescData = file_entity_request_proto_rawDesc +) + +func file_entity_request_proto_rawDescGZIP() []byte { + file_entity_request_proto_rawDescOnce.Do(func() { + file_entity_request_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_request_proto_rawDescData) + }) + return file_entity_request_proto_rawDescData +} + +var file_entity_request_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_entity_request_proto_goTypes = []interface{}{ + (*Request)(nil), // 0: grpc.Request +} +var file_entity_request_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_request_proto_init() } +func file_entity_request_proto_init() { + if File_entity_request_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_entity_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Request); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_request_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_request_proto_goTypes, + DependencyIndexes: file_entity_request_proto_depIdxs, + MessageInfos: file_entity_request_proto_msgTypes, + }.Build() + File_entity_request_proto = out.File + file_entity_request_proto_rawDesc = nil + file_entity_request_proto_goTypes = nil + file_entity_request_proto_depIdxs = nil +} diff --git a/grpc/response.pb.go b/grpc/response.pb.go new file mode 100644 index 000000000..ad32f8690 --- /dev/null +++ b/grpc/response.pb.go @@ -0,0 +1,190 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/response.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type Response struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Code ResponseCode `protobuf:"varint,1,opt,name=code,proto3,enum=ResponseCode" json:"code,omitempty"` + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + Data []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` + Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"` + Total int64 `protobuf:"varint,5,opt,name=total,proto3" json:"total,omitempty"` +} + +func (x *Response) Reset() { + *x = Response{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_response_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Response) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Response) ProtoMessage() {} + +func (x *Response) ProtoReflect() protoreflect.Message { + mi := &file_entity_response_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Response.ProtoReflect.Descriptor instead. +func (*Response) Descriptor() ([]byte, []int) { + return file_entity_response_proto_rawDescGZIP(), []int{0} +} + +func (x *Response) GetCode() ResponseCode { + if x != nil { + return x.Code + } + return ResponseCode_OK +} + +func (x *Response) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +func (x *Response) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *Response) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *Response) GetTotal() int64 { + if x != nil { + return x.Total + } + return 0 +} + +var File_entity_response_proto protoreflect.FileDescriptor + +var file_entity_response_proto_rawDesc = []byte{ + 0x0a, 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x1a, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x63, + 0x6f, 0x64, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x87, 0x01, 0x0a, 0x08, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43, + 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x14, 0x0a, + 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, + 0x74, 0x61, 0x6c, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_response_proto_rawDescOnce sync.Once + file_entity_response_proto_rawDescData = file_entity_response_proto_rawDesc +) + +func file_entity_response_proto_rawDescGZIP() []byte { + file_entity_response_proto_rawDescOnce.Do(func() { + file_entity_response_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_response_proto_rawDescData) + }) + return file_entity_response_proto_rawDescData +} + +var file_entity_response_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_entity_response_proto_goTypes = []interface{}{ + (*Response)(nil), // 0: grpc.Response + (ResponseCode)(0), // 1: ResponseCode +} +var file_entity_response_proto_depIdxs = []int32{ + 1, // 0: grpc.Response.code:type_name -> ResponseCode + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_entity_response_proto_init() } +func file_entity_response_proto_init() { + if File_entity_response_proto != nil { + return + } + file_entity_response_code_proto_init() + if !protoimpl.UnsafeEnabled { + file_entity_response_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Response); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_response_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_response_proto_goTypes, + DependencyIndexes: file_entity_response_proto_depIdxs, + MessageInfos: file_entity_response_proto_msgTypes, + }.Build() + File_entity_response_proto = out.File + file_entity_response_proto_rawDesc = nil + file_entity_response_proto_goTypes = nil + file_entity_response_proto_depIdxs = nil +} diff --git a/grpc/response_code.pb.go b/grpc/response_code.pb.go new file mode 100644 index 000000000..9c9700a85 --- /dev/null +++ b/grpc/response_code.pb.go @@ -0,0 +1,132 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/response_code.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type ResponseCode int32 + +const ( + ResponseCode_OK ResponseCode = 0 + ResponseCode_ERROR ResponseCode = 1 +) + +// Enum value maps for ResponseCode. +var ( + ResponseCode_name = map[int32]string{ + 0: "OK", + 1: "ERROR", + } + ResponseCode_value = map[string]int32{ + "OK": 0, + "ERROR": 1, + } +) + +func (x ResponseCode) Enum() *ResponseCode { + p := new(ResponseCode) + *p = x + return p +} + +func (x ResponseCode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ResponseCode) Descriptor() protoreflect.EnumDescriptor { + return file_entity_response_code_proto_enumTypes[0].Descriptor() +} + +func (ResponseCode) Type() protoreflect.EnumType { + return &file_entity_response_code_proto_enumTypes[0] +} + +func (x ResponseCode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ResponseCode.Descriptor instead. +func (ResponseCode) EnumDescriptor() ([]byte, []int) { + return file_entity_response_code_proto_rawDescGZIP(), []int{0} +} + +var File_entity_response_code_proto protoreflect.FileDescriptor + +var file_entity_response_code_proto_rawDesc = []byte{ + 0x0a, 0x1a, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2a, 0x21, 0x0a, 0x0c, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x06, 0x0a, 0x02, + 0x4f, 0x4b, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x01, 0x42, + 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, +} + +var ( + file_entity_response_code_proto_rawDescOnce sync.Once + file_entity_response_code_proto_rawDescData = file_entity_response_code_proto_rawDesc +) + +func file_entity_response_code_proto_rawDescGZIP() []byte { + file_entity_response_code_proto_rawDescOnce.Do(func() { + file_entity_response_code_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_response_code_proto_rawDescData) + }) + return file_entity_response_code_proto_rawDescData +} + +var file_entity_response_code_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_entity_response_code_proto_goTypes = []interface{}{ + (ResponseCode)(0), // 0: ResponseCode +} +var file_entity_response_code_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_response_code_proto_init() } +func file_entity_response_code_proto_init() { + if File_entity_response_code_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_response_code_proto_rawDesc, + NumEnums: 1, + NumMessages: 0, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_response_code_proto_goTypes, + DependencyIndexes: file_entity_response_code_proto_depIdxs, + EnumInfos: file_entity_response_code_proto_enumTypes, + }.Build() + File_entity_response_code_proto = out.File + file_entity_response_code_proto_rawDesc = nil + file_entity_response_code_proto_goTypes = nil + file_entity_response_code_proto_depIdxs = nil +} diff --git a/grpc/stream_message.pb.go b/grpc/stream_message.pb.go new file mode 100644 index 000000000..2e563dfc8 --- /dev/null +++ b/grpc/stream_message.pb.go @@ -0,0 +1,210 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/stream_message.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type StreamMessage struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Code StreamMessageCode `protobuf:"varint,1,opt,name=code,proto3,enum=grpc.StreamMessageCode" json:"code,omitempty"` + NodeKey string `protobuf:"bytes,2,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` + From string `protobuf:"bytes,4,opt,name=from,proto3" json:"from,omitempty"` + To string `protobuf:"bytes,5,opt,name=to,proto3" json:"to,omitempty"` + Data []byte `protobuf:"bytes,6,opt,name=data,proto3" json:"data,omitempty"` + Error string `protobuf:"bytes,7,opt,name=error,proto3" json:"error,omitempty"` +} + +func (x *StreamMessage) Reset() { + *x = StreamMessage{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_stream_message_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamMessage) ProtoMessage() {} + +func (x *StreamMessage) ProtoReflect() protoreflect.Message { + mi := &file_entity_stream_message_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamMessage.ProtoReflect.Descriptor instead. +func (*StreamMessage) Descriptor() ([]byte, []int) { + return file_entity_stream_message_proto_rawDescGZIP(), []int{0} +} + +func (x *StreamMessage) GetCode() StreamMessageCode { + if x != nil { + return x.Code + } + return StreamMessageCode_PING +} + +func (x *StreamMessage) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *StreamMessage) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *StreamMessage) GetFrom() string { + if x != nil { + return x.From + } + return "" +} + +func (x *StreamMessage) GetTo() string { + if x != nil { + return x.To + } + return "" +} + +func (x *StreamMessage) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *StreamMessage) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +var File_entity_stream_message_proto protoreflect.FileDescriptor + +var file_entity_stream_message_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, + 0x72, 0x70, 0x63, 0x1a, 0x20, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb7, 0x01, 0x0a, 0x0d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2b, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, + 0x63, 0x6f, 0x64, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x12, 0x0e, 0x0a, 0x02, 0x74, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, + 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, +} + +var ( + file_entity_stream_message_proto_rawDescOnce sync.Once + file_entity_stream_message_proto_rawDescData = file_entity_stream_message_proto_rawDesc +) + +func file_entity_stream_message_proto_rawDescGZIP() []byte { + file_entity_stream_message_proto_rawDescOnce.Do(func() { + file_entity_stream_message_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_stream_message_proto_rawDescData) + }) + return file_entity_stream_message_proto_rawDescData +} + +var file_entity_stream_message_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_entity_stream_message_proto_goTypes = []interface{}{ + (*StreamMessage)(nil), // 0: grpc.StreamMessage + (StreamMessageCode)(0), // 1: grpc.StreamMessageCode +} +var file_entity_stream_message_proto_depIdxs = []int32{ + 1, // 0: grpc.StreamMessage.code:type_name -> grpc.StreamMessageCode + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_entity_stream_message_proto_init() } +func file_entity_stream_message_proto_init() { + if File_entity_stream_message_proto != nil { + return + } + file_entity_stream_message_code_proto_init() + if !protoimpl.UnsafeEnabled { + file_entity_stream_message_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamMessage); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_stream_message_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_stream_message_proto_goTypes, + DependencyIndexes: file_entity_stream_message_proto_depIdxs, + MessageInfos: file_entity_stream_message_proto_msgTypes, + }.Build() + File_entity_stream_message_proto = out.File + file_entity_stream_message_proto_rawDesc = nil + file_entity_stream_message_proto_goTypes = nil + file_entity_stream_message_proto_depIdxs = nil +} diff --git a/grpc/stream_message_code.pb.go b/grpc/stream_message_code.pb.go new file mode 100644 index 000000000..90e1f9f6c --- /dev/null +++ b/grpc/stream_message_code.pb.go @@ -0,0 +1,190 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/stream_message_code.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type StreamMessageCode int32 + +const ( + // ping worker nodes to check their health + StreamMessageCode_PING StreamMessageCode = 0 + // ask worker node(s) to run task with given id + StreamMessageCode_RUN_TASK StreamMessageCode = 1 + // ask worker node(s) to cancel task with given id + StreamMessageCode_CANCEL_TASK StreamMessageCode = 2 + // insert data + StreamMessageCode_INSERT_DATA StreamMessageCode = 3 + // insert logs + StreamMessageCode_INSERT_LOGS StreamMessageCode = 4 + // send event + StreamMessageCode_SEND_EVENT StreamMessageCode = 5 + // install plugin + StreamMessageCode_INSTALL_PLUGIN StreamMessageCode = 6 + // uninstall plugin + StreamMessageCode_UNINSTALL_PLUGIN StreamMessageCode = 7 + // start plugin + StreamMessageCode_START_PLUGIN StreamMessageCode = 8 + // stop plugin + StreamMessageCode_STOP_PLUGIN StreamMessageCode = 9 + // connect + StreamMessageCode_CONNECT StreamMessageCode = 10 + // disconnect + StreamMessageCode_DISCONNECT StreamMessageCode = 11 + // send + StreamMessageCode_SEND StreamMessageCode = 12 +) + +// Enum value maps for StreamMessageCode. +var ( + StreamMessageCode_name = map[int32]string{ + 0: "PING", + 1: "RUN_TASK", + 2: "CANCEL_TASK", + 3: "INSERT_DATA", + 4: "INSERT_LOGS", + 5: "SEND_EVENT", + 6: "INSTALL_PLUGIN", + 7: "UNINSTALL_PLUGIN", + 8: "START_PLUGIN", + 9: "STOP_PLUGIN", + 10: "CONNECT", + 11: "DISCONNECT", + 12: "SEND", + } + StreamMessageCode_value = map[string]int32{ + "PING": 0, + "RUN_TASK": 1, + "CANCEL_TASK": 2, + "INSERT_DATA": 3, + "INSERT_LOGS": 4, + "SEND_EVENT": 5, + "INSTALL_PLUGIN": 6, + "UNINSTALL_PLUGIN": 7, + "START_PLUGIN": 8, + "STOP_PLUGIN": 9, + "CONNECT": 10, + "DISCONNECT": 11, + "SEND": 12, + } +) + +func (x StreamMessageCode) Enum() *StreamMessageCode { + p := new(StreamMessageCode) + *p = x + return p +} + +func (x StreamMessageCode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (StreamMessageCode) Descriptor() protoreflect.EnumDescriptor { + return file_entity_stream_message_code_proto_enumTypes[0].Descriptor() +} + +func (StreamMessageCode) Type() protoreflect.EnumType { + return &file_entity_stream_message_code_proto_enumTypes[0] +} + +func (x StreamMessageCode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use StreamMessageCode.Descriptor instead. +func (StreamMessageCode) EnumDescriptor() ([]byte, []int) { + return file_entity_stream_message_code_proto_rawDescGZIP(), []int{0} +} + +var File_entity_stream_message_code_proto protoreflect.FileDescriptor + +var file_entity_stream_message_code_proto_rawDesc = []byte{ + 0x0a, 0x20, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x2a, 0xe2, 0x01, 0x0a, 0x11, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x08, + 0x0a, 0x04, 0x50, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x55, 0x4e, 0x5f, + 0x54, 0x41, 0x53, 0x4b, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, + 0x5f, 0x54, 0x41, 0x53, 0x4b, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x49, 0x4e, 0x53, 0x45, 0x52, + 0x54, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x10, 0x03, 0x12, 0x0f, 0x0a, 0x0b, 0x49, 0x4e, 0x53, 0x45, + 0x52, 0x54, 0x5f, 0x4c, 0x4f, 0x47, 0x53, 0x10, 0x04, 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x45, 0x4e, + 0x44, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x10, 0x05, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x4e, 0x53, + 0x54, 0x41, 0x4c, 0x4c, 0x5f, 0x50, 0x4c, 0x55, 0x47, 0x49, 0x4e, 0x10, 0x06, 0x12, 0x14, 0x0a, + 0x10, 0x55, 0x4e, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x5f, 0x50, 0x4c, 0x55, 0x47, 0x49, + 0x4e, 0x10, 0x07, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x41, 0x52, 0x54, 0x5f, 0x50, 0x4c, 0x55, + 0x47, 0x49, 0x4e, 0x10, 0x08, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x54, 0x4f, 0x50, 0x5f, 0x50, 0x4c, + 0x55, 0x47, 0x49, 0x4e, 0x10, 0x09, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, + 0x54, 0x10, 0x0a, 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x49, 0x53, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, + 0x54, 0x10, 0x0b, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x45, 0x4e, 0x44, 0x10, 0x0c, 0x42, 0x08, 0x5a, + 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_stream_message_code_proto_rawDescOnce sync.Once + file_entity_stream_message_code_proto_rawDescData = file_entity_stream_message_code_proto_rawDesc +) + +func file_entity_stream_message_code_proto_rawDescGZIP() []byte { + file_entity_stream_message_code_proto_rawDescOnce.Do(func() { + file_entity_stream_message_code_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_stream_message_code_proto_rawDescData) + }) + return file_entity_stream_message_code_proto_rawDescData +} + +var file_entity_stream_message_code_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_entity_stream_message_code_proto_goTypes = []interface{}{ + (StreamMessageCode)(0), // 0: grpc.StreamMessageCode +} +var file_entity_stream_message_code_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_stream_message_code_proto_init() } +func file_entity_stream_message_code_proto_init() { + if File_entity_stream_message_code_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_stream_message_code_proto_rawDesc, + NumEnums: 1, + NumMessages: 0, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_stream_message_code_proto_goTypes, + DependencyIndexes: file_entity_stream_message_code_proto_depIdxs, + EnumInfos: file_entity_stream_message_code_proto_enumTypes, + }.Build() + File_entity_stream_message_code_proto = out.File + file_entity_stream_message_code_proto_rawDesc = nil + file_entity_stream_message_code_proto_goTypes = nil + file_entity_stream_message_code_proto_depIdxs = nil +} diff --git a/grpc/stream_message_data_task.pb.go b/grpc/stream_message_data_task.pb.go new file mode 100644 index 000000000..e2ffac6e7 --- /dev/null +++ b/grpc/stream_message_data_task.pb.go @@ -0,0 +1,158 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/stream_message_data_task.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type StreamMessageDataTask struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TaskId string `protobuf:"bytes,1,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` + Data string `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` +} + +func (x *StreamMessageDataTask) Reset() { + *x = StreamMessageDataTask{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_stream_message_data_task_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamMessageDataTask) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamMessageDataTask) ProtoMessage() {} + +func (x *StreamMessageDataTask) ProtoReflect() protoreflect.Message { + mi := &file_entity_stream_message_data_task_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamMessageDataTask.ProtoReflect.Descriptor instead. +func (*StreamMessageDataTask) Descriptor() ([]byte, []int) { + return file_entity_stream_message_data_task_proto_rawDescGZIP(), []int{0} +} + +func (x *StreamMessageDataTask) GetTaskId() string { + if x != nil { + return x.TaskId + } + return "" +} + +func (x *StreamMessageDataTask) GetData() string { + if x != nil { + return x.Data + } + return "" +} + +var File_entity_stream_message_data_task_proto protoreflect.FileDescriptor + +var file_entity_stream_message_data_task_proto_rawDesc = []byte{ + 0x0a, 0x25, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x74, 0x61, 0x73, + 0x6b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0x44, 0x0a, + 0x15, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x44, 0x61, + 0x74, 0x61, 0x54, 0x61, 0x73, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, + 0x61, 0x74, 0x61, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_stream_message_data_task_proto_rawDescOnce sync.Once + file_entity_stream_message_data_task_proto_rawDescData = file_entity_stream_message_data_task_proto_rawDesc +) + +func file_entity_stream_message_data_task_proto_rawDescGZIP() []byte { + file_entity_stream_message_data_task_proto_rawDescOnce.Do(func() { + file_entity_stream_message_data_task_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_stream_message_data_task_proto_rawDescData) + }) + return file_entity_stream_message_data_task_proto_rawDescData +} + +var file_entity_stream_message_data_task_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_entity_stream_message_data_task_proto_goTypes = []interface{}{ + (*StreamMessageDataTask)(nil), // 0: grpc.StreamMessageDataTask +} +var file_entity_stream_message_data_task_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_stream_message_data_task_proto_init() } +func file_entity_stream_message_data_task_proto_init() { + if File_entity_stream_message_data_task_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_entity_stream_message_data_task_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamMessageDataTask); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_stream_message_data_task_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_stream_message_data_task_proto_goTypes, + DependencyIndexes: file_entity_stream_message_data_task_proto_depIdxs, + MessageInfos: file_entity_stream_message_data_task_proto_msgTypes, + }.Build() + File_entity_stream_message_data_task_proto = out.File + file_entity_stream_message_data_task_proto_rawDesc = nil + file_entity_stream_message_data_task_proto_goTypes = nil + file_entity_stream_message_data_task_proto_depIdxs = nil +} diff --git a/grpc/task.pb.go b/grpc/task.pb.go new file mode 100644 index 000000000..af5aa474b --- /dev/null +++ b/grpc/task.pb.go @@ -0,0 +1,241 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: models/task.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type Task struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + XId string `protobuf:"bytes,1,opt,name=_id,json=Id,proto3" json:"_id,omitempty"` + SpiderId string `protobuf:"bytes,2,opt,name=spider_id,json=spiderId,proto3" json:"spider_id,omitempty"` + Status string `protobuf:"bytes,5,opt,name=status,proto3" json:"status,omitempty"` + NodeId string `protobuf:"bytes,6,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` + Cmd string `protobuf:"bytes,8,opt,name=cmd,proto3" json:"cmd,omitempty"` + Param string `protobuf:"bytes,9,opt,name=param,proto3" json:"param,omitempty"` + Error string `protobuf:"bytes,10,opt,name=error,proto3" json:"error,omitempty"` + Pid int32 `protobuf:"varint,16,opt,name=pid,proto3" json:"pid,omitempty"` + RunType string `protobuf:"bytes,17,opt,name=run_type,json=runType,proto3" json:"run_type,omitempty"` + ScheduleId string `protobuf:"bytes,18,opt,name=schedule_id,json=scheduleId,proto3" json:"schedule_id,omitempty"` + Type string `protobuf:"bytes,19,opt,name=type,proto3" json:"type,omitempty"` +} + +func (x *Task) Reset() { + *x = Task{} + if protoimpl.UnsafeEnabled { + mi := &file_models_task_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Task) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Task) ProtoMessage() {} + +func (x *Task) ProtoReflect() protoreflect.Message { + mi := &file_models_task_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Task.ProtoReflect.Descriptor instead. +func (*Task) Descriptor() ([]byte, []int) { + return file_models_task_proto_rawDescGZIP(), []int{0} +} + +func (x *Task) GetXId() string { + if x != nil { + return x.XId + } + return "" +} + +func (x *Task) GetSpiderId() string { + if x != nil { + return x.SpiderId + } + return "" +} + +func (x *Task) GetStatus() string { + if x != nil { + return x.Status + } + return "" +} + +func (x *Task) GetNodeId() string { + if x != nil { + return x.NodeId + } + return "" +} + +func (x *Task) GetCmd() string { + if x != nil { + return x.Cmd + } + return "" +} + +func (x *Task) GetParam() string { + if x != nil { + return x.Param + } + return "" +} + +func (x *Task) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *Task) GetPid() int32 { + if x != nil { + return x.Pid + } + return 0 +} + +func (x *Task) GetRunType() string { + if x != nil { + return x.RunType + } + return "" +} + +func (x *Task) GetScheduleId() string { + if x != nil { + return x.ScheduleId + } + return "" +} + +func (x *Task) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +var File_models_task_proto protoreflect.FileDescriptor + +var file_models_task_proto_rawDesc = []byte{ + 0x0a, 0x11, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x73, 0x2f, 0x74, 0x61, 0x73, 0x6b, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0x85, 0x02, 0x0a, 0x04, 0x54, 0x61, + 0x73, 0x6b, 0x12, 0x0f, 0x0a, 0x03, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x02, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x70, 0x69, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x70, 0x69, 0x64, 0x65, 0x72, 0x49, 0x64, + 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, + 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, + 0x64, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x6d, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x63, 0x6d, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x18, 0x09, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, + 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x10, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x70, 0x69, + 0x64, 0x12, 0x19, 0x0a, 0x08, 0x72, 0x75, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x11, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x75, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x0b, + 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x12, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x13, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, + 0x65, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var ( + file_models_task_proto_rawDescOnce sync.Once + file_models_task_proto_rawDescData = file_models_task_proto_rawDesc +) + +func file_models_task_proto_rawDescGZIP() []byte { + file_models_task_proto_rawDescOnce.Do(func() { + file_models_task_proto_rawDescData = protoimpl.X.CompressGZIP(file_models_task_proto_rawDescData) + }) + return file_models_task_proto_rawDescData +} + +var file_models_task_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_models_task_proto_goTypes = []interface{}{ + (*Task)(nil), // 0: grpc.Task +} +var file_models_task_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_models_task_proto_init() } +func file_models_task_proto_init() { + if File_models_task_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_models_task_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Task); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_models_task_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_models_task_proto_goTypes, + DependencyIndexes: file_models_task_proto_depIdxs, + MessageInfos: file_models_task_proto_msgTypes, + }.Build() + File_models_task_proto = out.File + file_models_task_proto_rawDesc = nil + file_models_task_proto_goTypes = nil + file_models_task_proto_depIdxs = nil +} diff --git a/grpc/task_service.pb.go b/grpc/task_service.pb.go new file mode 100644 index 000000000..7ea57caa5 --- /dev/null +++ b/grpc/task_service.pb.go @@ -0,0 +1,95 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/task_service.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_task_service_proto protoreflect.FileDescriptor + +var file_services_task_service_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x74, 0x61, 0x73, 0x6b, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, + 0x72, 0x70, 0x63, 0x1a, 0x14, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x1a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xa2, 0x01, + 0x0a, 0x0b, 0x54, 0x61, 0x73, 0x6b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x34, 0x0a, + 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, + 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x28, 0x01, 0x12, 0x28, 0x0a, 0x05, 0x46, 0x65, 0x74, 0x63, 0x68, 0x12, 0x0d, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, + 0x10, 0x53, 0x65, 0x6e, 0x64, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, +} + +var file_services_task_service_proto_goTypes = []interface{}{ + (*StreamMessage)(nil), // 0: grpc.StreamMessage + (*Request)(nil), // 1: grpc.Request + (*Response)(nil), // 2: grpc.Response +} +var file_services_task_service_proto_depIdxs = []int32{ + 0, // 0: grpc.TaskService.Subscribe:input_type -> grpc.StreamMessage + 1, // 1: grpc.TaskService.Fetch:input_type -> grpc.Request + 1, // 2: grpc.TaskService.SendNotification:input_type -> grpc.Request + 2, // 3: grpc.TaskService.Subscribe:output_type -> grpc.Response + 2, // 4: grpc.TaskService.Fetch:output_type -> grpc.Response + 2, // 5: grpc.TaskService.SendNotification:output_type -> grpc.Response + 3, // [3:6] is the sub-list for method output_type + 0, // [0:3] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_task_service_proto_init() } +func file_services_task_service_proto_init() { + if File_services_task_service_proto != nil { + return + } + file_entity_request_proto_init() + file_entity_response_proto_init() + file_entity_stream_message_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_task_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_task_service_proto_goTypes, + DependencyIndexes: file_services_task_service_proto_depIdxs, + }.Build() + File_services_task_service_proto = out.File + file_services_task_service_proto_rawDesc = nil + file_services_task_service_proto_goTypes = nil + file_services_task_service_proto_depIdxs = nil +} diff --git a/grpc/task_service_grpc.pb.go b/grpc/task_service_grpc.pb.go new file mode 100644 index 000000000..6afa12265 --- /dev/null +++ b/grpc/task_service_grpc.pb.go @@ -0,0 +1,212 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/task_service.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// TaskServiceClient is the client API for TaskService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type TaskServiceClient interface { + Subscribe(ctx context.Context, opts ...grpc.CallOption) (TaskService_SubscribeClient, error) + Fetch(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + SendNotification(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) +} + +type taskServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewTaskServiceClient(cc grpc.ClientConnInterface) TaskServiceClient { + return &taskServiceClient{cc} +} + +func (c *taskServiceClient) Subscribe(ctx context.Context, opts ...grpc.CallOption) (TaskService_SubscribeClient, error) { + stream, err := c.cc.NewStream(ctx, &TaskService_ServiceDesc.Streams[0], "/grpc.TaskService/Subscribe", opts...) + if err != nil { + return nil, err + } + x := &taskServiceSubscribeClient{stream} + return x, nil +} + +type TaskService_SubscribeClient interface { + Send(*StreamMessage) error + CloseAndRecv() (*Response, error) + grpc.ClientStream +} + +type taskServiceSubscribeClient struct { + grpc.ClientStream +} + +func (x *taskServiceSubscribeClient) Send(m *StreamMessage) error { + return x.ClientStream.SendMsg(m) +} + +func (x *taskServiceSubscribeClient) CloseAndRecv() (*Response, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(Response) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *taskServiceClient) Fetch(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.TaskService/Fetch", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *taskServiceClient) SendNotification(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.TaskService/SendNotification", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// TaskServiceServer is the server API for TaskService service. +// All implementations must embed UnimplementedTaskServiceServer +// for forward compatibility +type TaskServiceServer interface { + Subscribe(TaskService_SubscribeServer) error + Fetch(context.Context, *Request) (*Response, error) + SendNotification(context.Context, *Request) (*Response, error) + mustEmbedUnimplementedTaskServiceServer() +} + +// UnimplementedTaskServiceServer must be embedded to have forward compatible implementations. +type UnimplementedTaskServiceServer struct { +} + +func (UnimplementedTaskServiceServer) Subscribe(TaskService_SubscribeServer) error { + return status.Errorf(codes.Unimplemented, "method Subscribe not implemented") +} +func (UnimplementedTaskServiceServer) Fetch(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Fetch not implemented") +} +func (UnimplementedTaskServiceServer) SendNotification(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method SendNotification not implemented") +} +func (UnimplementedTaskServiceServer) mustEmbedUnimplementedTaskServiceServer() {} + +// UnsafeTaskServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to TaskServiceServer will +// result in compilation errors. +type UnsafeTaskServiceServer interface { + mustEmbedUnimplementedTaskServiceServer() +} + +func RegisterTaskServiceServer(s grpc.ServiceRegistrar, srv TaskServiceServer) { + s.RegisterService(&TaskService_ServiceDesc, srv) +} + +func _TaskService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(TaskServiceServer).Subscribe(&taskServiceSubscribeServer{stream}) +} + +type TaskService_SubscribeServer interface { + SendAndClose(*Response) error + Recv() (*StreamMessage, error) + grpc.ServerStream +} + +type taskServiceSubscribeServer struct { + grpc.ServerStream +} + +func (x *taskServiceSubscribeServer) SendAndClose(m *Response) error { + return x.ServerStream.SendMsg(m) +} + +func (x *taskServiceSubscribeServer) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _TaskService_Fetch_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TaskServiceServer).Fetch(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.TaskService/Fetch", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TaskServiceServer).Fetch(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _TaskService_SendNotification_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TaskServiceServer).SendNotification(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.TaskService/SendNotification", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TaskServiceServer).SendNotification(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +// TaskService_ServiceDesc is the grpc.ServiceDesc for TaskService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var TaskService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.TaskService", + HandlerType: (*TaskServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Fetch", + Handler: _TaskService_Fetch_Handler, + }, + { + MethodName: "SendNotification", + Handler: _TaskService_SendNotification_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Subscribe", + Handler: _TaskService_Subscribe_Handler, + ClientStreams: true, + }, + }, + Metadata: "services/task_service.proto", +}