diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md index 65276eb32..c99c1e705 100644 --- a/CHANGELOG-zh.md +++ b/CHANGELOG-zh.md @@ -1,3 +1,20 @@ +# 0.5.1 (2020-07-31) +### 功能 / 优化 +- **加入错误详情信息**. +- **加入 Golang 编程语言支持**. +- **加入 Chrome Driver 和 Firefox 的 Web Driver 安装脚本**. +- **支持系统任务**. "系统任务"跟普通爬虫任务相似，允许用户查看诸如安装语言之类的任务日志. +- **将安装语言从 RPC 更改为系统任务**. + +### Bug 修复 +- **修复在爬虫市场中第一次下载爬虫时会报500错误**. [#808](https://github.com/crawlab-team/crawlab/issues/808) +- **修复一部分翻译问题**. +- **修复任务详情 500 错误**. [#810](https://github.com/crawlab-team/crawlab/issues/810) +- **修复密码重置问题**. [#811](https://github.com/crawlab-team/crawlab/issues/811) +- **修复无法下载 CSV 问题**. [#812](https://github.com/crawlab-team/crawlab/issues/812) +- **修复无法安装 Node.js 问题**. [#813](https://github.com/crawlab-team/crawlab/issues/813) +- **修复批量添加定时任务时默认为禁用问题**. [#814](https://github.com/crawlab-team/crawlab/issues/814) + # 0.5.0 (2020-07-19) ### 功能 / 优化 - **爬虫市场**. 允许用户下载开源爬虫到 Crawlab. diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c78dca3a..9a4ffe2a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ +# 0.5.1 (2020-07-31) +### Features / Enhancement +- **Added error message details**. +- **Added Golang programming language support**. +- **Added web driver installation scripts for Chrome Driver and Firefox**. +- **Support system tasks**. A "system task" is similar to normal spider task, it allows users to view logs of general tasks such as installing languages. +- **Changed methods of installing languages from RPC to system tasks**. + +### Bug Fixes +- **Fixed first download repo 500 error in Spider Market page**. [#808](https://github.com/crawlab-team/crawlab/issues/808) +- **Fixed some translation issues**. +- **Fixed 500 error in task detail page**. [#810](https://github.com/crawlab-team/crawlab/issues/810) +- **Fixed password reset issue**. [#811](https://github.com/crawlab-team/crawlab/issues/811) +- **Fixed unable to download CSV issue**. [#812](https://github.com/crawlab-team/crawlab/issues/812) +- **Fixed unable to install node.js issue**. [#813](https://github.com/crawlab-team/crawlab/issues/813) +- **Fixed disabled status for batch adding schedules**. [#814](https://github.com/crawlab-team/crawlab/issues/814) + # 0.5.0 (2020-07-19) ### Features / Enhancement - **Spider Market**. Allow users to download open-source spiders into Crawlab. diff --git a/backend/conf/config.yml b/backend/conf/config.yml index cfb748814..bc06935b5 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -33,13 +33,14 @@ server: java: "N" dotnet: "N" php: "N" + scripts: "/app/backend/scripts" spider: path: "/app/spiders" task: workers: 16 other: tmppath: "/tmp" -version: 0.5.0 +version: 0.5.1 setting: crawlabLogToES: "N" # Send crawlab runtime log to ES, open this option "Y", remember to set esClient crawlabLogIndex: "crawlab-log" diff --git a/backend/constants/system.go b/backend/constants/system.go index 03491222d..14c45698f 100644 --- a/backend/constants/system.go +++ b/backend/constants/system.go @@ -18,3 +18,8 @@ const ( InstallStatusInstallingOther = "installing-other" InstallStatusInstalled = "installed" ) + +const ( + LangTypeLang = "lang" + LangTypeWebDriver = "webdriver" +) diff --git a/backend/constants/task.go b/backend/constants/task.go index 63144e8b3..085394328 100644 --- a/backend/constants/task.go +++ b/backend/constants/task.go @@ -25,3 +25,8 @@ const ( RunTypeRandom string = "random" RunTypeSelectedNodes string = "selected-nodes" ) + +const ( + TaskTypeSpider string = "spider" + TaskTypeSystem string = "system" +) diff --git a/backend/entity/system.go b/backend/entity/system.go index bb95216db..f1a24f4b2 100644 --- a/backend/entity/system.go +++ b/backend/entity/system.go @@ -24,6 +24,7 @@ type Lang struct { InstallStatus string `json:"install_status"` DepFileName string `json:"dep_file_name"` InstallDepArgs string `json:"install_dep_cmd"` + Type string `json:"type"` } type Dependency struct { diff --git a/backend/main.go b/backend/main.go index b636d3fd9..86dccf0a4 100644 --- a/backend/main.go +++ b/backend/main.go @@ -254,6 +254,11 @@ func main() { authGroup.POST("/tasks-cancel", routes.CancelSelectedTask) // 批量取消任务 authGroup.POST("/tasks-restart", routes.RestartSelectedTask) // 批量重试任务 } + // 系统任务/脚本 + { + authGroup.PUT("/system-tasks", routes.PutSystemTask) // 运行系统任务 + authGroup.GET("/system-scripts", routes.GetSystemScripts) // 获取系统脚本列表 + } // 定时任务 { authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表 @@ -269,13 +274,14 @@ func main() { } // 用户 { - authGroup.GET("/users", routes.GetUserList) // 用户列表 - authGroup.GET("/users/:id", routes.GetUser) // 用户详情 - authGroup.POST("/users/:id", routes.PostUser) // 更改用户 - authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户 - authGroup.PUT("/users-add", routes.PutUser) // 添加用户 - authGroup.GET("/me", routes.GetMe) // 获取自己账户 - authGroup.POST("/me", routes.PostMe) // 修改自己账户 + authGroup.GET("/users", routes.GetUserList) // 用户列表 + authGroup.GET("/users/:id", routes.GetUser) // 用户详情 + authGroup.POST("/users/:id", routes.PostUser) // 更改用户 + authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户 + authGroup.PUT("/users-add", routes.PutUser) // 添加用户 + authGroup.GET("/me", routes.GetMe) // 获取自己账户 + authGroup.POST("/me", routes.PostMe) // 修改自己账户 + authGroup.POST("/me/change-password", routes.PostMeChangePassword) // 修改自己密码 } // 系统 { diff --git a/backend/model/schedule.go b/backend/model/schedule.go index 5faba03b4..5d0da2869 100644 --- a/backend/model/schedule.go +++ b/backend/model/schedule.go @@ -69,7 +69,10 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) { if schedule.RunType == constants.RunTypeSelectedNodes { for _, nodeId := range schedule.NodeIds { // 选择单一节点 - node, _ := GetNode(nodeId) + node, err := GetNode(nodeId) + if err != nil { + continue + } schedule.Nodes = append(schedule.Nodes, node) } } diff --git a/backend/model/task.go b/backend/model/task.go index 6b8d91217..789bfafbe 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -5,6 +5,7 @@ import ( "crawlab/database" "crawlab/utils" "github.com/apex/log" + "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" "runtime/debug" "time" @@ -29,6 +30,7 @@ type Task struct { Pid int `json:"pid" bson:"pid"` RunType string `json:"run_type" bson:"run_type"` ScheduleId bson.ObjectId `json:"schedule_id" bson:"schedule_id"` + Type string `json:"type" bson:"type"` // 前端数据 SpiderName string `json:"spider_name"` @@ -514,3 +516,19 @@ func UpdateTaskErrorLogs(taskId string, errorRegexPattern string) error { return nil } + +func GetTaskByFilter(filter bson.M) (t Task, err error) { + s, c := database.GetCol("tasks") + defer s.Close() + + if err := c.Find(filter).One(&t); err != nil { + if err != mgo.ErrNotFound { + log.Errorf("find task by filter error: " + err.Error()) + debug.PrintStack() + return t, err + } + return t, err + } + + return t, nil +} diff --git a/backend/routes/schedule.go b/backend/routes/schedule.go index 61672c963..5da735e0a 100644 --- a/backend/routes/schedule.go +++ b/backend/routes/schedule.go @@ -242,6 +242,9 @@ func PutBatchSchedules(c *gin.Context) { // 添加 UserID s.UserId = services.GetCurrentUserId(c) + // 默认启用 + s.Enabled = true + // 添加定时任务 if err := model.AddSchedule(s); err != nil { log.Errorf("add schedule error: " + err.Error()) diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 310d802bc..87ec7c26b 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -812,6 +812,7 @@ func RunSelectedSpider(c *gin.Context) { UserId: services.GetCurrentUserId(c), RunType: constants.RunTypeAllNodes, ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), + Type: constants.TaskTypeSpider, } id, err := services.AddTask(t) @@ -830,6 +831,7 @@ func RunSelectedSpider(c *gin.Context) { UserId: services.GetCurrentUserId(c), RunType: constants.RunTypeRandom, ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), + Type: constants.TaskTypeSpider, } id, err := services.AddTask(t) if err != nil { @@ -847,6 +849,7 @@ func RunSelectedSpider(c *gin.Context) { UserId: services.GetCurrentUserId(c), RunType: constants.RunTypeSelectedNodes, ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), + Type: constants.TaskTypeSpider, } id, err := services.AddTask(t) diff --git a/backend/routes/system_tasks.go b/backend/routes/system_tasks.go new file mode 100644 index 000000000..087a2fb4f --- /dev/null +++ b/backend/routes/system_tasks.go @@ -0,0 +1,118 @@ +package routes + +import ( + "crawlab/constants" + "crawlab/model" + "crawlab/services" + "crawlab/utils" + "fmt" + "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" + "net/http" +) + +func GetSystemScripts(c *gin.Context) { + HandleSuccessData(c, utils.GetSystemScripts()) +} + +func PutSystemTask(c *gin.Context) { + type TaskRequestBody struct { + RunType string `json:"run_type"` + NodeIds []bson.ObjectId `json:"node_ids"` + Script string `json:"script"` + } + + // 绑定数据 + var reqBody TaskRequestBody + if err := c.ShouldBindJSON(&reqBody); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + // 校验脚本参数不为空 + if reqBody.Script == "" { + HandleErrorF(http.StatusBadRequest, c, "script cannot be empty") + return + } + + // 校验脚本参数是否存在 + var allScripts = utils.GetSystemScripts() + if !utils.StringArrayContains(allScripts, reqBody.Script) { + HandleErrorF(http.StatusBadRequest, c, "script does not exist") + return + } + + // TODO: 校验脚本是否正在运行 + + // 获取执行命令 + cmd := fmt.Sprintf("sh %s", utils.GetSystemScriptPath(reqBody.Script)) + + // 任务ID + var taskIds []string + + if reqBody.RunType == constants.RunTypeAllNodes { + // 所有节点 + nodes, err := model.GetNodeList(nil) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + for _, node := range nodes { + t := model.Task{ + SpiderId: bson.ObjectIdHex(constants.ObjectIdNull), + NodeId: node.Id, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeAllNodes, + ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), + Type: constants.TaskTypeSystem, + Cmd: cmd, + } + id, err := services.AddTask(t) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + taskIds = append(taskIds, id) + } + } else if reqBody.RunType == constants.RunTypeRandom { + // 随机 + t := model.Task{ + SpiderId: bson.ObjectIdHex(constants.ObjectIdNull), + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeRandom, + ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), + Type: constants.TaskTypeSystem, + Cmd: cmd, + } + id, err := services.AddTask(t) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + taskIds = append(taskIds, id) + } else if reqBody.RunType == constants.RunTypeSelectedNodes { + // 指定节点 + for _, nodeId := range reqBody.NodeIds { + t := model.Task{ + SpiderId: bson.ObjectIdHex(constants.ObjectIdNull), + NodeId: nodeId, + UserId: services.GetCurrentUserId(c), + RunType: constants.RunTypeSelectedNodes, + ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), + Type: constants.TaskTypeSystem, + Cmd: cmd, + } + id, err := services.AddTask(t) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + taskIds = append(taskIds, id) + } + } else { + HandleErrorF(http.StatusInternalServerError, c, "invalid run_type") + return + } + + HandleSuccessData(c, taskIds) +} diff --git a/backend/routes/task.go b/backend/routes/task.go index d8ea39a1f..2a74f4e87 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -19,6 +19,7 @@ type TaskListRequestData struct { SpiderId string `form:"spider_id"` ScheduleId string `form:"schedule_id"` Status string `form:"status"` + Type string `form:"type"` } type TaskResultsRequestData struct { @@ -64,6 +65,9 @@ func GetTaskList(c *gin.Context) { if data.ScheduleId != "" { query["schedule_id"] = bson.ObjectIdHex(data.ScheduleId) } + if data.Type != "" { + query["type"] = data.Type + } // 获取校验 query = services.GetAuthQuery(query, c) @@ -150,6 +154,7 @@ func PutTask(c *gin.Context) { UserId: services.GetCurrentUserId(c), RunType: constants.RunTypeAllNodes, ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), + Type: constants.TaskTypeSpider, } id, err := services.AddTask(t) @@ -168,6 +173,7 @@ func PutTask(c *gin.Context) { UserId: services.GetCurrentUserId(c), RunType: constants.RunTypeRandom, ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), + Type: constants.TaskTypeSpider, } id, err := services.AddTask(t) if err != nil { @@ -185,6 +191,7 @@ func PutTask(c *gin.Context) { UserId: services.GetCurrentUserId(c), RunType: constants.RunTypeSelectedNodes, ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), + Type: constants.TaskTypeSpider, } id, err := services.AddTask(t) @@ -225,6 +232,7 @@ func PutBatchTasks(c *gin.Context) { UserId: services.GetCurrentUserId(c), RunType: constants.RunTypeAllNodes, ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), + Type: constants.TaskTypeSpider, } id, err := services.AddTask(t) @@ -242,6 +250,7 @@ func PutBatchTasks(c *gin.Context) { UserId: services.GetCurrentUserId(c), RunType: constants.RunTypeRandom, ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), + Type: constants.TaskTypeSpider, } id, err := services.AddTask(t) if err != nil { @@ -259,6 +268,7 @@ func PutBatchTasks(c *gin.Context) { UserId: services.GetCurrentUserId(c), RunType: constants.RunTypeSelectedNodes, ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), + Type: constants.TaskTypeSpider, } id, err := services.AddTask(t) diff --git a/backend/routes/user.go b/backend/routes/user.go index 23391a48b..abb771706 100644 --- a/backend/routes/user.go +++ b/backend/routes/user.go @@ -279,9 +279,6 @@ func PostMe(c *gin.Context) { if reqBody.Email != "" { user.Email = reqBody.Email } - if reqBody.Password != "" { - user.Password = utils.EncryptPassword(reqBody.Password) - } if reqBody.Setting.NotificationTrigger != "" { user.Setting.NotificationTrigger = reqBody.Setting.NotificationTrigger } @@ -311,3 +308,33 @@ func PostMe(c *gin.Context) { Message: "success", }) } + +func PostMeChangePassword(c *gin.Context) { + ctx := context.WithGinContext(c) + user := ctx.User() + if user == nil { + ctx.FailedWithError(constants.ErrorUserNotFound, http.StatusUnauthorized) + return + } + var reqBody model.User + if err := c.ShouldBindJSON(&reqBody); err != nil { + HandleErrorF(http.StatusBadRequest, c, "invalid request") + return + } + if reqBody.Password == "" { + HandleErrorF(http.StatusBadRequest, c, "password is empty") + return + } + if user.UserId.Hex() == "" { + user.UserId = bson.ObjectIdHex(constants.ObjectIdNull) + } + user.Password = utils.EncryptPassword(reqBody.Password) + if err := user.Save(); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} diff --git a/backend/scripts/install-chromedriver.sh b/backend/scripts/install-chromedriver.sh new file mode 100644 index 000000000..c2e869391 --- /dev/null +++ b/backend/scripts/install-chromedriver.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# fail immediately if error +set -e + +# lock global +touch /tmp/install.lock + +# lock +touch /tmp/install-chromedriver.lock + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install unzip +DL=https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb +curl -sL "$DL" > /tmp/chrome.deb +apt install --no-install-recommends --no-install-suggests -y /tmp/chrome.deb +CHROMIUM_FLAGS='--no-sandbox --disable-dev-shm-usage' +sed -i '${s/$/'" $CHROMIUM_FLAGS"'/}' /opt/google/chrome/google-chrome +BASE_URL=https://chromedriver.storage.googleapis.com +VERSION=$(curl -sL "$BASE_URL/LATEST_RELEASE") +curl -sL "$BASE_URL/$VERSION/chromedriver_linux64.zip" -o /tmp/driver.zip +unzip /tmp/driver.zip +chmod 755 chromedriver +mv chromedriver /usr/local/bin + +# unlock global +rm /tmp/install.lock + +# unlock +rm /tmp/install-chromedriver.lock \ No newline at end of file diff --git a/backend/scripts/install-dotnet.sh b/backend/scripts/install-dotnet.sh index ae2464c0f..73f3d0cb8 100755 --- a/backend/scripts/install-dotnet.sh +++ b/backend/scripts/install-dotnet.sh @@ -1,3 +1,8 @@ +#!/bin/bash + +# fail immediately if error +set -e + # lock global touch /tmp/install.lock diff --git a/backend/scripts/install-firefox.sh b/backend/scripts/install-firefox.sh new file mode 100644 index 000000000..f316b3dbb --- /dev/null +++ b/backend/scripts/install-firefox.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# fail immediately if error +set -e + +# lock global +touch /tmp/install.lock + +# lock +touch /tmp/install-firefox.lock + +apt-get update +apt-get -y install firefox ttf-wqy-microhei ttf-wqy-zenhei xfonts-wqy +apt-get -y install libcanberra-gtk3-module + +# unlock global +rm /tmp/install.lock + +# unlock +rm /tmp/install-firefox.lock \ No newline at end of file diff --git a/backend/scripts/install-go.sh b/backend/scripts/install-go.sh new file mode 100644 index 000000000..44d437440 --- /dev/null +++ b/backend/scripts/install-go.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# fail immediately if error +set -e + +# lock global +touch /tmp/install.lock + +# lock +touch /tmp/install-go.lock + +# install golang +apt-get update +apt-get install -y golang + +# environment variables +export GOPROXY=https://goproxy.cn +export GOPATH=/opt/go + +# unlock global +rm /tmp/install.lock + +# unlock +rm /tmp/install-go.lock \ No newline at end of file diff --git a/backend/scripts/install-java.sh b/backend/scripts/install-java.sh index 7cb8972c2..ab797cb90 100755 --- a/backend/scripts/install-java.sh +++ b/backend/scripts/install-java.sh @@ -1,5 +1,8 @@ #!/bin/bash +# fail immediately if error +set -e + # lock global touch /tmp/install.lock @@ -7,9 +10,9 @@ touch /tmp/install.lock touch /tmp/install-java.lock # install java -apt-get clean && \ - apt-get update --fix-missing && \ - apt-get install -y --fix-missing default-jdk +apt-get clean +apt-get update --fix-missing +apt-get install -y --fix-missing default-jdk ln -s /usr/bin/java /usr/local/bin/java # unlock diff --git a/backend/scripts/install-nodejs.sh b/backend/scripts/install-nodejs.sh index 129bbc44a..802e001ea 100644 --- a/backend/scripts/install-nodejs.sh +++ b/backend/scripts/install-nodejs.sh @@ -1,28 +1,18 @@ #!/bin/bash +# fail immediately if error +set -e + # lock global touch /tmp/install.lock # lock touch /tmp/install-nodejs.lock -# install nvm -BASE_DIR=`dirname $0` -/bin/bash ${BASE_DIR}/install-nvm.sh -export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")" -[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm - -# install Node.js v10.19 -export NVM_NODEJS_ORG_MIRROR=http://npm.taobao.org/mirrors/node -nvm install 10.19 - -# create soft links -ln -s $HOME/.nvm/versions/node/v10.19.0/bin/npm /usr/local/bin/npm -ln -s $HOME/.nvm/versions/node/v10.19.0/bin/node /usr/local/bin/node - -# environments manipulation -export NODE_PATH=$HOME.nvm/versions/node/v10.19.0/lib/node_modules -export PATH=$NODE_PATH:$PATH +# install node.js +curl -sL https://deb.nodesource.com/setup_10.x | bash - +apt-get update && apt install -y nodejs nodejs-dev node-gyp libssl1.0-dev +apt-get update && apt install -y npm # install chromium # See https://crbug.com/795759 @@ -33,7 +23,17 @@ apt-get update && apt-get install -yq libgconf-2-4 # Note: this installs the necessary libs to make the bundled version # of Chromium that Puppeteer # installs, work. -apt-get update && apt-get install -y --no-install-recommends gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget +apt-get update \ + && apt-get install -y wget gnupg \ + && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ + && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ + && apt-get update \ + && apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \ + libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \ + libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ + libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \ + libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget \ + && rm -rf /var/lib/apt/lists/* # install default dependencies PUPPETEER_DOWNLOAD_HOST=https://npm.taobao.org/mirrors diff --git a/backend/scripts/install-nvm.sh b/backend/scripts/install-nvm.sh index b87baaed3..d02f884ce 100755 --- a/backend/scripts/install-nvm.sh +++ b/backend/scripts/install-nvm.sh @@ -1,4 +1,7 @@ -#!/usr/bin/env bash +#!/bin/bash + +# fail immediately if error +set -e { # this ensures the entire script is downloaded # diff --git a/backend/scripts/install-php.sh b/backend/scripts/install-php.sh index bc6e3ad63..d3d392f45 100755 --- a/backend/scripts/install-php.sh +++ b/backend/scripts/install-php.sh @@ -1,3 +1,8 @@ +#!/bin/bash + +# fail immediately if error +set -e + # lock global touch /tmp/install.lock diff --git a/backend/scripts/install.sh b/backend/scripts/install.sh index fe89d1b23..68c5b3ac2 100644 --- a/backend/scripts/install.sh +++ b/backend/scripts/install.sh @@ -1,5 +1,8 @@ #!/bin/bash +# fail immediately if error +set -e + # install node.js if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ]; then @@ -23,3 +26,19 @@ then /bin/sh /app/backend/scripts/install-dotnet.sh echo "installed dotnet" fi + +# install php +if [ "${CRAWLAB_SERVER_LANG_PHP}" = "Y" ]; +then + echo "installing php" + /bin/sh /app/backend/scripts/install-php.sh + echo "installed php" +fi + +# install go +if [ "${CRAWLAB_SERVER_LANG_GO}" = "Y" ]; +then + echo "installing go" + /bin/sh /app/backend/scripts/install-go.sh + echo "installed go" +fi diff --git a/backend/services/repo.go b/backend/services/repo.go index 9acc83878..a2afb8cd6 100644 --- a/backend/services/repo.go +++ b/backend/services/repo.go @@ -58,7 +58,7 @@ func DownloadRepo(fullName string, userId bson.ObjectId) (err error) { spider := model.GetSpiderByName(spiderName) if spider.Name == "" { // 新增 - spider := model.Spider{ + spider = model.Spider{ Id: bson.NewObjectId(), Name: spiderName, DisplayName: spiderName, diff --git a/backend/services/rpc/get_lang.go b/backend/services/rpc/get_lang.go index d06629841..93d8baf11 100644 --- a/backend/services/rpc/get_lang.go +++ b/backend/services/rpc/get_lang.go @@ -48,17 +48,17 @@ func GetLangLocal(lang entity.Lang) entity.Lang { } } - // 检查是否正在安装 - if utils.Exists(lang.LockPath) { - lang.InstallStatus = constants.InstallStatusInstalling - return lang - } - - // 检查其他语言是否在安装 - if utils.Exists("/tmp/install.lock") { - lang.InstallStatus = constants.InstallStatusInstallingOther - return lang - } + //// 检查是否正在安装 + //if utils.Exists(lang.LockPath) { + // lang.InstallStatus = constants.InstallStatusInstalling + // return lang + //} + // + //// 检查其他语言是否在安装 + //if utils.Exists("/tmp/install.lock") { + // lang.InstallStatus = constants.InstallStatusInstallingOther + // return lang + //} lang.InstallStatus = constants.InstallStatusNotInstalled return lang diff --git a/backend/services/schedule.go b/backend/services/schedule.go index 591fe4fb5..017fba7dc 100644 --- a/backend/services/schedule.go +++ b/backend/services/schedule.go @@ -58,6 +58,7 @@ func AddScheduleTask(s model.Schedule) func() { UserId: s.UserId, RunType: constants.RunTypeAllNodes, ScheduleId: s.Id, + Type: constants.TaskTypeSpider, } if _, err := AddTask(t); err != nil { @@ -73,6 +74,7 @@ func AddScheduleTask(s model.Schedule) func() { UserId: s.UserId, RunType: constants.RunTypeRandom, ScheduleId: s.Id, + Type: constants.TaskTypeSpider, } if _, err := AddTask(t); err != nil { log.Errorf(err.Error()) @@ -90,6 +92,7 @@ func AddScheduleTask(s model.Schedule) func() { UserId: s.UserId, RunType: constants.RunTypeSelectedNodes, ScheduleId: s.Id, + Type: constants.TaskTypeSpider, } if _, err := AddTask(t); err != nil { diff --git a/backend/services/system.go b/backend/services/system.go index 49a47219d..a4b19e982 100644 --- a/backend/services/system.go +++ b/backend/services/system.go @@ -5,12 +5,15 @@ import ( "crawlab/database" "crawlab/entity" "crawlab/lib/cron" + "crawlab/model" "crawlab/services/rpc" "crawlab/utils" "encoding/json" "errors" "fmt" "github.com/apex/log" + "github.com/globalsign/mgo" + "github.com/globalsign/mgo/bson" "github.com/imroc/req" "os/exec" "regexp" @@ -71,7 +74,24 @@ func GetLangList(nodeId string) []entity.Lang { return list } +// 获取语言安装状态 func GetLangInstallStatus(nodeId string, lang entity.Lang) (string, error) { + _, err := model.GetTaskByFilter(bson.M{ + "node_id": nodeId, + "cmd": fmt.Sprintf("sh %s", utils.GetSystemScriptPath(lang.InstallScript)), + "status": bson.M{ + "$in": []string{constants.StatusPending, constants.StatusRunning}, + }, + }) + if err == nil { + // 任务正在运行，正在安装 + return constants.InstallStatusInstalling, nil + } + if err != mgo.ErrNotFound { + // 发生错误 + return "", err + } + // 获取状态 if IsMasterNode(nodeId) { lang := rpc.GetLangLocal(lang) return lang.InstallStatus, nil diff --git a/backend/services/task.go b/backend/services/task.go index f553dea07..244214428 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -23,7 +23,6 @@ import ( "net/http" "os" "os/exec" - "path" "path/filepath" "runtime" "runtime/debug" @@ -116,9 +115,7 @@ func AssignTask(task model.Task) error { func SetEnv(cmd *exec.Cmd, envs []model.Env, task model.Task, spider model.Spider) *exec.Cmd { // 默认把Node.js的全局node_modules加入环境变量 envPath := os.Getenv("PATH") - homePath := os.Getenv("HOME") - nodeVersion := "v10.19.0" - nodePath := path.Join(homePath, ".nvm/versions/node", nodeVersion, "lib/node_modules") + nodePath := "/usr/lib/node_modules" if !strings.Contains(envPath, nodePath) { _ = os.Setenv("PATH", nodePath+":"+envPath) } @@ -411,37 +408,17 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider, u if err := WaitTaskProcess(cmd, t, s); err != nil { return err } - ch <- constants.TaskFinish - return nil -} - -// 生成日志目录 -func MakeLogDir(t model.Task) (fileDir string, err error) { - // 日志目录 - fileDir = filepath.Join(viper.GetString("log.path"), t.SpiderId.Hex()) - // 如果日志目录不存在，生成该目录 - if !utils.Exists(fileDir) { - if err := os.MkdirAll(fileDir, 0777); err != nil { - log.Errorf("execute task, make log dir error: %s", err.Error()) - debug.PrintStack() - return "", err - } + // 如果返回值不为0，返回错误 + returnCode := cmd.ProcessState.ExitCode() + if returnCode != 0 { + log.Errorf(fmt.Sprintf("task returned code not zero: %d", returnCode)) + debug.PrintStack() + return errors.New(fmt.Sprintf("task returned code not zero: %d", returnCode)) } - return fileDir, nil -} - -// 获取日志文件路径 -func GetLogFilePaths(fileDir string, t model.Task) (filePath string) { - // 时间戳 - ts := time.Now() - tsStr := ts.Format("20060102150405") - - // stdout日志文件 - filePath = filepath.Join(fileDir, t.Id+"_"+tsStr+".log") - - return filePath + ch <- constants.TaskFinish + return nil } // 生成执行任务方法 @@ -545,19 +522,14 @@ func ExecuteTask(id int) { } // 获取爬虫 - spider, err := t.GetSpider() - if err != nil { - log.Errorf("execute task, get spider error: %s", err.Error()) - return - } - - // 创建日志目录 - var fileDir string - if fileDir, err = MakeLogDir(t); err != nil { - return + var spider model.Spider + if t.Type == constants.TaskTypeSpider { + spider, err = t.GetSpider() + if err != nil { + log.Errorf("execute task, get spider error: %s", err.Error()) + return + } } - // 获取日志文件路径 - t.LogPath = GetLogFilePaths(fileDir, t) // 工作目录 cwd := filepath.Join( @@ -567,12 +539,19 @@ func ExecuteTask(id int) { // 执行命令 var cmd string - if spider.Type == constants.Configurable { - // 可配置爬虫命令 - cmd = "scrapy crawl config_spider" - } else { - // 自定义爬虫命令 - cmd = spider.Cmd + if t.Type == constants.TaskTypeSpider { + // 爬虫任务 + if spider.Type == constants.Configurable { + // 可配置爬虫命令 + cmd = "scrapy crawl config_spider" + } else { + // 自定义爬虫命令 + cmd = spider.Cmd + } + t.Cmd = cmd + } else if t.Type == constants.TaskTypeSystem { + // 系统任务 + cmd = t.Cmd } // 加入参数 @@ -593,48 +572,51 @@ func ExecuteTask(id int) { t.Status = constants.StatusRunning // 任务状态 t.WaitDuration = t.StartTs.Sub(t.CreateTs).Seconds() // 等待时长 + // 储存任务 + _ = t.Save() + // 发送 Web Hook 请求 (任务开始) go SendWebHookRequest(user, t, spider) - // 文件检查 - if err := SpiderFileCheck(t, spider); err != nil { - log.Errorf("spider file check error: %s", err.Error()) - return - } + // 爬虫任务专属逻辑 + if t.Type == constants.TaskTypeSpider { + // 文件检查 + if err := SpiderFileCheck(t, spider); err != nil { + log.Errorf("spider file check error: %s", err.Error()) + return + } - // 开始执行任务 - log.Infof(GetWorkerPrefix(id) + "start task (id:" + t.Id + ")") + // 开始执行任务 + log.Infof(GetWorkerPrefix(id) + "start task (id:" + t.Id + ")") - // 储存任务 - _ = t.Save() - - // 创建结果集索引 - go func() { - col := utils.GetSpiderCol(spider.Col, spider.Name) - CreateResultsIndexes(col) - }() + // 创建结果集索引 + go func() { + col := utils.GetSpiderCol(spider.Col, spider.Name) + CreateResultsIndexes(col) + }() - // 起一个cron执行器来统计任务结果数 - cronExec := cron.New(cron.WithSeconds()) - _, err = cronExec.AddFunc("*/5 * * * * *", SaveTaskResultCount(t.Id)) - if err != nil { - log.Errorf(GetWorkerPrefix(id) + err.Error()) - debug.PrintStack() - return - } - cronExec.Start() - defer cronExec.Stop() + // 起一个cron执行器来统计任务结果数 + cronExec := cron.New(cron.WithSeconds()) + _, err = cronExec.AddFunc("*/5 * * * * *", SaveTaskResultCount(t.Id)) + if err != nil { + log.Errorf(GetWorkerPrefix(id) + err.Error()) + debug.PrintStack() + return + } + cronExec.Start() + defer cronExec.Stop() - // 起一个cron来更新错误日志 - cronExecErrLog := cron.New(cron.WithSeconds()) - _, err = cronExecErrLog.AddFunc("*/30 * * * * *", ScanErrorLogs(t)) - if err != nil { - log.Errorf(GetWorkerPrefix(id) + err.Error()) - debug.PrintStack() - return + // 起一个cron来更新错误日志 + cronExecErrLog := cron.New(cron.WithSeconds()) + _, err = cronExecErrLog.AddFunc("*/30 * * * * *", ScanErrorLogs(t)) + if err != nil { + log.Errorf(GetWorkerPrefix(id) + err.Error()) + debug.PrintStack() + return + } + cronExecErrLog.Start() + defer cronExecErrLog.Stop() } - cronExecErrLog.Start() - defer cronExecErrLog.Stop() // 执行Shell命令 if err := ExecuteShellCmd(cmd, cwd, t, spider, user); err != nil { @@ -693,11 +675,13 @@ func ExecuteTask(id int) { func FinishUpTask(s model.Spider, t model.Task) { // 更新任务结果数 - go func() { - if err := model.UpdateTaskResultCount(t.Id); err != nil { - return - } - }() + if t.Type == constants.TaskTypeSpider { + go func() { + if err := model.UpdateTaskResultCount(t.Id); err != nil { + return + } + }() + } // 更新任务错误日志 go func() { @@ -812,10 +796,12 @@ func RestartTask(id string, uid bson.ObjectId) (err error) { newTask := model.Task{ SpiderId: oldTask.SpiderId, NodeId: oldTask.NodeId, + Cmd: oldTask.Cmd, Param: oldTask.Param, UserId: uid, RunType: oldTask.RunType, ScheduleId: bson.ObjectIdHex(constants.ObjectIdNull), + Type: oldTask.Type, } // 加入任务队列 diff --git a/backend/utils/system.go b/backend/utils/system.go index 5721aadf8..e6d2f5916 100644 --- a/backend/utils/system.go +++ b/backend/utils/system.go @@ -1,15 +1,20 @@ package utils import ( + "crawlab/constants" "crawlab/entity" "encoding/json" "github.com/apex/log" + "github.com/spf13/viper" "io/ioutil" + "path" "runtime/debug" + "strings" ) func GetLangList() []entity.Lang { list := []entity.Lang{ + // 语言 { Name: "Python", ExecutableName: "python", @@ -18,6 +23,7 @@ func GetLangList() []entity.Lang { LockPath: "/tmp/install-python.lock", DepFileName: "requirements.txt", InstallDepArgs: "install -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt", + Type: constants.LangTypeLang, }, { Name: "Node.js", @@ -28,6 +34,7 @@ func GetLangList() []entity.Lang { InstallScript: "install-nodejs.sh", DepFileName: "package.json", InstallDepArgs: "install -g --registry=https://registry.npm.taobao.org", + Type: constants.LangTypeLang, }, { Name: "Java", @@ -35,6 +42,7 @@ func GetLangList() []entity.Lang { ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"}, LockPath: "/tmp/install-java.lock", InstallScript: "install-java.sh", + Type: constants.LangTypeLang, }, { Name: ".Net Core", @@ -42,6 +50,7 @@ func GetLangList() []entity.Lang { ExecutablePaths: []string{"/usr/bin/dotnet", "/usr/local/bin/dotnet"}, LockPath: "/tmp/install-dotnet.lock", InstallScript: "install-dotnet.sh", + Type: constants.LangTypeLang, }, { Name: "PHP", @@ -49,6 +58,32 @@ func GetLangList() []entity.Lang { ExecutablePaths: []string{"/usr/bin/php", "/usr/local/bin/php"}, LockPath: "/tmp/install-php.lock", InstallScript: "install-php.sh", + Type: constants.LangTypeLang, + }, + { + Name: "Golang", + ExecutableName: "go", + ExecutablePaths: []string{"/usr/bin/go", "/usr/local/bin/go"}, + LockPath: "/tmp/install-go.lock", + InstallScript: "install-go.sh", + Type: constants.LangTypeLang, + }, + // WebDriver + { + Name: "Chrome Driver", + ExecutableName: "chromedriver", + ExecutablePaths: []string{"/usr/bin/chromedriver", "/usr/local/bin/chromedriver"}, + LockPath: "/tmp/install-chromedriver.lock", + InstallScript: "install-chromedriver.sh", + Type: constants.LangTypeWebDriver, + }, + { + Name: "Firefox", + ExecutableName: "firefox", + ExecutablePaths: []string{"/usr/bin/firefox", "/usr/local/bin/firefox"}, + LockPath: "/tmp/install-firefox.lock", + InstallScript: "install-firefox.sh", + Type: constants.LangTypeWebDriver, }, } return list @@ -91,3 +126,24 @@ func GetPackageJsonDeps(filepath string) (deps []string, err error) { return deps, nil } + +// 获取系统脚本列表 +func GetSystemScripts() (res []string) { + scriptsPath := viper.GetString("server.scripts") + for _, fInfo := range ListDir(scriptsPath) { + if !fInfo.IsDir() && strings.HasSuffix(fInfo.Name(), ".sh") { + res = append(res, fInfo.Name()) + } + } + return res +} + +func GetSystemScriptPath(scriptName string) string { + scriptsPath := viper.GetString("server.scripts") + for _, name := range GetSystemScripts() { + if name == scriptName { + return path.Join(scriptsPath, name) + } + } + return "" +} diff --git a/docker-compose.yml b/docker-compose.yml index f82c33c70..1c06f4221 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,7 @@ services: # CRAWLAB_SERVER_LANG_JAVA: "Y" # whether to pre-install Java 预安装 Java 语言环境 # CRAWLAB_SERVER_LANG_DOTNET: "Y" # whether to pre-install .Net core 预安装 .Net Core 语言环境 # CRAWLAB_SERVER_LANG_PHP: "Y" # whether to pre-install PHP 预安装 PHP 语言环境 + # CRAWLAB_SERVER_LANG_GO: "Y" # whether to pre-install Golang 预安装 Golang 语言环境 # CRAWLAB_SETTING_ALLOWREGISTER: "N" # whether to allow user registration 是否允许用户注册 # CRAWLAB_SETTING_ENABLETUTORIAL: "N" # whether to enable tutorial 是否启用教程 # CRAWLAB_SETTING_RUNONMASTER: "N" # whether to run on master node 是否在主节点上运行任务 diff --git a/docker_init.sh b/docker_init.sh index dce1137a1..dfef4eafc 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -23,7 +23,7 @@ fi service nginx start # install languages -if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ]; +if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_DOTNET}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_PHP}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_GO}" = "Y" ]; then echo "installing languages" echo "you can view log at /var/log/install.sh.log" diff --git a/frontend/src/components/InfoView/TaskInfoView.vue b/frontend/src/components/InfoView/TaskInfoView.vue index 1fd328527..c3a45d965 100644 --- a/frontend/src/components/InfoView/TaskInfoView.vue +++ b/frontend/src/components/InfoView/TaskInfoView.vue @@ -24,7 +24,7 @@ @@ -32,8 +32,8 @@ {{ $t('Empty results') }} - - + + diff --git a/frontend/src/components/Node/NodeInstallationMatrix.vue b/frontend/src/components/Node/NodeInstallationMatrix.vue index 25d4a3314..0951d53ea 100644 --- a/frontend/src/components/Node/NodeInstallationMatrix.vue +++ b/frontend/src/components/Node/NodeInstallationMatrix.vue @@ -72,7 +72,11 @@ {{ $t('Not Installed') }} - + {{ $t('Install') }} @@ -203,6 +207,97 @@ + +
+ + + + + + + + + + + + + +
+ @@ -220,12 +315,17 @@ }, data() { return { - langs: [ - { label: 'Python', name: 'python', hasDeps: true }, - { label: 'Node.js', name: 'node', hasDeps: true }, - { label: 'Java', name: 'java', hasDeps: false }, - { label: '.Net Core', name: 'dotnet', hasDeps: false }, - { label: 'PHP', name: 'php', hasDeps: false } + allLangs: [ + // 语言 + { label: 'Python', name: 'python', hasDeps: true, script: 'install-python.sh', type: 'lang' }, + { label: 'Node.js', name: 'node', hasDeps: true, script: 'install-nodejs.sh', type: 'lang' }, + { label: 'Java', name: 'java', hasDeps: false, script: 'install-java.sh', type: 'lang' }, + { label: '.Net Core', name: 'dotnet', hasDeps: false, script: 'install-dotnet.sh', type: 'lang' }, + { label: 'PHP', name: 'php', hasDeps: false, script: 'install-php.sh', type: 'lang' }, + { label: 'Golang', name: 'go', hasDeps: false, script: 'install-go.sh', type: 'lang' }, + // web driver + { label: 'Chrome Driver', name: 'chromedriver', script: 'install-chromedriver.sh', type: 'webdriver' }, + { label: 'Firefox', name: 'firefox', script: 'install-firefox.sh', type: 'webdriver' } ], langsDataDict: {}, handle: undefined, @@ -243,6 +343,12 @@ ...mapState('node', [ 'nodeList' ]), + langs() { + return this.allLangs.filter(d => d.type === 'lang') + }, + webdrivers() { + return this.allLangs.filter(d => d.type === 'webdriver') + }, activeNodes() { return this.nodeList.filter(d => d.status === 'online') }, @@ -254,7 +360,7 @@ }) }, langsWithDeps() { - return this.langs.filter(l => l.hasDeps) + return this.allLangs.filter(l => l.hasDeps) } }, watch: { @@ -315,8 +421,8 @@ return lang.install_status }, getLangFromLabel(label) { - for (let i = 0; i < this.langs.length; i++) { - const lang = this.langs[i] + for (let i = 0; i < this.allLangs.length; i++) { + const lang = this.allLangs[i] if (lang.label === label) { return lang } @@ -327,17 +433,19 @@ ev.stopPropagation() } const lang = this.getLangFromLabel(langLabel) - this.$request.post(`/nodes/${nodeId}/langs/install`, { - lang: lang.name + const res = await this.$request.put('/system-tasks', { + run_type: 'selected-nodes', + node_ids: [nodeId], + script: lang.script }) + if (res && res.data && !res.data.error) { + this.$message.success(this.$t('Started to install') + ' ' + lang.label) + } const key = nodeId + '|' + lang.name this.$set(this.langsDataDict[key], 'install_status', 'installing') setTimeout(() => { this.getLangsData() }, 1000) - this.$request.put('/actions', { - type: 'install_lang' - }) this.$st.sendEv('节点列表', '安装', '安装语言') }, async onInstallLangAll(langLabel, ev) { diff --git a/frontend/src/components/Overview/TaskOverview.vue b/frontend/src/components/Overview/TaskOverview.vue index 76a32c9a1..1b77f7171 100644 --- a/frontend/src/components/Overview/TaskOverview.vue +++ b/frontend/src/components/Overview/TaskOverview.vue @@ -2,6 +2,7 @@
- +

{{ $t('Spider Info') }}

@@ -66,6 +67,9 @@ ]), ...mapState('spider', [ 'spiderForm' + ]), + ...mapState('task', [ + 'taskForm' ]) }, created() { diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index 10c28b6e8..a27184e54 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -38,6 +38,7 @@ export default { Running: '进行中', Finished: '已完成', Error: '错误', + Errors: '错误', NA: '未知', Cancelled: '已取消', Abnormal: '异常', @@ -104,6 +105,9 @@ export default { 'Worker': '工作节点', 'Installation': '安装', 'Search Dependencies': '搜索依赖', + 'Monitor': '监控', + 'Time Range': '时间区间', + 'Started to install': '开始安装', // 节点列表 'IP': 'IP地址', @@ -114,6 +118,38 @@ export default { Offline: '离线', Unavailable: '未知', + // 监控指标 + 'node_stats_cpu_usage_percent': '节点 CPU 使用百分比', + 'node_stats_disk_total': '节点总磁盘大小', + 'node_stats_disk_usage': '节点磁盘使用量', + 'node_stats_disk_usage_percent': '节点磁盘使用百分比', + 'node_stats_mem_total': '节点总内存大小', + 'node_stats_mem_usage': '节点内存使用量', + 'node_stats_mem_usage_percent': '节点内存使用百分比', + 'node_stats_network_bytes_recv': '节点网络接收字节数', + 'node_stats_network_bytes_sent': '节点网络发送字节数', + 'node_stats_network_packets_recv': '节点网络接收包数', + 'node_stats_network_packets_sent': '节点网络发送包数', + 'mongo_stats_mem_resident': 'MongoDB 内存使用量', + 'mongo_stats_mem_virtual': 'MongoDB 虚拟内存大小', + 'mongo_stats_mem_usage_percent': 'MongoDB 内存使用百分比', + 'mongo_stats_fs_total': 'MongoDB 总文件系统大小', + 'mongo_stats_fs_used': 'MongoDB 文件系统使用量', + 'mongo_stats_fs_usage_percent': 'MongoDB 文件系统使用百分比', + 'mongo_stats_storage_size': 'MongoDB 储存大小', + 'mongo_stats_data_size': 'MongoDB 数据大小', + 'mongo_stats_index_size': 'MongoDB 索引大小', + 'mongo_stats_objects': 'MongoDB Object 数量', + 'mongo_stats_collections': 'MongoDB Collection 数量', + 'mongo_stats_indexes': 'MongoDB 索引数量', + 'mongo_stats_avg_obj_size': 'MongoDB 平均 Object 大小', + 'redis_stats_dataset_bytes': 'Redis 数据字节数', + 'redis_stats_keys_count': 'Redis Key 数量', + 'redis_stats_overhead_total': 'Redis Overhead 总大小', + 'redis_stats_peak_allocated': 'Redis 峰值分配大小', + 'redis_stats_startup_allocated': 'Redis 启动分配大小', + 'redis_stats_total_allocated': 'Redis 总分配大小', + // 爬虫 'Spider Info': '爬虫信息', 'Spider ID': '爬虫ID', @@ -121,6 +157,8 @@ export default { 'Source Folder': '代码目录', 'Execute Command': '执行命令', 'Results Collection': '结果集', + 'Results Table': '结果表', + 'Default': '默认', 'Spider Type': '爬虫类型', 'Language': '语言', 'Schedule Enabled': '是否开启定时任务', @@ -284,6 +322,9 @@ export default { 'Start Time': '开始时间', 'Finish Time': '结束时间', 'Update Time': '更新时间', + 'Type': '类别', + 'Spider Tasks': '爬虫任务', + 'System Tasks': '系统任务', // 部署 'Time': '时间', @@ -425,6 +466,8 @@ export default { 'Disclaimer': '免责声明', 'Please search dependencies': '请搜索依赖', 'No Data': '暂无数据', + 'No data available': '暂无数据', + 'No data available. Please check whether your spiders are missing dependencies or no spiders created.': '暂无数据。请检查您的爬虫是否缺少依赖，或者没有创建爬虫。', 'Show installed': '查看已安装', 'Installing dependency successful': '安装依赖成功', 'Installing dependency failed': '安装依赖失败', @@ -502,6 +545,14 @@ export default { 'Log Errors': '日志错误', 'No Expire': '不过期', 'Log Expire Duration': '日志过期时间', + 'Database': '数据库', + 'Data Source': '数据源', + 'Data Source Type': '数据源类别', + 'Host': '主机', + 'Host address, e.g. 192.168.0.1': '主机地址，例如 192.168.0.1', + 'Port, e.g. 27017': '端口，例如 27017', + 'Auth Source (Default: admin)': 'Auth Source (默认: admin)', + 'Change Password': '更改密码', // 挑战 'Challenge': '挑战', @@ -556,12 +607,21 @@ export default { // Cron Format: [second] [minute] [hour] [day of month] [month] [day of week] cron_format: 'Cron 格式: [秒] [分] [小时] [日] [月] [周]' }, - auth: { - login_expired_message: '您已注销，可以取消以保留在该页面上，或者再次登录', - login_expired_title: '确认登出', - login_expired_confirm: '确认', - login_expired_cancel: '取消' - }, + + // 监控 + 'Disk': '磁盘', + 'Data Size': '数据大小', + 'Storage Size': '储存大小', + 'Memory': '内存', + 'CPU': 'CPU', + 'Index Size': '索引大小', + 'Total Allocated': '总分配内存', + 'Peak Allocated': '峰值内存', + 'Dataset Size': '数据大小', + 'Overhead Size': '额外开销', + 'Disk Usage': '磁盘使用量', + 'Memory Usage': '内存使用量', + // 内容 addNodeInstruction: ` 您不能在 Crawlab 的 Web 界面直接添加节点。 @@ -665,6 +725,9 @@ export default { 'Are you sure to add an API token?': '确认创建 API Token?', 'Are you sure to delete this API token?': '确认删除该 API Token?', 'Please enter Web Hook URL': '请输入 Web Hook URL', + 'Change data source failed': '更改数据源失败', + 'Changed data source successfully': '更改数据源成功', + 'Are you sure to delete this data source?': '您确定删除该数据源?', 'Are you sure to download this spider?': '您确定要下载该爬虫?', 'Downloaded successfully': '下载成功', 'Unable to submit because of some errors': '有错误，无法提交', @@ -676,7 +739,11 @@ export default { 'Are you sure to stop this task?': '确认停止这个任务?', 'Enabled successfully': '成功启用', 'Disabled successfully': '成功禁用', + 'Request Error': '请求错误', + 'Changed password successfully': '成功修改密码', + 'Two passwords do not match': '两次密码不匹配', // 其他 - 'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧' + 'Star crawlab-team/crawlab on GitHub': '在 GitHub 上为 Crawlab 加星吧', + 'How to buy': '如何购买' } diff --git a/frontend/src/store/modules/task.js b/frontend/src/store/modules/task.js index 59e972c66..c87c0f9f4 100644 --- a/frontend/src/store/modules/task.js +++ b/frontend/src/store/modules/task.js @@ -13,7 +13,8 @@ const state = { node_id: '', spider_id: '', status: '', - schedule_id: '' + schedule_id: '', + type: 'spider' }, // pagination pageNum: 1, @@ -161,7 +162,9 @@ const actions = { .then(response => { const data = response.data.data commit('SET_TASK_FORM', data) - dispatch('spider/getSpiderData', data.spider_id, { root: true }) + if (data.type === 'spider') { + dispatch('spider/getSpiderData', data.spider_id, { root: true }) + } if (data.node_id && data.node_id !== '000000000000000000000000') { dispatch('node/getNodeData', data.node_id, { root: true }) } @@ -174,7 +177,8 @@ const actions = { node_id: state.filter.node_id || undefined, spider_id: state.filter.spider_id || undefined, status: state.filter.status || undefined, - schedule_id: state.filter.schedule_id || undefined + schedule_id: state.filter.schedule_id || undefined, + type: state.filter.type || undefined }) .then(response => { commit('SET_TASK_LIST', response.data.data || []) @@ -234,10 +238,9 @@ const actions = { }) }, async getTaskResultExcel({ state, commit }, id) { - const { data } = await request.request('GET', - '/tasks/' + id + '/results/download', {}, { - responseType: 'blob' // important - }) + const { data } = await request.get('/tasks/' + id + '/results/download', {}, { + responseType: 'blob' // important + }) const downloadUrl = window.URL.createObjectURL(new Blob([data])) const link = document.createElement('a') diff --git a/frontend/src/utils/request.js b/frontend/src/utils/request.js index b1d1d0a95..c234e84b0 100644 --- a/frontend/src/utils/request.js +++ b/frontend/src/utils/request.js @@ -5,23 +5,23 @@ import { getToken } from '@/utils/auth' import i18n from '@/i18n' import router from '@/router' -const codeMessage = { - 200: '服务器成功返回请求的数据。', - 201: '新建或修改数据成功。', - 202: '一个请求已经进入后台排队（异步任务）。', - 204: '删除数据成功。', - 400: '发出的请求有错误，服务器没有进行新建或修改数据的操作。', - 401: '用户没有权限（令牌、用户名、密码错误）。', - 403: '用户得到授权，但是访问是被禁止的。', - 404: '发出的请求针对的是不存在的记录，服务器没有进行操作。', - 406: '请求的格式不可得。', - 410: '请求的资源被永久删除，且不会再得到的。', - 422: '当创建一个对象时，发生一个验证错误。', - 500: '服务器发生错误，请检查服务器。', - 502: '网关错误。', - 503: '服务不可用，服务器暂时过载或维护。', - 504: '网关超时。' -} +// const codeMessage = { +// 200: '服务器成功返回请求的数据。', +// 201: '新建或修改数据成功。', +// 202: '一个请求已经进入后台排队（异步任务）。', +// 204: '删除数据成功。', +// 400: '发出的请求有错误，服务器没有进行新建或修改数据的操作。', +// 401: '用户没有权限（令牌、用户名、密码错误）。', +// 403: '用户得到授权，但是访问是被禁止的。', +// 404: '发出的请求针对的是不存在的记录，服务器没有进行操作。', +// 406: '请求的格式不可得。', +// 410: '请求的资源被永久删除，且不会再得到的。', +// 422: '当创建一个对象时，发生一个验证错误。', +// 500: '服务器发生错误，请检查服务器。', +// 502: '网关错误。', +// 503: '服务不可用，服务器暂时过载或维护。', +// 504: '网关超时。' +// } /** * 异常处理程序 @@ -30,10 +30,10 @@ const errorHandler = (error) => { const { response } = error const routePath = router.currentRoute.path if (response && response.status) { - const errorText = codeMessage[response.status] || response.statusText + const errorText = response.data.error const { status } = response Message({ - message: `请求错误 ${status}: ${response.request.responseURL},${errorText}`, + message: i18n.t('Request Error') + ` ${status}: ${response.request.responseURL}. ${errorText}`, type: 'error', duration: 5 * 1000 }) diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index 9200d245e..09517882b 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -688,7 +688,7 @@ ids: this.selectedSchedules.map(d => d._id) }) if (!res.data.error) { - this.$message.success('Deleted successfully') + this.$message.success(this.$t('Deleted successfully')) this.$refs['table'].clearSelection() await this.$store.dispatch('schedule/getScheduleList') } @@ -844,7 +844,7 @@ } .table { - min-height: 360px; + min-height: 720px; margin-top: 10px; } diff --git a/frontend/src/views/setting/Setting.vue b/frontend/src/views/setting/Setting.vue index ff0e98a31..d8986e54e 100644 --- a/frontend/src/views/setting/Setting.vue +++ b/frontend/src/views/setting/Setting.vue @@ -48,9 +48,6 @@ - - - + + + + + + + + + + +
+ + {{ $t('Save') }} + +
+ + + + + { + if (!valid) return + if (this.userInfo.password !== this.userInfo.confirm_password) { + this.$message.error(this.$t('Two passwords do not match')) + return + } + if (this.userInfo.password.length < 5) { + this.$message.error(this.$t('Password length should be no shorter than 5')) + return + } + const res = await this.$request.post(`/me/change-password`, { + password: this.userInfo.password + }) + if (!res.data.error) { + this.$message.success(this.$t('Changed password successfully')) + } + }) } } }