Stage READ/WRITE support for LOAD DATA, External Table and SELECT OUTFILE

CODEOWNERS

@@ -50,6 +50,7 @@
 /pkg/common @zhangxu19830126
 /pkg/common/mpool @m-schen
 /pkg/common/malloc @reusee
+/pkg/common/spool @reusee @m-schen
 /pkg/common/async @zhangxu19830126
 /pkg/common/bitmap @aunjgr
 /pkg/common/hashmap @aunjgr

README.md

@@ -14,11 +14,11 @@
   <a href="https://www.codefactor.io/repository/github/matrixorigin/matrixone">
     <img src="https://www.codefactor.io/repository/github/matrixorigin/matrixone/badge?s=7280f4312fca2f2e6938fb8de5b726c5252541f0" alt="codefactor"/>
   </a>
-  <a href="https://docs.matrixorigin.cn/en/1.2.0/MatrixOne/Release-Notes/v1.2.0/">
-   <img src="https://img.shields.io/badge/Release-v1.2.0-green.svg" alt="release"/>
+  <a href="https://docs.matrixorigin.cn/en/1.2.1/MatrixOne/Release-Notes/v1.2.1/">
+   <img src="https://img.shields.io/badge/Release-v1.2.1-green.svg" alt="release"/>
   </a>
   <br>
-  <a href="https://docs.matrixorigin.cn/en/1.2.0/">
+  <a href="https://docs.matrixorigin.cn/en/1.2.1/">
     <b>Docs</b>
   </a>
   <b>||</b>
@@ -165,20 +165,20 @@ MatrixOne's architecture is as below:
     <img alt="MatrixOne" height="420" src="https://github.com/matrixorigin/artwork/blob/main/docs/overview/architecture/archi-en-2.png?raw=true">
 </p>
 
-For more details, you can checkout [MatrixOne Architecture Design](https://docs.matrixorigin.cn/en/1.2.0/MatrixOne/Overview/architecture/matrixone-architecture-design/).
+For more details, you can checkout [MatrixOne Architecture Design](https://docs.matrixorigin.cn/en/1.2.1/MatrixOne/Overview/architecture/matrixone-architecture-design/).
 
 ## ⚡️ <a id="quick-start">Quick start</a>
 
 ### ⚙️ Install MatrixOne
 
 MatrixOne supports Linux and MacOS. You can install MatrixOne either by [building from source](#building-from-source) or [using docker](#using-docker).
-For other installation types, please refer to [MatrixOne installation](https://docs.matrixorigin.cn/en/1.2.0/MatrixOne/Get-Started/install-standalone-matrixone/) for more details.
+For other installation types, please refer to [MatrixOne installation](https://docs.matrixorigin.cn/en/1.2.1/MatrixOne/Get-Started/install-standalone-matrixone/) for more details.
 
 **Step 1.Install Dependency**
 
 - **Building from source**
 
-1. Install Go (version 1.20 is required)
+1. Install Go (version 1.22 is required)
 
     Click <a href="https://go.dev/doc/install" target="_blank">Go Download and install</a> to enter its official documentation, and follow the installation steps to complete the **Go** installation.
 
@@ -216,7 +216,7 @@ The mo_ctl tool can be installed through the following command:
 wget https://raw.githubusercontent.com/matrixorigin/mo_ctl_standalone/main/install.sh && sudo -u $(whoami) bash +x ./install.sh
 ```
 
-See [mo_ctl Tool](https://docs.matrixorigin.cn/en/1.2.0/MatrixOne/Maintain/mo_ctl/) for complete usage details.
+See [mo_ctl Tool](https://docs.matrixorigin.cn/en/1.2.1/MatrixOne/Maintain/mo_ctl/) for complete usage details.
 
 **Step 3. Set mo_ctl parameters**
 
@@ -248,10 +248,10 @@ mo_ctl deploy main
 
 - *Option 2*: Get the MatrixOne(Stable Version)
 
-If you want to get the latest stable version code released by MatrixOne, please switch to the branch of version **1.2.0** first.
+If you want to get the latest stable version code released by MatrixOne, please switch to the branch of version **1.2.1** first.
 
 ```
-mo_ctl deploy v1.2.0
+mo_ctl deploy v1.2.1
 ```
 
 **Step 5. Launch MatrixOne server**
@@ -264,12 +264,12 @@ __Tips__: The initial startup of MatrixOne approximately takes 20 to 30 seconds.
 
 One-click connection to MatrixOne service through `mo_ctl connect` command.
 
-__Note__: The login account in the above code snippet is the initial account; please change the initial password after logging in to MatrixOne; see [Password Management](https://docs.matrixorigin.cn/en/1.2.0/MatrixOne/Security/password-mgmt/).
+__Note__: The login account in the above code snippet is the initial account; please change the initial password after logging in to MatrixOne; see [Password Management](https://docs.matrixorigin.cn/en/1.2.1/MatrixOne/Security/password-mgmt/).
 
 ## 🙌 <a id="contributing">Contributing</a>
 
 Contributions to MatrixOne are welcome from everyone.  
- See [Contribution Guide](https://docs.matrixorigin.cn/en/1.2.0/MatrixOne/Contribution-Guide/make-your-first-contribution/) for details on submitting patches and the contribution workflow.
+ See [Contribution Guide](https://docs.matrixorigin.cn/en/1.2.1/MatrixOne/Contribution-Guide/make-your-first-contribution/) for details on submitting patches and the contribution workflow.
 
 ### 👏 All contributors
 

README_CN.md

@@ -13,11 +13,11 @@
   <a href="https://www.codefactor.io/repository/github/matrixorigin/matrixone">
     <img src="https://www.codefactor.io/repository/github/matrixorigin/matrixone/badge?s=7280f4312fca2f2e6938fb8de5b726c5252541f0" alt="codefactor"/>
   </a>
-  <a href="https://docs.matrixorigin.cn/1.2.0/MatrixOne/Release-Notes/v1.2.0/">
-   <img src="https://img.shields.io/badge/Release-1.2.0-green.svg" alt="release"/>
+  <a href="https://docs.matrixorigin.cn/1.2.1/MatrixOne/Release-Notes/v1.2.1/">
+   <img src="https://img.shields.io/badge/Release-1.2.1-green.svg" alt="release"/>
   </a>
   <br>
-  <a href="https://docs.matrixorigin.cn//1.2.0/">
+  <a href="https://docs.matrixorigin.cn//1.2.1/">
     <b>Docs</b>
   </a>
   <b>||</b>
@@ -159,24 +159,24 @@ MatrixOne 是一款超融合异构分布式数据库，通过云原生化和存
 
 MatrixOne 的架构图如下图所示：
 <p align="center">
-  <img alt="MatrixOne" height="500" src="https://community-shared-data-1308875761.cos.ap-beijing.myqcloud.com/artwork/docs/overview/architecture/architecture-1.png?raw=true">
+  <img alt="MatrixOne" height="500" src="https://community-shared-data-1308875761.cos.ap-beijing.myqcloud.com/artwork/docs/Release-Notes/release-notes-1.1.0.png">
 </p>
 
-关于更详细的 MatrixOne 技术架构，可以参考[MatrixOne 架构设计](https://docs.matrixorigin.cn/1.2.0/MatrixOne/Overview/architecture/matrixone-architecture-design/)。
+关于更详细的 MatrixOne 技术架构，可以参考[MatrixOne 架构设计](https://docs.matrixorigin.cn/1.2.1/MatrixOne/Overview/architecture/matrixone-architecture-design/)。
 
 ## ⚡️ <a id="quick-start">快速上手</a>
 
 ### ⚙️ 安装 MatrixOne
 
-MatrixOne 目前支持 Linux 及 MacOS 系统，您可以通过源码安装，二进制包安装或者 docker 安装。对于更详情的安装方式请参见[MatrixOne 安装指南](https://docs.matrixorigin.cn/1.2.0/MatrixOne/Get-Started/install-standalone-matrixone/)。
+MatrixOne 目前支持 Linux 及 MacOS 系统，您可以通过源码安装，二进制包安装或者 docker 安装。对于更详情的安装方式请参见[MatrixOne 安装指南](https://docs.matrixorigin.cn/1.2.1/MatrixOne/Get-Started/install-standalone-matrixone/)。
 
 以下为您介绍通过源码部署和docker部署两种方式:
 
 **步骤 1.前置依赖**
 
 - 源码部署
 
-1. 搭建 Go 语言环境(至少需要 1.20 版本)
+1. 搭建 Go 语言环境(至少需要 1.22 版本)
 
 点击 <a href="https://go.dev/doc/install" target="_blank">Go Download and install</a> 入到 **Go** 的官方文档，按照官方指导安装步骤完成 **Go** 语言的安装。
 
@@ -214,7 +214,7 @@ __Tips__: 建议 MySQL 客户端版本为 8.0.30 版本及以上。
 wget https://raw.githubusercontent.com/matrixorigin/mo_ctl_standalone/main/install.sh && sudo -u $(whoami) bash +x ./install.sh
 ```
 
-如需获取完整的使用细节可以参考 [mo_ctl 工具指南](https://docs.matrixorigin.cn/1.2.0/MatrixOne/Maintain/mo_ctl/#_3)。
+如需获取完整的使用细节可以参考 [mo_ctl 工具指南](https://docs.matrixorigin.cn/1.2.1/MatrixOne/Maintain/mo_ctl/#_3)。
 
 **步骤 3.设置 mo_ctl 的配置参数**
 
@@ -246,10 +246,10 @@ mo_ctl set_conf MO_DEPLOY_MODE=docker #设置MatrixOne部署方式，此为docke
 
 - *选项 2*:获取 MatrixOne (稳定版本)
 
-    如果您想获得 MatrixOne 发布的最新稳定版本，请先从 **main** 切换选择至 **1.2.0** 版本分支。
+    如果您想获得 MatrixOne 发布的最新稳定版本，请先从 **main** 切换选择至 **1.2.1** 版本分支。
 
     ```
-    mo_ctl deploy 1.2.0
+    mo_ctl deploy 1.2.1
     ```
 
 **步骤 5.启动 MatrixOne 服务**
@@ -262,7 +262,7 @@ __Tips__: 首次启动 MatrixOne 大致需要花费 20 至 30 秒的时间，在
 
 通过 `mo_ctl connect` 命令一键连接 MatrixOne 服务。
 
-__Tips__: 连接和登录账号为初始账号 `root` 和密码 `111`，请在登录 MatrixOne 后及时修改初始密码，参见[密码管理](https://docs.matrixorigin.cn/1.2.0/MatrixOne/Security/password-mgmt/)。修改登录用户名或密码后重新登录同样需要通过 `mo_ctl set_conf` 的方式设置新的用户名和密码。详情可以参考 [mo_ctl 工具指南](https://docs.matrixorigin.cn/1.2.0/MatrixOne/Maintain/mo_ctl/#_3)。
+__Tips__: 连接和登录账号为初始账号 `root` 和密码 `111`，请在登录 MatrixOne 后及时修改初始密码，参见[密码管理](https://docs.matrixorigin.cn/1.2.1/MatrixOne/Security/password-mgmt/)。修改登录用户名或密码后重新登录同样需要通过 `mo_ctl set_conf` 的方式设置新的用户名和密码。详情可以参考 [mo_ctl 工具指南](https://docs.matrixorigin.cn/1.2.1/MatrixOne/Maintain/mo_ctl/#_3)。
 
 
 ### 👏贡献者
@@ -1021,7 +1021,7 @@ __Tips__: 连接和登录账号为初始账号 `root` 和密码 `111`，请在
 ## 🙌 <a id="contributing">参与贡献</a>
 
 欢迎大家对 MatrixOne 的贡献。  
-请查看[贡献指南](https://docs.matrixorigin.cn/1.2.0/MatrixOne/Contribution-Guide/make-your-first-contribution/)来了解有关提交补丁和完成整个贡献流程的详细信息。
+请查看[贡献指南](https://docs.matrixorigin.cn/1.2.1/MatrixOne/Contribution-Guide/make-your-first-contribution/)来了解有关提交补丁和完成整个贡献流程的详细信息。
 
 ## <a id="license">License</a>
 

cmd/mo-service/config.go

@@ -529,6 +529,8 @@ func (c *Config) mustGetServiceUUID() string {
 		return c.LogService.UUID
 	case metadata.ServiceType_PROXY:
 		return c.ProxyConfig.UUID
+	case metadata.ServiceType_PYTHON_UDF:
+		return c.PythonUdfServerConfig.UUID
 	}
 	panic("impossible")
 }

cmd/mo-service/debug.go

@@ -15,6 +15,9 @@
 package main
 
 import (
+	"bytes"
+	"compress/gzip"
+	"context"
 	"flag"
 	"fmt"
 	"hash/fnv"
@@ -31,14 +34,15 @@ import (
 
 	"github.com/felixge/fgprof"
 	"github.com/google/uuid"
-	"go.uber.org/zap"
-
 	"github.com/matrixorigin/matrixone/pkg/catalog"
 	"github.com/matrixorigin/matrixone/pkg/cnservice"
+	"github.com/matrixorigin/matrixone/pkg/common/malloc"
+	"github.com/matrixorigin/matrixone/pkg/fileservice"
 	"github.com/matrixorigin/matrixone/pkg/logutil"
 	"github.com/matrixorigin/matrixone/pkg/perfcounter"
 	"github.com/matrixorigin/matrixone/pkg/util/profile"
 	"github.com/matrixorigin/matrixone/pkg/util/status"
+	"go.uber.org/zap"
 )
 
 var (
@@ -416,6 +420,8 @@ func saveProfiles() {
 	saveProfile(profile.HEAP)
 	//dump goroutine before stopping services
 	saveProfile(profile.GOROUTINE)
+	// dump malloc profile
+	saveMallocProfile()
 }
 
 func saveProfile(typ string) string {
@@ -425,3 +431,32 @@ func saveProfile(typ string) string {
 	cnservice.SaveProfile(profilePath, typ, globalEtlFS)
 	return profilePath
 }
+
+func saveMallocProfile() {
+	buf := bytes.Buffer{}
+	w := gzip.NewWriter(&buf)
+	if err := malloc.WriteProfileData(w); err != nil {
+		logutil.GetGlobalLogger().Error("failed to write malloc profile", zap.Error(err))
+		return
+	}
+	if err := w.Close(); err != nil {
+		return
+	}
+	name, _ := uuid.NewV7()
+	profilePath := catalog.BuildProfilePath(globalServiceType, globalNodeId, "malloc", name.String()) + ".gz"
+	writeVec := fileservice.IOVector{
+		FilePath: profilePath,
+		Entries: []fileservice.IOEntry{
+			{
+				Offset: 0,
+				Data:   buf.Bytes(),
+				Size:   int64(len(buf.Bytes())),
+			},
+		},
+	}
+	ctx, cancel := context.WithTimeout(context.TODO(), time.Minute*3)
+	defer cancel()
+	if err := globalEtlFS.Write(ctx, writeVec); err != nil {
+		logutil.GetGlobalLogger().Error("failed to save malloc profile", zap.Error(err))
+	}
+}

etc/launch-with-python-udf-server/python-udf-server.toml

@@ -1,5 +1,6 @@
 service-type = "PYTHON_UDF"
 
 [python-udf-server]
+uuid = "dd1dccb4-4d3c-41f8-b482-5251dc7a41b2"
 address = "127.0.0.1:50051"
 path = "pkg/udf/pythonservice/pyserver"

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/cockroachdb/errors v1.9.1
 	github.com/confluentinc/confluent-kafka-go/v2 v2.4.0
+	github.com/containerd/cgroups/v3 v3.0.1
 	github.com/docker/go-units v0.5.0
 	github.com/dolthub/maphash v0.1.0
 	github.com/elastic/gosigar v0.14.2
@@ -50,8 +51,8 @@ require (
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
 	github.com/ncruces/go-dns v1.2.5
 	github.com/panjf2000/ants/v2 v2.7.4
-	github.com/parquet-go/parquet-go v0.20.1
-	github.com/pierrec/lz4/v4 v4.1.18
+	github.com/parquet-go/parquet-go v0.23.0
+	github.com/pierrec/lz4/v4 v4.1.21
 	github.com/pkg/errors v0.9.1
 	github.com/plar/go-adaptive-radix-tree v1.0.5
 	github.com/prashantv/gostub v1.1.0
@@ -72,20 +73,19 @@ require (
 	go.uber.org/zap v1.24.0
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
 	golang.org/x/sync v0.6.0
-	golang.org/x/sys v0.20.0
+	golang.org/x/sys v0.21.0
 	gonum.org/v1/gonum v0.14.0
 	google.golang.org/grpc v1.62.1
-	google.golang.org/protobuf v1.33.0
+	google.golang.org/protobuf v1.34.2
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/K-Phoen/sdk v0.12.3 // indirect
-	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cilium/ebpf v0.9.1 // indirect
-	github.com/containerd/cgroups/v3 v3.0.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
@@ -154,7 +154,7 @@ require (
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jtolds/gls v4.20.0+incompatible // indirect
-	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect