Realize storage api

.dockerignore

@@ -0,0 +1,25 @@
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/.idea
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

.github/workflows/dotnet-build.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Setup .NET
       uses: actions/setup-dotnet@v3
       with:
-        dotnet-version: 7.0.x
+        dotnet-version: 8.0.x
     - name: Fetch submodules
       run: git submodule update --init --recursive
     - name: Restore dependencies

Mocha.sln

@@ -16,12 +16,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mocha.Core.Tests", "tests\M
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{71D20701-E657-443F-8000-7958FB5E6BD7}"
 	ProjectSection(SolutionItems) = preProject
+		.dockerignore = .dockerignore
 		.editorconfig = .editorconfig
 		.gitignore = .gitignore
 		.gitmodules = .gitmodules
 		LICENSE = LICENSE
 		README.md = README.md
 		.github\workflows\dotnet-build.yml = .github\workflows\dotnet-build.yml
+		README.zh-CN.md = README.zh-CN.md
 	EndProjectSection
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mocha.Storage", "src\Mocha.Storage\Mocha.Storage.csproj", "{8EEB6697-B975-430D-9CC3-3048E76C5ECA}"
@@ -30,6 +32,30 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mocha.Core.Benchmarks", "te
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mocha.Storage.Tests", "tests\Mocha.Storage.Tests\Mocha.Storage.Tests.csproj", "{FC0D810E-4ACC-4567-95D8-D7F617E412FE}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mocha.Query.Jaeger", "src\Mocha.Query.Jaeger\Mocha.Query.Jaeger.csproj", "{DC281C3B-455F-4391-92EF-D5D99FC2B9AA}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{53AF2923-4CB8-44C8-885B-B0EEB8574FEB}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "mysql", "mysql", "{1DCDFC33-1401-4CCA-AAAE-FC150AD147F5}"
+	ProjectSection(SolutionItems) = preProject
+		scripts\mysql\init.sql = scripts\mysql\init.sql
+	EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docker", "docker", "{D598862A-999C-40FD-A190-EBD00376D077}"
+	ProjectSection(SolutionItems) = preProject
+		docker\docker-compose.yml = docker\docker-compose.yml
+	EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "distributor", "distributor", "{959DCB4E-A070-4C66-A27F-D83CB933F0D8}"
+	ProjectSection(SolutionItems) = preProject
+		docker\distributor\Dockerfile = docker\distributor\Dockerfile
+	EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "jaeger-query", "jaeger-query", "{C7222A9C-C50C-4FF0-A02D-778A9BB4DD2C}"
+	ProjectSection(SolutionItems) = preProject
+		docker\jaeger-query\Dockerfile = docker\jaeger-query\Dockerfile
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -44,6 +70,10 @@ Global
 		{8EEB6697-B975-430D-9CC3-3048E76C5ECA} = {6983D239-07DA-4DFA-9AAA-F6876029FF8D}
 		{2107E75D-9717-4CCD-BE85-713BEF75366A} = {24F9E34A-D92A-4C0A-851F-1E864181BF97}
 		{FC0D810E-4ACC-4567-95D8-D7F617E412FE} = {24F9E34A-D92A-4C0A-851F-1E864181BF97}
+		{DC281C3B-455F-4391-92EF-D5D99FC2B9AA} = {6983D239-07DA-4DFA-9AAA-F6876029FF8D}
+		{1DCDFC33-1401-4CCA-AAAE-FC150AD147F5} = {53AF2923-4CB8-44C8-885B-B0EEB8574FEB}
+		{959DCB4E-A070-4C66-A27F-D83CB933F0D8} = {D598862A-999C-40FD-A190-EBD00376D077}
+		{C7222A9C-C50C-4FF0-A02D-778A9BB4DD2C} = {D598862A-999C-40FD-A190-EBD00376D077}
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{DCA600F0-4D6C-44DA-A493-F63097CCE74E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
@@ -78,5 +108,9 @@ Global
 		{FC0D810E-4ACC-4567-95D8-D7F617E412FE}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{FC0D810E-4ACC-4567-95D8-D7F617E412FE}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{FC0D810E-4ACC-4567-95D8-D7F617E412FE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DC281C3B-455F-4391-92EF-D5D99FC2B9AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DC281C3B-455F-4391-92EF-D5D99FC2B9AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DC281C3B-455F-4391-92EF-D5D99FC2B9AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DC281C3B-455F-4391-92EF-D5D99FC2B9AA}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

README.md

@@ -1,7 +1,18 @@
-# mocha
+Mocha
+=====
+
+[![codecov](https://codecov.io/gh/dotnetcore/mocha/branch/main/graph/badge.svg?token=v9OE7dV8ZS)](https://codecov.io/gh/dotnetcore/mocha)
+
+English | [简体中文](./README.zh-CN.md)
+
 Mocha is an application performance monitor tools based on [OpenTelemetry](https://opentelemetry.io), which also provides a scalable platform for observability data analysis and storage.
 
-# functional architecture
+## Quick Start
+In the beta phase, we provide a Docker Compose file for users to experience our system locally.
+
++ [Quick Start](./docs/quick-start/docker-compose/quick-start.en-US.md)
+
+## Functional Architecture
 ![](./docs/assets/functional_architecture.png)
 
 The set of features that Mocha will provide:
@@ -21,7 +32,7 @@ The set of features that Mocha will provide:
   - Alert notifications
 - Metrics/Logs/Traces data explore
 
-# technical architecture
+## Technical Architecture
 ![](./docs/assets/technical_architecture.png)
 
 The components of Mocha are as follows:
@@ -32,8 +43,8 @@ The components of Mocha are as follows:
 - Mocha Manager : Consisting of a manager server, dashboard, and ETCD for cluster metadata and data analysis rules storage.
 - OTel SDK / Collector : Open-source OpenTelemetry collection kits
 
-# contribute
+## Contribute
 One of the easiest ways to contribute is to participate in discussions and discuss issues. You can also contribute by submitting pull requests with code changes.
 
-# license
-Mocha is under the MIT license. See the [LICENSE](LICENSE) file for details.
+## License
+Mocha is under the MIT license. See the [LICENSE](LICENSE) file for details.

README.zh-CN.md

@@ -0,0 +1,50 @@
+Mocha
+=====
+
+[![codecov](https://codecov.io/gh/dotnetcore/mocha/branch/main/graph/badge.svg?token=v9OE7dV8ZS)](https://codecov.io/gh/dotnetcore/mocha)
+
+[English](./README.md) | 简体中文
+
+Mocha 是一个基于 [OpenTelemetry](https://opentelemetry.io) 的 APM 系统，同时提供可伸缩的可观测性数据分析和存储平台。
+
+## 快速开始
+现阶段，我们提供了 Docker Compose 文件，方便用户在本地体验我们的系统。
+
++ [快速开始](./docs/quick-start/docker-compose/quick-start.zh-CN.md)
+
+# 平台功能
+![](./docs/assets/functional_architecture.png)
+Mocha 将要提供的功能集合：
+- APM 和 分布式追踪
+    - 服务概览、R.E.D 指标和可用性监控
+    - 服务拓扑
+    - 调用监控，包括 HTTP、RPC、Cache、DB、MQ 等
+    - 调用链路查询和检索
+- 基础设施监控
+    - 主机监控
+    - 容器和 Kubernetes 监控
+    - 主流中间件监控
+- 日志
+    - 日志查询
+    - 日志聚合分析
+- 报警
+    - 报警规则管理
+    - 报警通知
+- M.T.L 数据探索 [Data Explore / Inspect]
+
+# 技术架构
+![](./docs/assets/technical_architecture.png)
+
+Mocha 整体架构由下面的部分组成
+- Mocha Distributor Cluster：作为 mocha 系统的数据入口，负责接收 OTel SDK 和 Collector 上报的数据，并通过一致性Hash 将数据路由到对应的 aggregator 节点上。为了保证数据不丢失，最终 Distributor 应该具备本地 FIFO 队列的能力。
+- Mocha Streaming Cluster：mocha 的核心组件，通过读取预配置或者用户配置的 aggr rule dsl 生成对应的 streaming data flow 并执行。Streaming 是具备分布式 shuffle 的能力的有状态组件，需要将自身信息注册到ETCD中。
+- Storage：mocha M.T.L 存储，可以选用开源存储组件，如 ClickHouse、ElasticSearch、victoriametrics 等。
+- Mocha Querier + Grafana: 从存储查询数据并提供给 grafana 做展示。因此要兼容 promql / jeager / loki 等数据源的查询。
+- Mocha Manager : 包括 manager server、dashboard和ETCE组件，集群元数据和 M.T.L 数据分析规则存储。
+- OTel SDK / Collector : 开源 OpenTelemetry 采集套件。
+
+## 参与贡献
+参与贡献的最简单的方式是参与讨论并讨论问题。您也可以通过提交代码更改的拉取请求来进行贡献。
+
+## 许可证
+Mocha 是在 MIT 许可下发布的。有关详细信息，请参阅 [LICENSE](LICENSE) 文件。

docker/distributor/Dockerfile

@@ -0,0 +1,25 @@
+FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
+USER $APP_UID
+WORKDIR /app
+EXPOSE 4319
+
+FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
+ARG BUILD_CONFIGURATION=Release
+WORKDIR /src
+COPY ["src/Mocha.Distributor/Mocha.Distributor.csproj", "src/Mocha.Distributor/"]
+COPY ["src/Mocha.Core/Mocha.Core.csproj", "src/Mocha.Core/"]
+COPY ["src/Mocha.Protocol.Generated/Mocha.Protocol.Generated.csproj", "src/Mocha.Protocol.Generated/"]
+COPY ["src/Mocha.Storage/Mocha.Storage.csproj", "src/Mocha.Storage/"]
+RUN dotnet restore "src/Mocha.Distributor/Mocha.Distributor.csproj"
+COPY . .
+WORKDIR "/src/src/Mocha.Distributor"
+RUN dotnet build "Mocha.Distributor.csproj" -c $BUILD_CONFIGURATION -o /app/build
+
+FROM build AS publish
+ARG BUILD_CONFIGURATION=Release
+RUN dotnet publish "Mocha.Distributor.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false
+
+FROM base AS final
+WORKDIR /app
+COPY --from=publish /app/publish .
+ENTRYPOINT ["dotnet", "Mocha.Distributor.dll"]

docker/docker-compose.yml

@@ -0,0 +1,81 @@
+# Licensed to the .NET Core Community under one or more agreements.
+# The .NET Core Community licenses this file to you under the MIT license.
+
+version: "3.8"
+
+name: mocha
+
+services:
+  mysql:
+    image: mysql:8.2.0
+    container_name: mocha-mysql
+    environment:
+      MYSQL_ROOT_PASSWORD: root
+      MYSQL_DATABASE: mocha
+      MYSQL_USER: mocha
+      MYSQL_PASSWORD: mocha
+    ports:
+      - "3306:3306"
+    expose:
+      - "3306"
+    volumes:
+      - ../scripts/mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
+#      - ./mysql:/var/lib/mysql
+    restart: always
+    networks:
+      - mocha
+
+  grafana:
+    image: grafana/grafana:10.0.10
+    container_name: mocha-grafana
+    ports:
+      - "3000:3000"
+#    volumes:
+#      - ./grafana:/var/lib/grafana
+    restart: always
+    networks:
+      - mocha
+
+  distributor:
+    build:
+      context: ..
+      dockerfile: ./docker/distributor/Dockerfile
+    container_name: mocha-distributor
+    ports:
+      - "4317:4317"
+    expose:
+        - "4317"
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Production
+      - ConnectionStrings__EF=server=mysql;port=3306;database=mocha;userid=root;password=root
+      - Logging__LogLevel__Microsoft.EntityFrameworkCore=Warning
+    depends_on:
+      - mysql
+    restart: always
+    networks:
+      - mocha
+
+  jaeger-query:
+    build:
+      context: ..
+      dockerfile: ./docker/jaeger-query/Dockerfile
+    container_name: mocha-jaeger-query
+    ports:
+      - "5775:5775"
+    expose:
+      - "5775"
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Production
+      - ConnectionStrings__EF=server=mysql;port=3306;database=mocha;userid=mocha;password=mocha
+      - Logging__LogLevel__Microsoft.EntityFrameworkCore=Warning
+    depends_on:
+      - mysql
+    restart: always
+    networks:
+      - mocha
+
+networks:
+  mocha:
+    driver: bridge
+
+

docker/jaeger-query/Dockerfile

@@ -0,0 +1,24 @@
+FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
+WORKDIR /app
+EXPOSE 8080
+
+FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
+ARG BUILD_CONFIGURATION=Release
+WORKDIR /src
+COPY ["src/Mocha.Query.Jaeger/Mocha.Query.Jaeger.csproj", "src/Mocha.Query.Jaeger/"]
+COPY ["src/Mocha.Core/Mocha.Core.csproj", "src/Mocha.Core/"]
+COPY ["src/Mocha.Protocol.Generated/Mocha.Protocol.Generated.csproj", "src/Mocha.Protocol.Generated/"]
+COPY ["src/Mocha.Storage/Mocha.Storage.csproj", "src/Mocha.Storage/"]
+RUN dotnet restore "src/Mocha.Query.Jaeger/Mocha.Query.Jaeger.csproj"
+COPY . .
+WORKDIR "/src/src/Mocha.Query.Jaeger"
+RUN dotnet build "Mocha.Query.Jaeger.csproj" -c $BUILD_CONFIGURATION -o /app/build
+
+FROM build AS publish
+ARG BUILD_CONFIGURATION=Release
+RUN dotnet publish "Mocha.Query.Jaeger.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false
+
+FROM base AS final
+WORKDIR /app
+COPY --from=publish /app/publish .
+ENTRYPOINT ["dotnet", "Mocha.Query.Jaeger.dll"]

docs/quick-start/docker-compose/quick-start.en-US.md

@@ -0,0 +1,53 @@
+## Start the Project
+
+Execute the following command in the docker directory under the project root directory to start the project:
+
+```bash
+docker-compose up -d
+```
+
+After the startup is successful, you can see the following containers:
+
++ distributor: Provides gRPC API for receiving OTLP data
++ jaeger-query: Provides HTTP API for receiving Jaeger query protocol
++ mysql: Used to store data
++ grafana: Used to display data
+
+## Send Trace Data
+
+Configure the OTLP exporter of the SDK as `http://localhost:4317` to send data to the distributor.
+
+## Configure Jaeger Data Source
+
+We have implemented an API that supports the Jaeger query protocol, so you can configure the Jaeger data source directly in Grafana.
+
+Visit http://localhost:3000/ to see the grafana login page. Both the username and password are admin.
+
+After logging in, click the menu on the left, select Data Sources, and then click Add data source.
+
+![](./asserts/add-jaeger-data-source.png)
+
+![](./asserts/add-jaeger-data-source-2.png)
+
+Select Jaeger.
+![](./asserts/add-jaeger-data-source-3.png)
+
+Configure the URL of the Jaeger data source as `http://jaeger-query:5775`.
+
+![](./asserts/add-jaeger-data-source-4.png)
+
+Click Save & Test. If the following information is displayed, the configuration is successful.
+
+![](./asserts/add-jaeger-data-source-5.png)
+
+If no data has been sent to the distributor yet, the following warning message will be displayed.
+
+![](./asserts/add-jaeger-data-source-warning.png)
+
+## Query Trace Data
+
+Click the menu on the left, select Explore, and then select the Jaeger data source to see the Trace data.
+
+![](./asserts/query-trace.png)
+
+![](./asserts/query-trace-2.png)