Document remaining opensearch compatible apis, manager apis, Admin UI (…

…#869) * Document remaining opensearch compatible apis, manager apis, Admin UI * Add aggregations reference * Expand logs documentation, app installation steps * Expand logs, traces, and migration documentation & site configs --------- Co-authored-by: Bryan Burkholder <[email protected]>
slackhq · Apr 18, 2024 · 6f5d83d · 6f5d83d
1 parent 53ae18c
commit 6f5d83d
Show file tree

Hide file tree

Showing 26 changed files with 1,761 additions and 59 deletions.
diff --git a/docs/api/manager_api.yaml b/docs/api/manager_api.yaml
@@ -92,6 +92,9 @@ paths:
   /slack.proto.astra.ManagerApiService/RestoreReplica:
     post:
       summary: RestoreReplica
+      description: |
+        <tldr>experimental</tldr>
+        Restores replicas matching provided query parameters.
       tags:
         - ManagerApiService
       operationId: ManagerApiService_RestoreReplica
@@ -123,6 +126,9 @@ paths:
                 $ref: '#/components/schemas/Status'
   /slack.proto.astra.ManagerApiService/RestoreReplicaIds:
     summary: RestoreReplicaIds
+    description: |
+      <tldr>experimental</tldr>
+      Restores replicas by ID
     post:
       tags:
         - ManagerApiService
@@ -192,8 +198,6 @@ paths:
         UpdatePartitionAssignment allocates a new partition assignment for a dataset. If a rate and a list of
          partition_ids are provided, it will use it use the list of partition ids as the current allocation
          and invalidates the existing assignment.
-        
-         In the future if only a rate is provided with an empty list the allocation will be automatically assigned.
       operationId: ManagerApiService_UpdatePartitionAssignment
       parameters:
         - name: name

diff --git a/docs/api/preprocessor_api.yaml b/docs/api/preprocessor_api.yaml
@@ -0,0 +1,99 @@
+openapi: 3.0.0
+info:
+  title: Preprocessor Node API
+  version: alpha
+paths:
+  /_bulk:
+    post:
+      summary: Bulk ingest
+      description: |
+        Adds multiple docs to an index in a single request.
+      requestBody:
+        description: Bulk ingest
+        required: true
+        content:
+          application/ndjson:
+            schema:
+              type: object
+              properties:
+                '{action_metadata}':
+                  type: object
+                  description: |
+                    Newline delineated JSON for setting action & metadata.
+                    Supports `index` and `create` actions, does not support `update` or `delete`.
+                  required: true
+                  properties:
+                    index:
+                      type: object
+                      properties:
+                        _index:
+                          type: string
+                          required: true
+                          description: Index name
+                        _id:
+                          type: string
+                          required: false
+                          description: Document ID, if no ID is provided one will be generated
+                    create:
+                      type: object
+                      properties:
+                        _index:
+                          type: string
+                          required: true
+                          description: Index name
+                        _id:
+                          type: string
+                          required: false
+                          description: Document ID, if no ID is provided one will be generated
+                '{document}':
+                  type: object
+                  description: "Newline delineated JSON for setting document"
+                  required: false
+                  properties:
+                    '{fieldname}':
+                      type: object
+                      description: Key/value pairs to add to document
+
+
+      responses:
+        '200':
+          description: Bulk ingest response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  totalDocs:
+                    type: integer
+                  failedDocs:
+                    type: integer
+                  errorMsg:
+                    type: string
+        '400':
+          summary: Foo
+          description: |
+            Rate limit exceeded (configurable)
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  totalDocs:
+                    type: integer
+                  failedDocs:
+                    type: integer
+                  errorMsg:
+                    type: string
+        '500':
+          description: Internal server error
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  totalDocs:
+                    type: integer
+                  failedDocs:
+                    type: integer
+                  errorMsg:
+                    type: string
diff --git a/docs/api/query_api.yaml b/docs/api/query_api.yaml
@@ -7,6 +7,8 @@ paths:
   /:
     get:
       summary: Cluster metadata
+      description: |
+        Describes the cluster version and supported Lucene version. Used for cluster feature discovery.
       responses:
         '200':
           description: Cluster metadata
@@ -27,12 +29,12 @@ paths:
                       lucene_version:
                         description: "Lucene index/query version of cluster"
                         type: string
-                    #example:
-                    #  distribution: astra
-                    #  number: 0.0.1
-                    #  lucene_version: 9.7.0
   /_msearch:
     post:
+      summary: Bulk search
+      description: | 
+        Primary search API for the Astra cluster. Supports multiple queries in a single request by adding additional 
+        header and body pairs.
       requestBody:
         description: Bulk search
         required: true
@@ -94,16 +96,47 @@ paths:
                       properties:
                         '{name}':
                           type: object
-                          description: "Keyed by name of requested aggregation"
+                          description: |
+                            Keyed by name of requested aggregation. See aggregations docs for full list of supported
+                            aggregation requests.
                           properties:
                             aggs:
                               type: object
                               description: "Nested aggregations"
+                            auto_date_histogram:
+                              type: object
+                            avg:
+                              type: object
+                            cumulative_sum:
+                              type: object
                             date_histogram:
                               type: object
+                            derivative:
+                              type: object
+                            extended_stats:
+                              type: object
+                            filters:
+                              type: object
+                            histogram:
+                              type: object
+                            max:
+                              type: object
+                            min:
+                              type: object
+                            moving_avg:
+                              type: object
+                            moving_fn:
+                              type: object
+                            percentiles:
+                              type: object
+                            sum:
+                              type: object
                             terms:
                               type: object
-
+                            cardinality:
+                              type: object
+                            value_count:
+                              type: object
       responses:
         '200':
           description: Bulk search results
@@ -172,7 +205,7 @@ paths:
           schema:
             type: string
           required: true
-          description: Index name as stored by cluster
+          description: Index name as set in cluster
       responses:
         '200':
           description: Index field mappings

diff --git a/docs/assets/images/astra_app_explore.png b/docs/assets/images/astra_app_explore.png
diff --git a/docs/assets/images/astra_app_settings.png b/docs/assets/images/astra_app_settings.png
diff --git a/docs/assets/images/astra_datasource.png b/docs/assets/images/astra_datasource.png
diff --git a/docs/assets/images/astra_datasource_link.png b/docs/assets/images/astra_datasource_link.png
diff --git a/docs/assets/images/astra_datasource_link_demo.png b/docs/assets/images/astra_datasource_link_demo.png
diff --git a/docs/assets/images/astra_grafana_trace.png b/docs/assets/images/astra_grafana_trace.png
diff --git a/docs/assets/images/docservice.png b/docs/assets/images/docservice.png
diff --git a/docs/assets/images/docservice_after_response.png b/docs/assets/images/docservice_after_response.png
diff --git a/docs/assets/images/docservice_error.png b/docs/assets/images/docservice_error.png
diff --git a/docs/assets/images/docservice_no_response.png b/docs/assets/images/docservice_no_response.png
diff --git a/docs/assets/images/es_compatibility_mode.png b/docs/assets/images/es_compatibility_mode.png
diff --git a/docs/astra.tree b/docs/astra.tree
@@ -10,11 +10,13 @@
     <toc-element topic="Using-Astra.topic">
         <toc-element topic="Logs.md"/>
         <toc-element topic="Traces.md"/>
+        <toc-element topic="Aggregations.md"/>
         <toc-element topic="API-opensearch.md"/>
         <toc-element topic="API-zipkin.md"/>
         <toc-element topic="Migrating.md"/>
     </toc-element>
     <toc-element toc-title="Operators &amp; admins">
+        <toc-element topic="Admin-UI.md"/>
         <toc-element topic="System-properties-reference.md"/>
         <toc-element topic="Config-options.md"/>
         <toc-element topic="Troubleshooting.md"/>

diff --git a/docs/cfg/buildprofiles.xml b/docs/cfg/buildprofiles.xml
@@ -14,7 +14,12 @@
             <showDownloadButton>true</showDownloadButton>
             <offline-docs>true</offline-docs>
             <noindex-content>false</noindex-content>
+            <include-in-head>include_head.html</include-in-head>
+            <enable-browser-edits>true</enable-browser-edits>
+            <browser-edits-url>https://github.com/slackhq/astra/edit/master/docs/</browser-edits-url>
+            <web-root>https://slackhq.github.io/astra/</web-root>
         </variables>
+        <sitemap priority="0.35" change-frequency="monthly"/>
     </build-profile>
     <footer>
         <copyright>Slack 2024</copyright>

diff --git a/docs/cfg/include_head.html b/docs/cfg/include_head.html
@@ -0,0 +1 @@
+<meta name="google-site-verification" content="1si6dlrl0APt5FHnbiuUUcCbCS2FiMHvYrzRUVnnp8M" />
diff --git a/docs/topics/API-manager.md b/docs/topics/API-manager.md
@@ -62,52 +62,127 @@ API definitions for manager nodes, accessed via manager Docs service and admin t
         </response>
     </api-endpoint>
     <api-endpoint endpoint="/slack.proto.astra.ManagerApiService/ListDatasetMetadata" method="POST">
-        <request>
-            <sample lang="JSON">
-            </sample>
-        </request>
         <response type="200">
             <sample lang="JSON">
+                {
+                  "datasetMetadata": [
+                    {
+                      "name": "example",
+                      "owner": "example",
+                      "throughputBytes": "4000000",
+                      "partitionConfigs": [
+                        {
+                          "startTimeEpochMs": "1698967727980",
+                          "endTimeEpochMs": "1698969132808",
+                          "partitions": [
+                            "0"
+                          ]
+                        },
+                        {
+                          "startTimeEpochMs": "1698969132809",
+                          "endTimeEpochMs": "1699035709531",
+                          "partitions": [
+                            "0"
+                          ]
+                        },
+                        {
+                          "startTimeEpochMs": "1699035709532",
+                          "endTimeEpochMs": "9223372036854775807",
+                          "partitions": [
+                            "0",
+                            "1"
+                          ]
+                        }
+                      ],
+                      "serviceNamePattern": "_all"
+                    }
+                  ]
+                }
             </sample>
         </response>
     </api-endpoint>
     <api-endpoint endpoint="/slack.proto.astra.ManagerApiService/RestoreReplica" method="POST">
         <request> 
             <sample lang="JSON">
+                {
+                  "serviceName": "example",
+                  "startTimeEpochMs": "1713375159221",
+                  "endTimeEpochMs": "1713378759221"
+                }
             </sample>
         </request>
         <response type="200">
-            <sample lang="JSON"> 
+            <sample lang="JSON">
+                {
+                    "status": "success"
+                }
             </sample>
         </response>
     </api-endpoint>
     <api-endpoint endpoint="/slack.proto.astra.ManagerApiService/RestoreReplicaIds" method="POST">
         <request>
             <sample lang="JSON">
+                {
+                  "idsToRestore": [
+                    "f3a94795-71a1-4735-9eb1-60968fbae196"
+                  ]
+                }
             </sample>
         </request>
         <response type="200">
             <sample lang="JSON">
+                {
+                    "status": "success"
+                }
             </sample>
         </response>
     </api-endpoint>
     <api-endpoint endpoint="/slack.proto.astra.ManagerApiService/UpdateDatasetMetadata" method="POST">
         <request>
             <sample lang="JSON">
+                {
+                  "name": "example",
+                  "owner": "Updated owner",
+                  "serviceNamePattern": "_all"
+                }
             </sample>
         </request>
         <response type="200">
             <sample lang="JSON">
+                {
+                  "name": "example",
+                  "owner": "Updated owner",
+                  "serviceNamePattern": "_all"
+                }
             </sample>
         </response>
     </api-endpoint>
     <api-endpoint endpoint="/slack.proto.astra.ManagerApiService/UpdatePartitionAssignment" method="POST">
         <request>
-            <sample lang="JSON">
+            <sample lang="JSON" title="Complete update">
+                {
+                  "name": "example",
+                  "throughputBytes": "4000000",
+                  "partitionIds": [
+                    "0"
+                  ]
+                }
+            </sample>
+            <sample lang="JSON" title="Throughput only">
+                {
+                  "name": "example",
+                  "throughputBytes": "4000000",
+                  "partitionIds": []
+                }
             </sample>
         </request>
         <response type="200">
             <sample lang="JSON">
+                {
+                  "assignedPartitionIds": [
+                    "0"
+                  ]
+                }
             </sample>
         </response>
     </api-endpoint>