Merge pull request #1040 from SiaFoundation/chris/merge-master

Merge dev -> master

.github/workflows/codeql.yml

@@ -0,0 +1,84 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "dev", "master" ]
+  pull_request:
+    branches: [ "dev", "master" ]
+  schedule:
+    - cron: '22 22 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners
+    # Consider using larger runners for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ]
+        # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #     echo "Run, Build Application using script"
+    #     ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/test.yml

@@ -47,10 +47,20 @@ jobs:
         uses: n8maninger/action-golang-test@v1
         with:
           args: "-race;-short"
+      - name: Test Stores - MySQL
+        if: matrix.os == 'ubuntu-latest'
+        uses: n8maninger/action-golang-test@v1
+        env:
+          RENTERD_DB_URI: 127.0.0.1:3800
+          RENTERD_DB_USER: root
+          RENTERD_DB_PASSWORD: test
+        with:
+          package: "./stores"
+          args: "-race;-short"
       - name: Test Integration
         uses: n8maninger/action-golang-test@v1
         with:
-          package: "./internal/testing/..."
+          package: "./internal/test/e2e/..."
           args: "-failfast;-race;-tags=testing;-timeout=30m"
       - name: Test Integration - MySQL
         if: matrix.os == 'ubuntu-latest'
@@ -60,7 +70,7 @@ jobs:
           RENTERD_DB_USER: root
           RENTERD_DB_PASSWORD: test
         with:
-          package: "./internal/testing/..."
+          package: "./internal/test/e2e/..."
           args: "-failfast;-race;-tags=testing;-timeout=30m"
       - name: Build
         run: go build -o bin/ ./cmd/renterd

README.md

@@ -214,14 +214,14 @@ setting does not have a default value, it can be updated using the settings API:
 
 In most cases the default set should match the set from your autopilot
 configuration in order for migrations to work properly. The contract set can be
-overriden by passing it as a query string parameter to the worker's upload and
+overridden by passing it as a query string parameter to the worker's upload and
 migrate endpoints.
 
 - `PUT /api/worker/objects/foo?contractset=foo`
 
 ### Redundancy
 
-The default redundancy on mainnet is 30-10, on testnet it is 6-2. The redunancy
+The default redundancy on mainnet is 30-10, on testnet it is 6-2. The redundancy
 can be updated using the settings API:
 
 - `GET /api/bus/setting/redundancy`

alerts/alerts.go

@@ -35,6 +35,7 @@ const (
 
 type (
 	Alerter interface {
+		Alerts(_ context.Context, opts AlertsOpts) (resp AlertsResponse, err error)
 		RegisterAlert(_ context.Context, a Alert) error
 		DismissAlerts(_ context.Context, ids ...types.Hash256) error
 	}
@@ -63,8 +64,29 @@ type (
 		alerts             map[types.Hash256]Alert
 		webhookBroadcaster webhooks.Broadcaster
 	}
+
+	AlertsOpts struct {
+		Offset   int
+		Limit    int
+		Severity Severity
+	}
+
+	AlertsResponse struct {
+		Alerts  []Alert `json:"alerts"`
+		HasMore bool    `json:"hasMore"`
+		Totals  struct {
+			Info     int `json:"info"`
+			Warning  int `json:"warning"`
+			Error    int `json:"error"`
+			Critical int `json:"critical"`
+		} `json:"totals"`
+	}
 )
 
+func (ar AlertsResponse) Total() int {
+	return ar.Totals.Info + ar.Totals.Warning + ar.Totals.Error + ar.Totals.Critical
+}
+
 // String implements the fmt.Stringer interface.
 func (s Severity) String() string {
 	switch s {
@@ -81,15 +103,8 @@ func (s Severity) String() string {
 	}
 }
 
-// MarshalJSON implements the json.Marshaler interface.
-func (s Severity) MarshalJSON() ([]byte, error) {
-	return []byte(fmt.Sprintf(`%q`, s.String())), nil
-}
-
-// UnmarshalJSON implements the json.Unmarshaler interface.
-func (s *Severity) UnmarshalJSON(b []byte) error {
-	status := strings.Trim(string(b), `"`)
-	switch status {
+func (s *Severity) LoadString(str string) error {
+	switch str {
 	case severityInfoStr:
 		*s = SeverityInfo
 	case severityWarningStr:
@@ -99,11 +114,21 @@ func (s *Severity) UnmarshalJSON(b []byte) error {
 	case severityCriticalStr:
 		*s = SeverityCritical
 	default:
-		return fmt.Errorf("unrecognized severity: %v", status)
+		return fmt.Errorf("unrecognized severity: %v", str)
 	}
 	return nil
 }
 
+// MarshalJSON implements the json.Marshaler interface.
+func (s Severity) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`%q`, s.String())), nil
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface.
+func (s *Severity) UnmarshalJSON(b []byte) error {
+	return s.LoadString(strings.Trim(string(b), `"`))
+}
+
 // RegisterAlert implements the Alerter interface.
 func (m *Manager) RegisterAlert(ctx context.Context, alert Alert) error {
 	if alert.ID == (types.Hash256{}) {
@@ -158,19 +183,46 @@ func (m *Manager) DismissAlerts(ctx context.Context, ids ...types.Hash256) error
 	})
 }
 
-// Active returns the host's active alerts.
-func (m *Manager) Active() []Alert {
+// Alerts returns the host's active alerts.
+func (m *Manager) Alerts(_ context.Context, opts AlertsOpts) (AlertsResponse, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
+	offset, limit := opts.Offset, opts.Limit
+	resp := AlertsResponse{}
+
+	if offset >= len(m.alerts) {
+		return resp, nil
+	} else if limit == -1 {
+		limit = len(m.alerts)
+	}
+
 	alerts := make([]Alert, 0, len(m.alerts))
 	for _, a := range m.alerts {
+		if a.Severity == SeverityInfo {
+			resp.Totals.Info++
+		} else if a.Severity == SeverityWarning {
+			resp.Totals.Warning++
+		} else if a.Severity == SeverityError {
+			resp.Totals.Error++
+		} else if a.Severity == SeverityCritical {
+			resp.Totals.Critical++
+		}
+		if opts.Severity != 0 && a.Severity != opts.Severity {
+			continue // filter by severity
+		}
 		alerts = append(alerts, a)
 	}
 	sort.Slice(alerts, func(i, j int) bool {
 		return alerts[i].Timestamp.After(alerts[j].Timestamp)
 	})
-	return alerts
+	alerts = alerts[offset:]
+	if limit < len(alerts) {
+		alerts = alerts[:limit]
+		resp.HasMore = true
+	}
+	resp.Alerts = alerts
+	return resp, nil
 }
 
 func (m *Manager) RegisterWebhookBroadcaster(b webhooks.Broadcaster) {
@@ -204,6 +256,11 @@ func WithOrigin(alerter Alerter, origin string) Alerter {
 	}
 }
 
+// Alerts implements the Alerter interface.
+func (a *originAlerter) Alerts(ctx context.Context, opts AlertsOpts) (resp AlertsResponse, err error) {
+	return a.alerter.Alerts(ctx, opts)
+}
+
 // RegisterAlert implements the Alerter interface.
 func (a *originAlerter) RegisterAlert(ctx context.Context, alert Alert) error {
 	if alert.Data == nil {

api/autopilot.go

@@ -91,9 +91,37 @@ type (
 		StartTime TimeRFC3339 `json:"startTime"`
 		BuildState
 	}
-)
 
-type (
+	ConfigEvaluationRequest struct {
+		AutopilotConfig    AutopilotConfig    `json:"autopilotConfig"`
+		GougingSettings    GougingSettings    `json:"gougingSettings"`
+		RedundancySettings RedundancySettings `json:"redundancySettings"`
+	}
+
+	ConfigRecommendation struct {
+		GougingSettings GougingSettings `json:"gougingSettings"`
+	}
+
+	// ConfigEvaluationResponse is the response type for /evaluate
+	ConfigEvaluationResponse struct {
+		Hosts    uint64 `json:"hosts"`
+		Usable   uint64 `json:"usable"`
+		Unusable struct {
+			Blocked uint64 `json:"blocked"`
+			Gouging struct {
+				Contract uint64 `json:"contract"`
+				Download uint64 `json:"download"`
+				Gouging  uint64 `json:"gouging"`
+				Pruning  uint64 `json:"pruning"`
+				Upload   uint64 `json:"upload"`
+			} `json:"gouging"`
+			NotAcceptingContracts uint64 `json:"notAcceptingContracts"`
+			NotScanned            uint64 `json:"notScanned"`
+			Unknown               uint64 `json:"unknown"`
+		}
+		Recommendation *ConfigRecommendation `json:"recommendation,omitempty"`
+	}
+
 	// HostHandlerResponse is the response type for the /host/:hostkey endpoint.
 	HostHandlerResponse struct {
 		Host   hostdb.Host                `json:"host"`

api/host.go

@@ -112,6 +112,6 @@ func (opts HostsForScanningOptions) Apply(values url.Values) {
 		values.Set("limit", fmt.Sprint(opts.Limit))
 	}
 	if !opts.MaxLastScan.IsZero() {
-		values.Set("maxLastScan", fmt.Sprint(TimeRFC3339(opts.MaxLastScan)))
+		values.Set("lastScan", fmt.Sprint(TimeRFC3339(opts.MaxLastScan)))
 	}
 }

api/multipart.go

@@ -46,9 +46,10 @@ type (
 	}
 
 	CreateMultipartOptions struct {
-		Key      object.EncryptionKey
-		MimeType string
-		Metadata ObjectUserMetadata
+		GenerateKey bool
+		Key         *object.EncryptionKey
+		MimeType    string
+		Metadata    ObjectUserMetadata
 	}
 )
 
@@ -81,11 +82,15 @@ type (
 	}
 
 	MultipartCreateRequest struct {
-		Bucket   string               `json:"bucket"`
-		Path     string               `json:"path"`
-		Key      object.EncryptionKey `json:"key"`
-		MimeType string               `json:"mimeType"`
-		Metadata ObjectUserMetadata   `json:"metadata"`
+		Bucket   string                `json:"bucket"`
+		Path     string                `json:"path"`
+		Key      *object.EncryptionKey `json:"key"`
+		MimeType string                `json:"mimeType"`
+		Metadata ObjectUserMetadata    `json:"metadata"`
+
+		// TODO: The next major version change should invert this to create a
+		// key by default
+		GenerateKey bool `json:"generateKey"`
 	}
 
 	MultipartCreateResponse struct {