Add load testing scripts, profiler and data race fixes (#1441)

* Add load testing scripts

* Fix race conditon

* Add CI pipeline to use race detector

* Build binaries using race detector for e2e race

* Allow stale to use more issues each run

* Fix race e2e

* Remove race tests from CI

* Update dev scripts with race detector

* PR comments

.github/workflows/stale.yaml

@@ -12,11 +12,12 @@ jobs:
     steps:
       - uses: actions/stale@v5
         with:
+          operations-per-run: 60
           days-before-issue-stale: 60
           days-before-issue-close: 30
           stale-issue-label: "stale"
-          stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
-          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+          stale-issue-message: "This issue is stale because it has been open for 60 days with no activity."
+          close-issue-message: "This issue was closed because it has been inactive for 30 days since being marked as stale."
           days-before-pr-stale: -1
           days-before-pr-close: -1
           repo-token: ${{ secrets.GITHUB_TOKEN }}

cmd/profile.go

@@ -0,0 +1,34 @@
+//go:build profile
+
+package cmd
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/pprof"
+	"os"
+)
+
+func init() {
+	go func() {
+		myMux := http.NewServeMux()
+
+		myMux.HandleFunc("/debug/pprof/", pprof.Index)
+		myMux.HandleFunc("/debug/pprof/{action}", pprof.Index)
+		myMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+
+		listener, err := net.Listen("tcp", ":0")
+		if err != nil {
+			return
+		}
+
+		f, err := os.OpenFile("/tmp/pprof_ports", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		if err == nil {
+			f.Write([]byte(fmt.Sprintf("%d=%d\n", os.Getpid(), listener.Addr().(*net.TCPAddr).Port)))
+			f.Close()
+		}
+
+		http.Serve(listener, myMux)
+	}()
+}

hack/dev_devpod_pro.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env zsh
+
+set -e
+
+NS=${1:-"default"}
+RACE=${2:-"no"}
+
+if [[ ! $PWD == *"/go/src/devpod"* ]]; then
+  echo "Please run this script from the /workspace/loft/devpod directory"
+  exit 1
+fi
+
+if [[ $RACE == "yes" ]]; then
+  echo "Building devpod with race detector"
+  CGO_ENABLED=1 go build -ldflags "-s -w" -tags profile -race -o devpod-cli
+else
+  CGO_ENABLED=0 go build -ldflags "-s -w" -tags profile -o devpod-cli
+fi
+
+kubectl -n $NS cp --no-preserve=true ./devpod-cli $(kubectl -n $NS get pods -l app=loft -o jsonpath="{.items[0].metadata.name}"):/usr/local/bin/devpod

hack/rebuild.sh

@@ -13,7 +13,12 @@ for os in $BUILD_PLATFORMS; do
             continue
         fi
         echo "[INFO] Building for $os/$arch"
-        CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -ldflags "-s -w" -o test/devpod-cli-$os-$arch
+        if [[ $RACE == "yes" ]]; then
+            echo "Building devpod with race detector"
+            CGO_ENABLED=1 GOOS=$os GOARCH=$arch go build -race -ldflags "-s -w" -o test/devpod-cli-$os-$arch
+        else
+            CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -ldflags "-s -w" -o test/devpod-cli-$os-$arch
+        fi
     done
 done
 

loadtest/README.md

@@ -0,0 +1,36 @@
+## Load Testing DevPod
+
+### Create the workspaces
+
+`./startWorkspaces.sh`
+
+### Run the load test and generate wait times
+
+`./run.sh`
+
+Update NUM_WORKSPACES or NUM_COMMANDS_PER_WORKSPACE to adjust load signature
+
+### Clean up
+
+`./deleteWorkspaces.sh`
+
+### Things to note
+
+`generateLoad.sh` contains the SSH command to generate load, change the command here to adjust how you want to generate traffic
+
+### Get core dump from loft
+
+```
+kubectl -n devpod-pro set env deployment/loft LOFTDEBUG=true
+
+kubectl -n devpod-pro port-forward loft-55df4d875f-j9vnd 8080:8080 &
+
+curl -s -v http://localhost:8080/debug/pprof/heap > $(date '+%Y-%m-%d-%H:%M:%S').out
+```
+
+### Profile the server every 30 seconds
+
+```
+while true; do curl -s -v http://localhost:8080/debug/pprof/heap > $(date '+%Y-%m-%d-%H:%M:%S').out; sleep 30; done
+```
+

loadtest/deleteWorkspaces.sh

@@ -0,0 +1,12 @@
+#!/bin/zsh
+
+export NUM_WORKSPACES=60
+
+# Start the workspaces
+for i in $(seq 1 $NUM_WORKSPACES);
+do
+    devpod delete --force "loadtest$i" &
+    sleep 2
+done
+
+wait

loadtest/emulateTraffic.sh

@@ -0,0 +1,8 @@
+#!/bin/zsh
+
+# SSH to the workspace and execute command
+for j in $(seq 1 $NUM_COMMANDS_PER_WORKSPACE);
+do
+    ./generateLoad.sh $1
+    sleep 1
+done

loadtest/generateLoad.sh

@@ -0,0 +1,3 @@
+#!/bin/zsh
+
+devpod ssh "loadtest$1" --command="tr -dc A-Za-z0-9 </dev/urandom | head -c 100000000; echo" > /dev/null

loadtest/init_monitor.sh

@@ -0,0 +1,5 @@
+#!/bin/zsh
+
+#kubectl -n devpod-pro set env deployment/loft LOFTDEBUG=true
+
+kubectl -n devpod-pro port-forward $(kubectl -n devpod-pro get pods -l app=loft -o jsonpath="{.items[0].metadata.name}") 8080:8080

loadtest/monitor.sh

@@ -0,0 +1,16 @@
+#!/bin/zsh
+
+mkdir results
+
+export INTERVAL_SECONDS=30
+
+echo "Monitoring heap, go routines, and threads every $INTERVAL_SECONDS seconds ..."
+
+while true; do curl -s -k https://localhost:8080/debug/pprof/heap > ./results/$(date '+%Y-%m-%d-%H:%M:%S').heap; sleep $(echo $INTERVAL_SECONDS); done &
+
+while true; do curl -s -k https://localhost:8080/debug/pprof/goroutine > ./results/$(date '+%Y-%m-%d-%H:%M:%S').cpu; sleep $(echo $INTERVAL_SECONDS); done &
+
+while true; do curl -s -k https://localhost:8080/debug/pprof/threadcreate > ./results/$(date '+%Y-%m-%d-%H:%M:%S').threads; sleep $(echo $INTERVAL_SECONDS); done &
+
+trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
+wait

loadtest/run.sh

@@ -0,0 +1,16 @@
+#!/bin/zsh
+
+export NUM_WORKSPACES=10
+export NUM_COMMANDS_PER_WORKSPACE=1
+
+echo "Running $NUM_WORKSPACES workspaces with $NUM_COMMANDS_PER_WORKSPACE commands each ..."
+
+# SSH to the workspace and execute command
+for j in $(seq 1 $NUM_WORKSPACES);
+do
+    time ./emulateTraffic.sh $j &
+    sleep 2
+done
+
+# Keep the session active to allow the commands to execute and use STDOUT
+wait

loadtest/startWorkspaces.sh

@@ -0,0 +1,11 @@
+#!/bin/zsh
+
+export NUM_WORKSPACES=20
+
+# Start the workspaces
+for i in $(seq 11 $NUM_WORKSPACES);
+do
+    devpod up --id "loadtest$i" --debug --ide none http://github.com/kubernetes/kubernetes
+done
+
+wait

pkg/inject/inject.go

@@ -62,7 +62,6 @@ func InjectAndExecute(
 	if err != nil {
 		return true, err
 	}
-	defer stdoutWriter.Close()
 
 	// delayed stderr
 	delayedStderr := newDelayedWriter(stderr)
@@ -82,7 +81,6 @@ func InjectAndExecute(
 	execErrChan := make(chan error, 1)
 	go func() {
 		defer stdoutWriter.Close()
-		defer stdinWriter.Close()
 		defer log.Debugf("done exec")
 
 		err := exec(cancelCtx, scriptRawCode, stdinReader, stdoutWriter, delayedStderr)
@@ -96,10 +94,8 @@ func InjectAndExecute(
 	// inject file
 	injectChan := make(chan injectResult, 1)
 	go func() {
-		defer stdoutWriter.Close()
 		defer stdinWriter.Close()
 		defer log.Debugf("done inject")
-		defer cancel()
 
 		wasExecuted, err := inject(localFile, stdinWriter, stdin, stdoutReader, stdout, delayedStderr, timeout, log)
 		injectChan <- injectResult{

pkg/tunnel/services.go

@@ -75,7 +75,6 @@ func RunInContainer(
 		if err != nil {
 			return err
 		}
-		defer stdinWriter.Close()
 
 		// start server on stdio
 		cancelCtx, cancel := context.WithCancel(ctx)
@@ -90,8 +89,9 @@ func RunInContainer(
 		errChan := make(chan error, 1)
 		go func() {
 			defer cancel()
+			defer stdinWriter.Close()
 			// forward credentials to container
-			err = tunnelserver.RunServicesServer(
+			err := tunnelserver.RunServicesServer(
 				cancelCtx,
 				stdoutReader,
 				stdinWriter,