diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 89876c3c8..6f38cfc31 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -12,11 +12,12 @@ jobs: steps: - uses: actions/stale@v5 with: + operations-per-run: 60 days-before-issue-stale: 60 days-before-issue-close: 30 stale-issue-label: "stale" - stale-issue-message: "This issue is stale because it has been open for 30 days with no activity." - close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale." + stale-issue-message: "This issue is stale because it has been open for 60 days with no activity." + close-issue-message: "This issue was closed because it has been inactive for 30 days since being marked as stale." days-before-pr-stale: -1 days-before-pr-close: -1 repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/cmd/profile.go b/cmd/profile.go new file mode 100644 index 000000000..8a2ccd5e5 --- /dev/null +++ b/cmd/profile.go @@ -0,0 +1,34 @@ +//go:build profile + +package cmd + +import ( + "fmt" + "net" + "net/http" + "net/http/pprof" + "os" +) + +func init() { + go func() { + myMux := http.NewServeMux() + + myMux.HandleFunc("/debug/pprof/", pprof.Index) + myMux.HandleFunc("/debug/pprof/{action}", pprof.Index) + myMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + + listener, err := net.Listen("tcp", ":0") + if err != nil { + return + } + + f, err := os.OpenFile("/tmp/pprof_ports", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err == nil { + f.Write([]byte(fmt.Sprintf("%d=%d\n", os.Getpid(), listener.Addr().(*net.TCPAddr).Port))) + f.Close() + } + + http.Serve(listener, myMux) + }() +} diff --git a/hack/dev_devpod_pro.sh b/hack/dev_devpod_pro.sh new file mode 100755 index 000000000..aec3c049c --- /dev/null +++ b/hack/dev_devpod_pro.sh @@ -0,0 +1,20 @@ +#! /usr/bin/env zsh + +set -e + +NS=${1:-"default"} +RACE=${2:-"no"} + +if [[ ! $PWD == *"/go/src/devpod"* ]]; then + echo "Please run this script from the /workspace/loft/devpod directory" + exit 1 +fi + +if [[ $RACE == "yes" ]]; then + echo "Building devpod with race detector" + CGO_ENABLED=1 go build -ldflags "-s -w" -tags profile -race -o devpod-cli +else + CGO_ENABLED=0 go build -ldflags "-s -w" -tags profile -o devpod-cli +fi + +kubectl -n $NS cp --no-preserve=true ./devpod-cli $(kubectl -n $NS get pods -l app=loft -o jsonpath="{.items[0].metadata.name}"):/usr/local/bin/devpod diff --git a/hack/rebuild.sh b/hack/rebuild.sh index 660b5acaa..8a1a5cd26 100755 --- a/hack/rebuild.sh +++ b/hack/rebuild.sh @@ -13,7 +13,12 @@ for os in $BUILD_PLATFORMS; do continue fi echo "[INFO] Building for $os/$arch" - CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -ldflags "-s -w" -o test/devpod-cli-$os-$arch + if [[ $RACE == "yes" ]]; then + echo "Building devpod with race detector" + CGO_ENABLED=1 GOOS=$os GOARCH=$arch go build -race -ldflags "-s -w" -o test/devpod-cli-$os-$arch + else + CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -ldflags "-s -w" -o test/devpod-cli-$os-$arch + fi done done diff --git a/loadtest/README.md b/loadtest/README.md new file mode 100644 index 000000000..db09f0c67 --- /dev/null +++ b/loadtest/README.md @@ -0,0 +1,36 @@ +## Load Testing DevPod + +### Create the workspaces + +`./startWorkspaces.sh` + +### Run the load test and generate wait times + +`./run.sh` + +Update NUM_WORKSPACES or NUM_COMMANDS_PER_WORKSPACE to adjust load signature + +### Clean up + +`./deleteWorkspaces.sh` + +### Things to note + +`generateLoad.sh` contains the SSH command to generate load, change the command here to adjust how you want to generate traffic + +### Get core dump from loft + +``` +kubectl -n devpod-pro set env deployment/loft LOFTDEBUG=true + +kubectl -n devpod-pro port-forward loft-55df4d875f-j9vnd 8080:8080 & + +curl -s -v http://localhost:8080/debug/pprof/heap > $(date '+%Y-%m-%d-%H:%M:%S').out +``` + +### Profile the server every 30 seconds + +``` +while true; do curl -s -v http://localhost:8080/debug/pprof/heap > $(date '+%Y-%m-%d-%H:%M:%S').out; sleep 30; done +``` + diff --git a/loadtest/deleteWorkspaces.sh b/loadtest/deleteWorkspaces.sh new file mode 100755 index 000000000..05a2987e2 --- /dev/null +++ b/loadtest/deleteWorkspaces.sh @@ -0,0 +1,12 @@ +#!/bin/zsh + +export NUM_WORKSPACES=60 + +# Start the workspaces +for i in $(seq 1 $NUM_WORKSPACES); +do + devpod delete --force "loadtest$i" & + sleep 2 +done + +wait diff --git a/loadtest/emulateTraffic.sh b/loadtest/emulateTraffic.sh new file mode 100755 index 000000000..3c22bf86f --- /dev/null +++ b/loadtest/emulateTraffic.sh @@ -0,0 +1,8 @@ +#!/bin/zsh + +# SSH to the workspace and execute command +for j in $(seq 1 $NUM_COMMANDS_PER_WORKSPACE); +do + ./generateLoad.sh $1 + sleep 1 +done diff --git a/loadtest/generateLoad.sh b/loadtest/generateLoad.sh new file mode 100755 index 000000000..a63c847cc --- /dev/null +++ b/loadtest/generateLoad.sh @@ -0,0 +1,3 @@ +#!/bin/zsh + +devpod ssh "loadtest$1" --command="tr -dc A-Za-z0-9 /dev/null diff --git a/loadtest/init_monitor.sh b/loadtest/init_monitor.sh new file mode 100644 index 000000000..72fe7d26f --- /dev/null +++ b/loadtest/init_monitor.sh @@ -0,0 +1,5 @@ +#!/bin/zsh + +#kubectl -n devpod-pro set env deployment/loft LOFTDEBUG=true + +kubectl -n devpod-pro port-forward $(kubectl -n devpod-pro get pods -l app=loft -o jsonpath="{.items[0].metadata.name}") 8080:8080 diff --git a/loadtest/monitor.sh b/loadtest/monitor.sh new file mode 100755 index 000000000..89e76add7 --- /dev/null +++ b/loadtest/monitor.sh @@ -0,0 +1,16 @@ +#!/bin/zsh + +mkdir results + +export INTERVAL_SECONDS=30 + +echo "Monitoring heap, go routines, and threads every $INTERVAL_SECONDS seconds ..." + +while true; do curl -s -k https://localhost:8080/debug/pprof/heap > ./results/$(date '+%Y-%m-%d-%H:%M:%S').heap; sleep $(echo $INTERVAL_SECONDS); done & + +while true; do curl -s -k https://localhost:8080/debug/pprof/goroutine > ./results/$(date '+%Y-%m-%d-%H:%M:%S').cpu; sleep $(echo $INTERVAL_SECONDS); done & + +while true; do curl -s -k https://localhost:8080/debug/pprof/threadcreate > ./results/$(date '+%Y-%m-%d-%H:%M:%S').threads; sleep $(echo $INTERVAL_SECONDS); done & + +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT +wait diff --git a/loadtest/run.sh b/loadtest/run.sh new file mode 100755 index 000000000..9da1e7ded --- /dev/null +++ b/loadtest/run.sh @@ -0,0 +1,16 @@ +#!/bin/zsh + +export NUM_WORKSPACES=10 +export NUM_COMMANDS_PER_WORKSPACE=1 + +echo "Running $NUM_WORKSPACES workspaces with $NUM_COMMANDS_PER_WORKSPACE commands each ..." + +# SSH to the workspace and execute command +for j in $(seq 1 $NUM_WORKSPACES); +do + time ./emulateTraffic.sh $j & + sleep 2 +done + +# Keep the session active to allow the commands to execute and use STDOUT +wait diff --git a/loadtest/startWorkspaces.sh b/loadtest/startWorkspaces.sh new file mode 100755 index 000000000..f77cbd029 --- /dev/null +++ b/loadtest/startWorkspaces.sh @@ -0,0 +1,11 @@ +#!/bin/zsh + +export NUM_WORKSPACES=20 + +# Start the workspaces +for i in $(seq 11 $NUM_WORKSPACES); +do + devpod up --id "loadtest$i" --debug --ide none http://github.com/kubernetes/kubernetes +done + +wait diff --git a/pkg/inject/inject.go b/pkg/inject/inject.go index 1217ff5f8..536ca3869 100644 --- a/pkg/inject/inject.go +++ b/pkg/inject/inject.go @@ -62,7 +62,6 @@ func InjectAndExecute( if err != nil { return true, err } - defer stdoutWriter.Close() // delayed stderr delayedStderr := newDelayedWriter(stderr) @@ -82,7 +81,6 @@ func InjectAndExecute( execErrChan := make(chan error, 1) go func() { defer stdoutWriter.Close() - defer stdinWriter.Close() defer log.Debugf("done exec") err := exec(cancelCtx, scriptRawCode, stdinReader, stdoutWriter, delayedStderr) @@ -96,10 +94,8 @@ func InjectAndExecute( // inject file injectChan := make(chan injectResult, 1) go func() { - defer stdoutWriter.Close() defer stdinWriter.Close() defer log.Debugf("done inject") - defer cancel() wasExecuted, err := inject(localFile, stdinWriter, stdin, stdoutReader, stdout, delayedStderr, timeout, log) injectChan <- injectResult{ diff --git a/pkg/tunnel/services.go b/pkg/tunnel/services.go index d929ba905..97eabb3be 100644 --- a/pkg/tunnel/services.go +++ b/pkg/tunnel/services.go @@ -75,7 +75,6 @@ func RunInContainer( if err != nil { return err } - defer stdinWriter.Close() // start server on stdio cancelCtx, cancel := context.WithCancel(ctx) @@ -90,8 +89,9 @@ func RunInContainer( errChan := make(chan error, 1) go func() { defer cancel() + defer stdinWriter.Close() // forward credentials to container - err = tunnelserver.RunServicesServer( + err := tunnelserver.RunServicesServer( cancelCtx, stdoutReader, stdinWriter,