Bugfixes (#535)

* Docker works with clusters of 100 instances

* Restore default boh settings

* Disabled some tests

* style fixes

* Fixed subnet timeout bug

* Changed window size

* Changed cache

* Other changes

* Reintroducing bash error in order to install remotedesktop by default

* Mishandled error

README.md

@@ -128,7 +128,6 @@ SafeScale is currently under active development and does not yet offer all the a
     - OpenTelekom
     - CloudFerro
     - Generic OpenStack
-    - local provider (unstable, not compiled by default)
     - AWS
     - GCP (Google Cloud Platform)
     - Outscale

cli/cachetest/cachemain.go

@@ -25,7 +25,7 @@ func main() {
 
 	cacheManager := cache.New(ristrettoStore)
 	ctx := context.TODO() // nolint
-	err = cacheManager.Set(ctx, "my-key", "my-value", &store.Options{Cost: 2, Expiration: 15 * time.Minute})
+	err = cacheManager.Set(ctx, "my-key", "my-value", &store.Options{Cost: 2, Expiration: 120 * time.Minute})
 	if err != nil {
 		panic(err)
 	}

cli/safescaled/expose_debug.go

@@ -59,6 +59,7 @@ func expose() {
 	expvar.NewInt("cluster.cache.hit")
 	expvar.NewInt("newhost.cache.hit")
 	expvar.NewInt("newhost.cache.read")
+	expvar.NewInt("host.inspections")
 
 	exportstats.NewStatCount("stats")
 	http.Handle("/debug/metrics", metric.Handler(metric.Exposed))

doc/USAGE.md

@@ -62,7 +62,6 @@ Each `tenants` section contains specific authentication parameters for each Clou
 >    - cloudferro
 >    - flexibleengine
 >    - gcp
->    - local (currently broken, not compiled by default, cf this [documentation](LIBVIRT_PROVIDER.md))
 >    - openstack (pure OpenStack support)
 >    - outscale
 >    - opentelekom

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/ovh/go-ovh v1.3.0
 	github.com/pelletier/go-toml/v2 v2.0.5
 	github.com/pkg/errors v0.9.1
+	github.com/pkg/profile v1.7.0
 	github.com/pkg/sftp v1.13.5
 	github.com/quasilyte/go-ruleguard/dsl v0.3.21
 	github.com/sanity-io/litter v1.5.5

go.sum

@@ -785,6 +785,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
+github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
+github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=

lib/backend/handlers/host.go

@@ -30,14 +30,12 @@ import (
 	labelfactory "github.com/CS-SI/SafeScale/v22/lib/backend/resources/factories/label"
 	securitygroupfactory "github.com/CS-SI/SafeScale/v22/lib/backend/resources/factories/securitygroup"
 	propertiesv1 "github.com/CS-SI/SafeScale/v22/lib/backend/resources/properties/v1"
-	propertiesv2 "github.com/CS-SI/SafeScale/v22/lib/backend/resources/properties/v2"
 	"github.com/CS-SI/SafeScale/v22/lib/protocol"
 	"github.com/CS-SI/SafeScale/v22/lib/utils/data"
 	"github.com/CS-SI/SafeScale/v22/lib/utils/data/serialize"
 	"github.com/CS-SI/SafeScale/v22/lib/utils/debug"
 	"github.com/CS-SI/SafeScale/v22/lib/utils/debug/tracing"
 	"github.com/CS-SI/SafeScale/v22/lib/utils/fail"
-	"github.com/sirupsen/logrus"
 )
 
 //go:generate minimock -i github.com/CS-SI/SafeScale/v22/lib/backend/handlers.HostHandler -o mocks/mock_host.go
@@ -53,7 +51,6 @@ type HostHandler interface {
 	List(bool) (abstract.HostList, fail.Error)
 	ListSecurityGroups(string) ([]*propertiesv1.SecurityGroupBond, fail.Error)
 	Reboot(string) fail.Error
-	Resize(string, abstract.HostSizingRequirements) (resources.Host, fail.Error)
 	Start(string) fail.Error
 	Status(string) (hoststate.Enum, fail.Error)
 	Stop(string) fail.Error
@@ -225,60 +222,6 @@ func (handler *hostHandler) Create(req abstract.HostRequest, sizing abstract.Hos
 	return hostInstance, nil
 }
 
-// Resize a Host
-func (handler *hostHandler) Resize(ref string, sizing abstract.HostSizingRequirements) (_ resources.Host, ferr fail.Error) {
-	defer func() {
-		if ferr != nil {
-			ferr.WithContext(handler.job.Context())
-		}
-	}()
-	defer fail.OnPanic(&ferr)
-
-	if handler == nil {
-		return nil, fail.InvalidInstanceError()
-	}
-
-	tracer := debug.NewTracer(handler.job.Context(), tracing.ShouldTrace("handlers.host"), "('%s')", ref).WithStopwatch().Entering()
-	defer tracer.Exiting()
-	defer fail.OnExitLogError(handler.job.Context(), &ferr, tracer.TraceMessage())
-
-	hostInstance, xerr := hostfactory.Load(handler.job.Context(), handler.job.Service(), ref)
-	if xerr != nil {
-		return nil, xerr
-	}
-
-	reduce := false
-	xerr = hostInstance.Inspect(handler.job.Context(), func(_ data.Clonable, props *serialize.JSONProperties) fail.Error {
-		return props.Inspect(hostproperty.SizingV2, func(clonable data.Clonable) fail.Error {
-			hostSizingV2, ok := clonable.(*propertiesv2.HostSizing)
-			if !ok {
-				return fail.InconsistentError("'*propertiesv1.HostSizing' expected, '%s' provided", reflect.TypeOf(clonable).String())
-			}
-
-			reduce = reduce || (sizing.MinCores < hostSizingV2.RequestedSize.MinCores)
-			reduce = reduce || (sizing.MinRAMSize < hostSizingV2.RequestedSize.MinRAMSize)
-			reduce = reduce || (sizing.MinGPU < hostSizingV2.RequestedSize.MinGPU)
-			reduce = reduce || (sizing.MinCPUFreq < hostSizingV2.RequestedSize.MinCPUFreq)
-			reduce = reduce || (sizing.MinDiskSize < hostSizingV2.RequestedSize.MinDiskSize)
-			return nil
-		})
-	})
-	if xerr != nil {
-		return nil, xerr
-	}
-
-	if reduce {
-		logrus.WithContext(handler.job.Context()).Warn("Asking for less resource... is not going to happen")
-	}
-
-	xerr = hostInstance.Resize(handler.job.Context(), sizing)
-	if xerr != nil {
-		return nil, xerr
-	}
-
-	return hostInstance, nil
-}
-
 // Status returns the status of a host (running or stopped mainly)
 func (handler *hostHandler) Status(ref string) (_ hoststate.Enum, ferr fail.Error) {
 	defer func() {
@@ -335,11 +278,6 @@ func (handler *hostHandler) Inspect(ref string) (_ resources.Host, ferr fail.Err
 		return nil, xerr
 	}
 
-	_, xerr = hostInstance.ForceGetState(handler.job.Context())
-	if xerr != nil {
-		return nil, xerr
-	}
-
 	return hostInstance, nil
 }
 

lib/backend/iaas/factory.go

@@ -152,9 +152,9 @@ func UseService(inctx context.Context, tenantName string, metadataVersion string
 		}
 
 		ristrettoCache, err := ristretto.NewCache(&ristretto.Config{
-			NumCounters: 1000,
-			MaxCost:     100,
-			BufferItems: 1024,
+			NumCounters: 1024,
+			MaxCost:     50000000,
+			BufferItems: 128,
 		})
 		if err != nil {
 			return nil, fail.ConvertError(err)
@@ -163,23 +163,13 @@ func UseService(inctx context.Context, tenantName string, metadataVersion string
 		newS := &service{
 			Provider:     providerInstance,
 			tenantName:   tenantName,
-			cacheManager: NewWrappedCache(cache.New(store.NewRistretto(ristrettoCache, &store.Options{Expiration: 1 * time.Minute}))),
+			cacheManager: NewWrappedCache(cache.New(store.NewRistretto(ristrettoCache, &store.Options{Expiration: 120 * time.Minute}))),
 		}
 
-		if beta := os.Getenv("SAFESCALE_CACHE"); beta != "" {
+		if beta := os.Getenv("SAFESCALE_CACHE"); beta != "disabled" {
 			logrus.WithContext(ctx).Infof("Created a cache in: %p", newS.cacheManager)
 		}
 
-		// allRegions, xerr := newS.ListRegions()
-		// if xerr != nil {
-		// 	switch xerr.(type) {
-		// 	case *fail.ErrNotFound:
-		// 		break
-		// 	default:
-		// 		return NullService(), xerr
-		// 	}
-		// }
-
 		authOpts, xerr := providerInstance.GetAuthenticationOptions(ctx)
 		if xerr != nil {
 			return NullService(), xerr

lib/backend/iaas/providers/providerproxy.go

@@ -610,16 +610,6 @@ func (s ProviderProxy) RebootHost(ctx context.Context, parameter stacks.HostPara
 	return xerr
 }
 
-func (s ProviderProxy) ResizeHost(ctx context.Context, parameter stacks.HostParameter, requirements abstract.HostSizingRequirements) (_ *abstract.HostFull, ferr fail.Error) {
-	defer fail.OnPanic(&ferr)
-
-	host, xerr := s.Provider.ResizeHost(ctx, parameter, requirements)
-	if xerr != nil {
-		xerr.WithContext(ctx)
-	}
-	return host, xerr
-}
-
 func (s ProviderProxy) WaitHostReady(ctx context.Context, hostParam stacks.HostParameter, timeout time.Duration) (_ *abstract.HostCore, ferr fail.Error) {
 	defer fail.OnPanic(&ferr)
 

lib/backend/iaas/service.go

@@ -62,7 +62,7 @@ type Service interface {
 	GetMetadataKey() (*crypt.Key, fail.Error)
 	GetCache(context.Context) (cache.CacheInterface, fail.Error)
 	InspectSecurityGroupByName(ctx context.Context, networkID string, name string) (*abstract.SecurityGroup, fail.Error)
-	ListHostsByName(context.Context, bool) (map[string]*abstract.HostFull, fail.Error)
+	ListHostsWithTags(context.Context, []string, map[string]string) ([]*abstract.HostFull, fail.Error)
 	ListTemplatesBySizing(context.Context, abstract.HostSizingRequirements, bool) ([]*abstract.HostTemplate, fail.Error)
 	ObjectStorageConfiguration(ctx context.Context) (objectstorage.Config, fail.Error)
 	SearchImage(context.Context, string) (*abstract.Image, fail.Error)
@@ -1109,8 +1109,8 @@ func (instance service) CreateHostWithKeyPair(inctx context.Context, request abs
 
 }
 
-// ListHostsByName list hosts by name
-func (instance service) ListHostsByName(inctx context.Context, details bool) (map[string]*abstract.HostFull, fail.Error) {
+// ListHostsWithTags list hosts with tags
+func (instance service) ListHostsWithTags(inctx context.Context, labels []string, details map[string]string) ([]*abstract.HostFull, fail.Error) {
 	if valid.IsNil(instance) {
 		return nil, fail.InvalidInstanceError()
 	}
@@ -1119,24 +1119,57 @@ func (instance service) ListHostsByName(inctx context.Context, details bool) (ma
 	defer cancel()
 
 	type result struct {
-		rTr  map[string]*abstract.HostFull
+		rTr  []*abstract.HostFull
 		rErr fail.Error
 	}
 	chRes := make(chan result)
 	go func() {
 		defer close(chRes)
 
-		hosts, err := instance.ListHosts(ctx, details)
+		var varhosts []*abstract.HostFull
+
+		hosts, err := instance.ListHosts(ctx, true)
 		if err != nil {
 			chRes <- result{nil, err}
 			return
 		}
-		hostMap := make(map[string]*abstract.HostFull)
+
+		if len(labels) > 0 {
+			for _, host := range hosts {
+				there := true
+				for _, k := range labels {
+					_, ok := host.Core.Tags[k]
+					if !ok {
+						there = false
+						break
+					}
+				}
+				if there {
+					varhosts = append(varhosts, host)
+				}
+			}
+		}
+
 		for _, host := range hosts {
-			hostMap[host.Core.Name] = host
+			there := true
+			for k, v := range details {
+				av, ok := host.Core.Tags[k]
+				if ok {
+					if av != v {
+						there = false
+						break
+					}
+				} else {
+					there = false
+					break
+				}
+			}
+			if there {
+				varhosts = append(varhosts, host)
+			}
 		}
-		chRes <- result{hostMap, nil}
 
+		chRes <- result{varhosts, nil}
 	}()
 	select {
 	case res := <-chRes:
@@ -1239,7 +1272,8 @@ func (instance service) GetCache(inctx context.Context) (cache.CacheInterface, f
 		return nil, fail.InvalidInstanceError()
 	}
 
-	if beta := os.Getenv("SAFESCALE_CACHE"); beta != "" {
+	// Enable cache by default, too many consistency errors otherwise
+	if beta := os.Getenv("SAFESCALE_CACHE"); beta != "disabled" {
 		return instance.cacheManager, nil
 	}
 

lib/backend/iaas/service_test.go

@@ -0,0 +1,66 @@
+package iaas
+
+import (
+	"context"
+	"reflect"
+	"regexp"
+	"testing"
+
+	"github.com/CS-SI/SafeScale/v22/lib/backend/iaas/objectstorage"
+	"github.com/CS-SI/SafeScale/v22/lib/backend/iaas/providers"
+	"github.com/CS-SI/SafeScale/v22/lib/backend/resources/abstract"
+	"github.com/CS-SI/SafeScale/v22/lib/utils/crypt"
+	"github.com/CS-SI/SafeScale/v22/lib/utils/fail"
+)
+
+func Test_service_ListHostsWithTags(t *testing.T) {
+	type fields struct {
+		Provider             providers.Provider
+		Location             objectstorage.Location
+		tenantName           string
+		cacheManager         *wrappedCache
+		metadataBucket       abstract.ObjectStorageBucket
+		metadataKey          *crypt.Key
+		whitelistTemplateREs []*regexp.Regexp
+		blacklistTemplateREs []*regexp.Regexp
+		whitelistImageREs    []*regexp.Regexp
+		blacklistImageREs    []*regexp.Regexp
+	}
+	type args struct {
+		inctx   context.Context
+		labels  []string
+		details map[string]string
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   []*abstract.HostFull
+		want1  fail.Error
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			instance := service{
+				Provider:             tt.fields.Provider,
+				Location:             tt.fields.Location,
+				tenantName:           tt.fields.tenantName,
+				cacheManager:         tt.fields.cacheManager,
+				metadataBucket:       tt.fields.metadataBucket,
+				metadataKey:          tt.fields.metadataKey,
+				whitelistTemplateREs: tt.fields.whitelistTemplateREs,
+				blacklistTemplateREs: tt.fields.blacklistTemplateREs,
+				whitelistImageREs:    tt.fields.whitelistImageREs,
+				blacklistImageREs:    tt.fields.blacklistImageREs,
+			}
+			got, got1 := instance.ListHostsWithTags(tt.args.inctx, tt.args.labels, tt.args.details)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ListHostsWithTags() got = %v, want %v", got, tt.want)
+			}
+			if !reflect.DeepEqual(got1, tt.want1) {
+				t.Errorf("ListHostsWithTags() got1 = %v, want %v", got1, tt.want1)
+			}
+		})
+	}
+}

lib/backend/iaas/stacks/api/stack.go

@@ -136,8 +136,6 @@ type Stack interface {
 	StartHost(context.Context, stacks.HostParameter) fail.Error
 	// RebootHost reboots a host
 	RebootHost(context.Context, stacks.HostParameter) fail.Error
-	// ResizeHost resizes a host
-	ResizeHost(context.Context, stacks.HostParameter, abstract.HostSizingRequirements) (*abstract.HostFull, fail.Error)
 	// WaitHostReady waits until host defined in hostParam is reachable by SSH
 	WaitHostReady(ctx context.Context, hostParam stacks.HostParameter, timeout time.Duration) (*abstract.HostCore, fail.Error)
 	// BindSecurityGroupToHost attaches a security group to a host

lib/backend/iaas/stacks/api/stackproxy.go

@@ -538,16 +538,6 @@ func (s StackProxy) RebootHost(ctx context.Context, parameter stacks.HostParamet
 	return xerr
 }
 
-func (s StackProxy) ResizeHost(ctx context.Context, parameter stacks.HostParameter, requirements abstract.HostSizingRequirements) (_ *abstract.HostFull, ferr fail.Error) {
-	defer fail.OnPanic(&ferr)
-
-	host, xerr := s.FullStack.ResizeHost(ctx, parameter, requirements)
-	if xerr != nil {
-		xerr.WithContext(ctx)
-	}
-	return host, xerr
-}
-
 func (s StackProxy) WaitHostReady(ctx context.Context, hostParam stacks.HostParameter, timeout time.Duration) (_ *abstract.HostCore, ferr fail.Error) {
 	defer fail.OnPanic(&ferr)