diff --git a/feeder/feeder-template/feederv2/feederv2.go b/feeder/feeder-template/feederv2/feederv2.go index 88d8b292..6475fba1 100644 --- a/feeder/feeder-template/feederv2/feederv2.go +++ b/feeder/feeder-template/feederv2/feederv2.go @@ -13,10 +13,12 @@ import ( "github.com/akamensky/argparse" "github.com/go-redis/redis" _ "github.com/mattn/go-sqlite3" + "github.com/bradfitz/gomemcache/memcache" "github.com/w3c/automotive-viss2/utils" "math/rand" "net" "os" + "os/exec" "sort" "strconv" "strings" @@ -39,6 +41,7 @@ type FeederMap struct { var scalingDataList []string var redisClient *redis.Client +var memcacheClient *memcache.Client var dbHandle *sql.DB var stateDbType string @@ -180,6 +183,14 @@ func statestorageSet(path string, val string, ts string) int { return -1 } return 0 + case "memcache": + dp := `{"val":"` + val + `", "ts":"` + ts + `"}` + err := memcacheClient.Set(&memcache.Item{Key: path, Value: []byte(dp)}) + if err != nil { + utils.Error.Printf("Job failed. Err=%s", err) + return -1 + } + return 0 } return -1 } @@ -384,8 +395,8 @@ func main() { Required: false, Help: "changes log output level", Default: "info"}) - stateDB := parser.Selector("d", "statestorage", []string{"sqlite", "redis", "none"}, &argparse.Options{Required: false, - Help: "Statestorage must be either sqlite, redis, or none", Default: "redis"}) + stateDB := parser.Selector("d", "statestorage", []string{"sqlite", "redis", "memcache", "none"}, &argparse.Options{Required: false, + Help: "Statestorage must be either sqlite, redis, memcache, or none", Default: "redis"}) dbFile := parser.String("f", "dbfile", &argparse.Options{ Required: false, Help: "statestorage database filename", @@ -427,6 +438,19 @@ func main() { } else { utils.Info.Printf("Redis state storage initialised.") } + case "memcache": + memcacheClient = memcache.New("/var/tmp/vissv2/memcacheDB.sock") + err := memcacheClient.Ping() + if err != nil { + utils.Info.Printf("Memcache daemon not alive. Trying to start it") + cmd := exec.Command("/usr/bin/bash", "memcacheNativeInit.sh") + err := cmd.Run() + if err != nil { + utils.Error.Printf("Memcache daemon startup failed, err=%s", err) + os.Exit(1) + } + } + utils.Info.Printf("Memcache daemon alive.") default: utils.Error.Printf("Unknown state storage type = %s", stateDbType) os.Exit(1) diff --git a/go.mod b/go.mod index 6bcbd6b7..eb0979d0 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( require ( github.com/apache/thrift v0.15.0 // indirect + github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect golang.org/x/net v0.19.0 // indirect diff --git a/go.sum b/go.sum index a5006a84..392374bc 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/apache/iotdb-client-go v1.1.7 h1:MH87LJNquMxHmFKWuS06CFwxgEjxK5oug6H5 github.com/apache/iotdb-client-go v1.1.7/go.mod h1:3D6QYkqRmASS/4HsjU+U/3fscyc5M9xKRfywZsKuoZY= github.com/apache/thrift v0.15.0 h1:aGvdaR0v1t9XLgjtBYwxcBvBOTMqClzwE26CHOgjW1Y= github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 h1:N7oVaKyGp8bttX0bfZGmcGkjz7DLQXhAn3DNd3T0ous= +github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/server/vissv2server/memcacheNativeInit.sh b/server/vissv2server/memcacheNativeInit.sh new file mode 100755 index 00000000..ef959bd6 --- /dev/null +++ b/server/vissv2server/memcacheNativeInit.sh @@ -0,0 +1,3 @@ +#! /usr/bin/bash +memcached -d --unix-mask=755 --unix-socket=/var/tmp/vissv2/memcacheDB.sock + diff --git a/server/vissv2server/serverCoreHandlerLib.go b/server/vissv2server/serverCoreHandlerLib.go index 49962069..3f294e1f 100644 --- a/server/vissv2server/serverCoreHandlerLib.go +++ b/server/vissv2server/serverCoreHandlerLib.go @@ -29,8 +29,9 @@ func (pathList *PathList) VssPathListHandler(w http.ResponseWriter, r *http.Requ w.Header().Set("Content-Type", "application/json") w.Write(bytes) - truncatedIndex := min(len(bytes), 101) - utils.Info.Printf("initVssPathListServer():Response=%s...(truncated to %d bytes)", truncatedIndex-1, bytes[0:truncatedIndex]) +// truncatedIndex := min(len(bytes), 101) +// utils.Info.Printf("initVssPathListServer():Response=%s...(truncated to %d bytes)", truncatedIndex-1, bytes[0:truncatedIndex]) + utils.Info.Printf("initVssPathListServer():Response length=%d", len(bytes)) } func min(a, b int) int { diff --git a/server/vissv2server/serviceMgr/serviceMgr.go b/server/vissv2server/serviceMgr/serviceMgr.go index cde3be1c..d1ed6377 100644 --- a/server/vissv2server/serviceMgr/serviceMgr.go +++ b/server/vissv2server/serviceMgr/serviceMgr.go @@ -19,6 +19,7 @@ import ( "github.com/apache/iotdb-client-go/client" "github.com/go-redis/redis" _ "github.com/mattn/go-sqlite3" + "github.com/bradfitz/gomemcache/memcache" "github.com/w3c/automotive-viss2/utils" "io" "net" @@ -72,6 +73,7 @@ var errorResponseMap = map[string]interface{}{ var dbHandle *sql.DB var dbErr error var redisClient *redis.Client +var memcacheClient *memcache.Client var stateDbType string var historySupport bool @@ -472,19 +474,19 @@ func getVehicleData(path string) string { // returns {"value":"Y", "ts":"Z"} rows.Next() err = rows.Scan(&value, ×tamp) if err != nil { - utils.Warning.Printf("Data not found: %s for path=%s\n", err, path) + utils.Warning.Printf("Data not found: %s for path=%s", err, path) return `{"value":"Data-not-available", "ts":"` + utils.GetRfcTime() + `"}` } return `{"value":"` + value + `", "ts":"` + timestamp + `"}` case "redis": - utils.Info.Printf(path) +// utils.Info.Printf(path) dp, err := redisClient.Get(path).Result() if err != nil { if err.Error() != "redis: nil" { - utils.Error.Printf("Job failed. Error()=%s\n", err.Error()) + utils.Error.Printf("Job failed. Error()=%s", err.Error()) return `{"value":"Database-error", "ts":"` + utils.GetRfcTime() + `"}` } else { - utils.Warning.Printf("Data not found.\n") + utils.Warning.Printf("Data not found.") return `{"value":"Data-not-found", "ts":"` + utils.GetRfcTime() + `"}` } } else { @@ -515,6 +517,19 @@ func getVehicleData(path string) string { // returns {"value":"Y", "ts":"Z"} return `{"value":"Data-not-found", "ts":"` + utils.GetRfcTime() + `"}` } return `{"value":"` + value + `", "ts":"` + ts + `"}` + case "memcache": + mcItem, err := memcacheClient.Get(path) + if err != nil { + if err.Error() != "memcache: cache miss" { + utils.Error.Printf("Job failed. Error()=%s", err.Error()) + return `{"value":"Database-error", "ts":"` + utils.GetRfcTime() + `"}` + } else { + utils.Warning.Printf("Data not found.") + return `{"value":"Data-not-found", "ts":"` + utils.GetRfcTime() + `"}` + } + } else { + return string(mcItem.Value) + } case "none": return `{"value":"` + strconv.Itoa(dummyValue) + `", "ts":"` + utils.GetRfcTime() + `"}` } @@ -538,15 +553,8 @@ func setVehicleData(path string, value string) string { return "" } return ts + case "memcache": fallthrough case "redis": - /* dp := `{"val":"` + value + `", "ts":"` + ts + `"}` - dPath := path + ".D" // path to "desired" dp. Must be created identically by feeder reading it. - err := redisClient.Set(dPath, dp, time.Duration(0)).Err() - if err != nil { - utils.Error.Printf("Could not update statestorage. Err=%s\n", err) - return "" - } - return ts*/ udsConn := utils.GetUdsConn(path, "serverFeeder") if udsConn == nil { utils.Error.Printf("setVehicleData:Failed to UDS connect to feeder for path = %s", path) @@ -945,16 +953,21 @@ func ServiceMgrInit(mgrId int, serviceMgrChan chan string, stateStorageType stri } case "redis": addr := utils.GetUdsPath("Vehicle", "redis") + if len(addr) == 0 { + utils.Error.Printf("redis-server socket address not found.") + // os.Exit(1) should terminate the process + return + } utils.Info.Printf(addr) redisClient = redis.NewClient(&redis.Options{ Network: "unix", - Addr: addr, //TODO replace with check and exit if not defined. + Addr: addr, Password: "", DB: 1, }) err := redisClient.Ping().Err() if err != nil { - utils.Error.Printf("redis-server ,ping err = %s", err) + utils.Info.Printf("Redis-server not started. Trying to start it.") if utils.FileExists("redis.log") { os.Remove("redis.log") } @@ -977,6 +990,26 @@ func ServiceMgrInit(mgrId int, serviceMgrChan chan string, stateStorageType stri os.Exit(1) } defer IoTDBsession.Close() + case "memcache": + addr := utils.GetUdsPath("Vehicle", "memcache") + if len(addr) == 0 { + utils.Error.Printf("memcache socket address not found.") + // os.Exit(1) should terminate the process + return + } + memcacheClient = memcache.New(addr) + err := memcacheClient.Ping() + if err != nil { + utils.Info.Printf("Memcache daemon not alive. Trying to start it") + cmd := exec.Command("/usr/bin/bash", "memcacheNativeInit.sh") + err := cmd.Run() + if err != nil { + utils.Error.Printf("Memcache daemon startup failed, err=%s", err) + // os.Exit(1) should terminate the process + return + } + } + utils.Info.Printf("Memcache daemon alive.") default: utils.Error.Printf("Unknown state storage type = %s", stateDbType) } diff --git a/server/vissv2server/uds-registration.docker.json b/server/vissv2server/uds-registration.docker.json index 2f81d4f0..a794e9e1 100644 --- a/server/vissv2server/uds-registration.docker.json +++ b/server/vissv2server/uds-registration.docker.json @@ -3,7 +3,8 @@ "root":"Vehicle", "serverFeeder":"/tmp/docker/server-feeder-channel.sock", "redis": "/tmp/docker/redisDB.sock", - "history": "/var/tmp/vissv2/histctrlserver.sock" + "memcache": "/tmp/docker/memcacheDB.sock", + "history": "/tmp/docker/histctrlserver.sock" } ] diff --git a/server/vissv2server/uds-registration.json b/server/vissv2server/uds-registration.json index d7ff5d20..b09a0c45 100644 --- a/server/vissv2server/uds-registration.json +++ b/server/vissv2server/uds-registration.json @@ -3,6 +3,7 @@ "root":"Vehicle", "serverFeeder":"/var/tmp/vissv2/serverFeeder.sock", "redis": "/var/tmp/vissv2/redisDB.sock", + "memcache": "/var/tmp/vissv2/memcacheDB.sock", "history": "/var/tmp/vissv2/histctrlserver.sock" } diff --git a/server/vissv2server/vissv2server.go b/server/vissv2server/vissv2server.go index 65480d9f..05d2e846 100644 --- a/server/vissv2server/vissv2server.go +++ b/server/vissv2server/vissv2server.go @@ -793,8 +793,8 @@ func main() { Default: "info"}) dryRun := parser.Flag("", "dryrun", &argparse.Options{Required: false, Help: "dry run to generate vsspathlist file", Default: false}) vssJson := parser.String("", "vssJson", &argparse.Options{Required: false, Help: "path and name vssPathlist json file", Default: "../vsspathlist.json"}) - stateDB := parser.Selector("s", "statestorage", []string{"sqlite", "redis", "apache-iotdb", "none"}, &argparse.Options{Required: false, - Help: "Statestorage must be either sqlite, redis, apache-iotdb, or none", Default: "redis"}) + stateDB := parser.Selector("s", "statestorage", []string{"sqlite", "redis", "memcache", "apache-iotdb", "none"}, &argparse.Options{Required: false, + Help: "Statestorage must be either sqlite, redis, memcache, apache-iotdb, or none", Default: "redis"}) historySupport := parser.Flag("j", "history", &argparse.Options{Required: false, Help: "Support for historic data requests", Default: false}) dbFile := parser.String("", "dbfile", &argparse.Options{ Required: false, diff --git a/tutorial/content/build-system/_index.md b/tutorial/content/build-system/_index.md index ae95ae12..04e415e9 100644 --- a/tutorial/content/build-system/_index.md +++ b/tutorial/content/build-system/_index.md @@ -49,12 +49,24 @@ There are multiple Software components on this repo, such as feeders, simulators If it is forgotten to be mentiond in the README, one way of determining whether a separate build is needed or not is to check the package statement in the source code. If it says "package main" it is a separate executable and shall then be built and run as described above. -### Loggging +### Logging Logging can be command line configured at startup. * logging level can be set to either of [trace, debug, info, warn, error, fatal, panic]. * logging output destination. It can either be written to file, or directed to standard output. The levels currently used are mainly info, warn, error. Info is appropriate during testing and debugging, while error is appropriate when performance is important. +### Transport protocols +Besides the transport protocols +* HTTP +* Websocke +* MQTT +that the specification lists, this server also supports +* gRPC +They are all except MQTT activated by the server per default. +To enable MQTT, or disable any other, the string array serverComponents in the vissv2server.go file contains a list of the components that are spawned on +separate threads by the main server process, and these can be commented in or out before building the server. +For the server to be operationable, the service manager, and at least one transport protocol must not be commented out. + ### Go modules Go modules are used in multiple places in this project, below follows some commands that may be helpful in managing this. diff --git a/utils/common.go b/utils/common.go index 828e480f..3e0ea9d6 100644 --- a/utils/common.go +++ b/utils/common.go @@ -52,6 +52,7 @@ type UdsReg struct { RootName string `json:"root"` ServerFeeder string `json:"serverFeeder"` Redis string `json:"redis"` + Memcache string `json:"memcache"` History string `json:"history"` } @@ -83,7 +84,7 @@ func GetUdsConn(path string, connectionName string) net.Conn { func GetUdsPath(path string, connectionName string) string { root := ExtractRootName(path) - Info.Printf("GetUdsPath:root=%s, connectionName=%s", root, connectionName) +// Info.Printf("GetUdsPath:root=%s, connectionName=%s", root, connectionName) for i := 0; i < len(udsRegList); i++ { if root == udsRegList[i].RootName { return getSocketPath(i, connectionName) @@ -99,6 +100,8 @@ func getSocketPath(listIndex int, connectionName string) string { return udsRegList[listIndex].ServerFeeder case "redis": return udsRegList[listIndex].Redis + case "memcache": + return udsRegList[listIndex].Memcache case "history": return udsRegList[listIndex].History default: @@ -325,7 +328,7 @@ func FileExists(filename string) bool { func ExtractRootName(path string) string { dotDelimiter := strings.Index(path, ".") if dotDelimiter == -1 { - Info.Print("ExtractRootName():Could not find root node name in path=%s", path) + Info.Print("ExtractRootName():Dot limited segments not found in path=%s", path) return path } return path[:dotDelimiter]