From 041c44cf411260d3a565d34b604af1cbc80c46c4 Mon Sep 17 00:00:00 2001 From: Ivan Perez Date: Sun, 22 Sep 2024 22:17:13 +0000 Subject: [PATCH 1/6] ogma-core: Use template expansion to generate cFS monitoring application. Refs #157. The cFS application generation backend has a fixed template that it uses to generate the cFS application. That template does not fit all use cases, so we are finding users heavily modifying the output (which is hard to keep up with when there are changes), and or not using ogma altogether for that reason. This commit modifies the ogma-core cfs command to use mustache to generate the cFS monitoring application template using variable expansion. --- ogma-core/ogma-core.cabal | 4 + ogma-core/src/Command/CFSApp.hs | 239 ++++++------------ .../copilot-cfs/fsw/src/copilot_cfs.c | 21 +- 3 files changed, 85 insertions(+), 179 deletions(-) diff --git a/ogma-core/ogma-core.cabal b/ogma-core/ogma-core.cabal index 975a0f8..891e3e8 100644 --- a/ogma-core/ogma-core.cabal +++ b/ogma-core/ogma-core.cabal @@ -106,8 +106,12 @@ library base >= 4.11.0.0 && < 5 , aeson >= 2.0.0.0 && < 2.2 , bytestring + , Cabal >= 2.4 && < 3.10 + , directory >= 1.3.1.0 && < 1.4 , filepath + , microstache >= 1.0 && < 1.1 , mtl + , text >= 1.2.3.1 && < 2.1 , ogma-extra >= 1.4.1 && < 1.5 , ogma-language-c >= 1.4.1 && < 1.5 diff --git a/ogma-core/src/Command/CFSApp.hs b/ogma-core/src/Command/CFSApp.hs index 26bdc8f..de195cd 100644 --- a/ogma-core/src/Command/CFSApp.hs +++ b/ogma-core/src/Command/CFSApp.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE OverloadedStrings #-} -- Copyright 2020 United States Government as represented by the Administrator -- of the National Aeronautics and Space Administration. All Rights Reserved. -- @@ -45,12 +46,20 @@ module Command.CFSApp where -- External imports -import qualified Control.Exception as E -import Data.List ( find ) -import System.FilePath ( () ) - --- External imports: auxiliary -import System.Directory.Extra ( copyDirectoryRecursive ) +import qualified Control.Exception as E +import Control.Monad (filterM, forM_) +import Data.Aeson (Value (..), decode, object, (.=)) +import qualified Data.ByteString.Lazy as B +import Data.List (find) +import Data.Text (Text) +import Data.Text.Lazy (pack, unpack) +import Data.Text.Lazy.Encoding (encodeUtf8) +import Distribution.Simple.Utils (getDirectoryContentsRecursive) +import System.Directory (createDirectoryIfMissing, + doesFileExist) +import System.FilePath (makeRelative, splitFileName, ()) +import Text.Microstache (compileMustacheFile, + compileMustacheText, renderMustache) -- Internal imports: auxiliary import Command.Result ( Result (..) ) @@ -101,8 +110,6 @@ cFSApp targetDir varNameFile varDBFile = do let templateDir = dataDir "templates" "copilot-cfs" E.handle (return . cannotCopyTemplate) $ do - -- Expand template - copyDirectoryRecursive templateDir targetDir let f n o@(oVars, oIds, oInfos, oDatas) = case variableMap varDB n of @@ -113,10 +120,18 @@ cFSApp targetDir varNameFile varDBFile = do -- This is a Data.List.unzip4 let (vars, ids, infos, datas) = foldr f ([], [], [], []) varNames - let cfsFileName = targetDir "fsw" "src" "copilot_cfs.c" - cfsFileContents = unlines $ fileContents vars ids infos datas + let (variablesS, msgSubscriptionS, msgCasesS, msgHandlerS) = + appComponents vars ids infos datas + subst = object $ + [ "variablesS" .= pack variablesS + , "msgSubscriptionsS" .= pack msgSubscriptionS + , "msgCasesS" .= pack msgCasesS + , "msgHandlerS" .= pack msgHandlerS + ] + + -- Expand template + copyTemplate templateDir subst targetDir - writeFile cfsFileName cfsFileContents return Success -- | Predefined list of Icarous variables that are known to Ogma @@ -170,9 +185,10 @@ data MsgData = MsgData , msgDataVarType :: String } --- | Return the contents of the main CFS application. -fileContents :: [VarDecl] -> [MsgInfoId] -> [MsgInfo] -> [MsgData] -> [String] -fileContents variables msgIds msgNames msgDatas = cfsFileContents +-- | Return the components that are customized in a cFS application. +appComponents :: [VarDecl] -> [MsgInfoId] -> [MsgInfo] -> [MsgData] + -> (String, String, String, String) +appComponents variables msgIds msgNames msgDatas = cfsFileContents where variablesS = unlines $ map toVarDecl variables toVarDecl varDecl = varDeclType varDecl ++ " " ++ varDeclName varDecl ++ ";" @@ -205,157 +221,11 @@ fileContents variables msgIds msgNames msgDatas = cfsFileContents ] cfsFileContents = - [ "/********************************************************************" - , "** File: copilot_cfs.c" - , "**" - , "** Purpose:" - , "** This file contains the source code for the Copilot App." - , "**" - , "*********************************************************************/" - , "" - , "/*" - , "** Include Files:" - , "*/" - , "" - , "#include \"copilot_cfs.h\"" - , "#include \"copilot_cfs_perfids.h\"" - , "#include \"copilot_cfs_msgids.h\"" - , "#include \"copilot_cfs_msg.h\"" - , "#include \"copilot_cfs_events.h\"" - , "#include \"copilot_cfs_version.h\"" - , "#include \"Icarous_msgids.h\"" - , "#include \"Icarous_msg.h\"" - , "" - , variablesS - , "void split(void);" - , "void step(void);" - , "" - , "/*" - , "** global data" - , "*/" - , "" - , "copilot_hk_tlm_t COPILOT_HkTelemetryPkt;" - , "CFE_SB_PipeId_t COPILOT_CommandPipe;" - , "CFE_SB_MsgPtr_t COPILOTMsgPtr;" - , "" - , "static CFE_EVS_BinFilter_t COPILOT_EventFilters[] =" - , " { /* Event ID mask */" - , " {COPILOT_STARTUP_INF_EID, 0x0000}," - , " {COPILOT_COMMAND_ERR_EID, 0x0000}," - , " {COPILOT_COMMANDCPVIOL_INF_EID, 0x0000}," - , " };" - , "" - , "/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */" - , "/* COPILOT_AppMain() -- App entry point and main process loop */" - , "/* */" - , "/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */" - , "void COPILOT_AppMain( void )" - , "{" - , " int32 status;" - , " uint32 RunStatus = CFE_ES_APP_RUN;" - , "" - , " CFE_ES_PerfLogEntry(COPILOT_CFS_PERF_ID);" - , "" - , " COPILOT_AppInit();" - , "" - , " /*" - , " ** COPILOT Runloop" - , " */" - , " while (CFE_ES_RunLoop(&RunStatus) == TRUE)" - , " {" - , " CFE_ES_PerfLogExit(COPILOT_CFS_PERF_ID);" - , "" - , " // Pend on receipt of command packet" - , " // (timeout set to 500 millisecs)" - , " status = CFE_SB_RcvMsg (&COPILOTMsgPtr," - , " COPILOT_CommandPipe," - , " 500);" - , " " - , " CFE_ES_PerfLogEntry(COPILOT_CFS_PERF_ID);" - , "" - , " if (status == CFE_SUCCESS)" - , " {" - , " COPILOT_ProcessCommandPacket();" - , " }" - , "" - , " }" - , "" - , " CFE_ES_ExitApp(RunStatus);" - , "" - , "} /* End of COPILOT_AppMain() */" - , "" - , "/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */" - , "/* */" - , "/* COPILOT_AppInit() -- initialization */" - , "/* */" - , "/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */" - , "void COPILOT_AppInit(void)" - , "{" - , " // Register the app with Executive services" - , " CFE_ES_RegisterApp();" - , "" - , " // Register the events" - , " CFE_EVS_Register(COPILOT_EventFilters," - , " sizeof(COPILOT_EventFilters) / " - ++ "sizeof(CFE_EVS_BinFilter_t)," - , " CFE_EVS_BINARY_FILTER);" - , "" - , " // Create the Software Bus command pipe and subscribe to " - , " // housekeeping messages" - , " CFE_SB_CreatePipe(&COPILOT_CommandPipe, COPILOT_PIPE_DEPTH," - ++ "\"COPILOT_CMD_PIPE\");" + ( variablesS , msgSubscriptionS - , "" - , " CFE_EVS_SendEvent (COPILOT_STARTUP_INF_EID," - , " CFE_EVS_INFORMATION," - , " \"COPILOT App Initialized. Ver %d.%d.%d.%d\"," - , " COPILOT_CFS_MAJOR_VERSION," - , " COPILOT_CFS_MINOR_VERSION, " - , " COPILOT_CFS_REVISION, " - , " COPILOT_CFS_MISSION_REV);" - , "" - , "} /* End of COPILOT_AppInit() */" - , "" - , "/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */" - , "/* Name: COPILOT_ProcessCommandPacket */" - , "/* */" - , "/* Purpose: */" - , "/* This routine will process any packet that is received */" - , "/* on the COPILOT command pipe. */" - , "/* */" - , "/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */" - , "void COPILOT_ProcessCommandPacket(void)" - , "{" - , " CFE_SB_MsgId_t MsgId;" - , "" - , " MsgId = CFE_SB_GetMsgId(COPILOTMsgPtr);" - , "" - , " switch (MsgId)" - , " {" , msgCasesS - , " default:" - , " COPILOT_HkTelemetryPkt.copilot_command_error_count++;" - , " CFE_EVS_SendEvent(COPILOT_COMMAND_ERR_EID,CFE_EVS_ERROR," - , " " - ++ "\"COPILOT: invalid command packet,MID = 0x%x\"," - , " MsgId);" - , " break;" - , " }" - , "" - , " return;" - , "" - , "} /* End COPILOT_ProcessCommandPacket */" - , "" , msgHandlerS - , "" - , "/**" - , " * Report copilot property violations." - , " */" - , "void split(void) {" - , " CFE_EVS_SendEvent(COPILOT_COMMANDCPVIOL_INF_EID, CFE_EVS_ERROR," - , " \"COPILOT: violation\");" - , "}" - ] + ) -- * Exception handlers @@ -429,3 +299,48 @@ ecCannotEmptyVarList = 1 -- permissions or some I/O error. ecCannotCopyTemplate :: ErrorCode ecCannotCopyTemplate = 1 + +-- * Generic template handling + +-- | Copy a template directory into a target location, expanding variables +-- provided in a map in a JSON value, both in the file contents and in the +-- filepaths themselves. +copyTemplate :: FilePath -> Value -> FilePath -> IO () +copyTemplate templateDir subst targetDir = do + + -- Get all files (not directories) in the template dir. To keep a directory, + -- create an empty file in it (e.g., .keep). + tmplContents <- map (templateDir ) . filter (`notElem` ["..", "."]) + <$> getDirectoryContentsRecursive templateDir + tmplFiles <- filterM doesFileExist tmplContents + + -- Copy files to new locations, expanding their name and contents as + -- mustache templates. + forM_ tmplFiles $ \fp -> do + + -- New file name in target directory, treating file + -- name as mustache template. + let fullPath = targetDir newFP + where + -- If file name has mustache markers, expand, otherwise use + -- relative file path + newFP = either (const relFP) + (unpack . (`renderMustache` subst)) + fpAsTemplateE + + -- Local file name within template dir + relFP = makeRelative templateDir fp + + -- Apply mustache substitutions to file name + fpAsTemplateE = compileMustacheText "fp" (pack relFP) + + -- File contents, treated as a mustache template. + contents <- encodeUtf8 <$> (`renderMustache` subst) + <$> compileMustacheFile fp + + -- Create target directory if necessary + let dirName = fst $ splitFileName fullPath + createDirectoryIfMissing True dirName + + -- Write expanded contents to expanded file path + B.writeFile fullPath contents diff --git a/ogma-core/templates/copilot-cfs/fsw/src/copilot_cfs.c b/ogma-core/templates/copilot-cfs/fsw/src/copilot_cfs.c index 74917cf..7f9d0ff 100644 --- a/ogma-core/templates/copilot-cfs/fsw/src/copilot_cfs.c +++ b/ogma-core/templates/copilot-cfs/fsw/src/copilot_cfs.c @@ -19,7 +19,7 @@ #include "Icarous_msgids.h" #include "Icarous_msg.h" -position_t my_position; +{{variablesS}} void split(void); void step(void); @@ -98,7 +98,7 @@ void COPILOT_AppInit(void) ** messages */ CFE_SB_CreatePipe(&COPILOT_CommandPipe, COPILOT_PIPE_DEPTH,"COPILOT_CMD_PIPE"); - CFE_SB_Subscribe(ICAROUS_POSITION_MID, COPILOT_CommandPipe); +{{msgSubscriptionsS}} CFE_EVS_SendEvent (COPILOT_STARTUP_INF_EID, CFE_EVS_INFORMATION, "COPILOT App Initialized. Version %d.%d.%d.%d", @@ -125,9 +125,7 @@ void COPILOT_ProcessCommandPacket(void) switch (MsgId) { - case ICAROUS_POSITION_MID: - COPILOT_ProcessIcarousPosition(); - break; +{{ msgCasesS }} default: COPILOT_HkTelemetryPkt.copilot_command_error_count++; CFE_EVS_SendEvent(COPILOT_COMMAND_ERR_EID,CFE_EVS_ERROR, @@ -139,18 +137,7 @@ void COPILOT_ProcessCommandPacket(void) } /* End COPILOT_ProcessCommandPacket */ -/** - * Make ICAROUS data available to Copilot and run monitors. - */ -void COPILOT_ProcessIcarousPosition(void) -{ - position_t* msg; - msg = (position_t*) COPILOTMsgPtr; - my_position = *msg; - - // Run all copilot monitors. - step(); -} +{{msgHandlerS}} /** * Report copilot property violations. From 7c9db6d5edb264513dbc9acd6be1fc1b2d2b5254 Mon Sep 17 00:00:00 2001 From: Ivan Perez Date: Tue, 12 Nov 2024 00:29:57 +0000 Subject: [PATCH 2/6] ogma-core: Enable customizing template directory in cFS app generation. Refs #157. The cFS application generation backend has a fixed template that it uses to generate the cFS application. That template does not fit all use cases, so we are finding users heavily modifying the output (which is hard to keep up with when there are changes), and or not using ogma altogether for that reason. A recent commit introduced the ability to use mustache to expand variables in a template. This commit modifies the cfs command to accept an additional argument that points to a user-provided directory with a custom template. --- ogma-core/src/Command/CFSApp.hs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ogma-core/src/Command/CFSApp.hs b/ogma-core/src/Command/CFSApp.hs index de195cd..59bd5ce 100644 --- a/ogma-core/src/Command/CFSApp.hs +++ b/ogma-core/src/Command/CFSApp.hs @@ -71,13 +71,14 @@ import Paths_ogma_core ( getDataDir ) -- | Generate a new CFS application connected to Copilot. cFSApp :: FilePath -- ^ Target directory where the application -- should be created. + -> Maybe FilePath -- ^ Directory where the template is to be found. -> FilePath -- ^ File containing a list of variables to make -- available to Copilot. -> Maybe FilePath -- ^ File containing a list of known variables -- with their types and the message IDs they -- can be obtained from. -> IO (Result ErrorCode) -cFSApp targetDir varNameFile varDBFile = do +cFSApp targetDir mTemplateDir varNameFile varDBFile = do -- We first try to open the two files we need to fill in details in the CFS -- app template. @@ -106,8 +107,11 @@ cFSApp targetDir varNameFile varDBFile = do Right varNames -> do -- Obtain template dir - dataDir <- getDataDir - let templateDir = dataDir "templates" "copilot-cfs" + templateDir <- case mTemplateDir of + Just x -> return x + Nothing -> do + dataDir <- getDataDir + return $ dataDir "templates" "copilot-cfs" E.handle (return . cannotCopyTemplate) $ do From 2905a0e1ccc76cc03a0e51470970cf7da0fd4a83 Mon Sep 17 00:00:00 2001 From: Ivan Perez Date: Tue, 12 Nov 2024 00:30:19 +0000 Subject: [PATCH 3/6] ogma-cli: Provide CLI argument to pass an auxiliary template source directory to cFS app command. Refs #157. The cFS application generation backend has a fixed template that it uses to generate the cFS application. That template does not fit all use cases, so we are finding users heavily modifying the output (which is hard to keep up with when there are changes), and or not using ogma altogether for that reason. A recent commit introduced into ogma-core the ability to use a custom provided template and expand variables using mustache. This commit exposes that new parameter to the user in the CLI. --- ogma-cli/src/CLI/CommandCFSApp.hs | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/ogma-cli/src/CLI/CommandCFSApp.hs b/ogma-cli/src/CLI/CommandCFSApp.hs index a557408..1a44a72 100644 --- a/ogma-cli/src/CLI/CommandCFSApp.hs +++ b/ogma-cli/src/CLI/CommandCFSApp.hs @@ -56,9 +56,10 @@ import Command.CFSApp ( ErrorCode, cFSApp ) -- | Options needed to generate the cFS application. data CommandOpts = CommandOpts - { cFSAppTarget :: String - , cFSAppVarNames :: String - , cFSAppVarDB :: Maybe String + { cFSAppTarget :: String + , cFSAppTemplateDir :: Maybe String + , cFSAppVarNames :: String + , cFSAppVarDB :: Maybe String } -- | Create (cFS) @@ -68,7 +69,11 @@ data CommandOpts = CommandOpts -- This is just an uncurried version of "Command.CFSApp". command :: CommandOpts -> IO (Result ErrorCode) command c = - cFSApp (cFSAppTarget c) (cFSAppVarNames c) (cFSAppVarDB c) + cFSApp + (cFSAppTarget c) + (cFSAppTemplateDir c) + (cFSAppVarNames c) + (cFSAppVarDB c) -- * CLI @@ -87,6 +92,13 @@ commandOptsParser = CommandOpts <> value "copilot-cfs-demo" <> help strCFSAppDirArgDesc ) + <*> optional + ( strOption + ( long "app-template-dir" + <> metavar "DIR" + <> help strCFSAppTemplateDirArgDesc + ) + ) <*> strOption ( long "variable-file" <> metavar "FILENAME" @@ -106,6 +118,11 @@ commandOptsParser = CommandOpts strCFSAppDirArgDesc :: String strCFSAppDirArgDesc = "Target directory" +-- | Argument template directory to cFS app generation command +strCFSAppTemplateDirArgDesc :: String +strCFSAppTemplateDirArgDesc = + "Directory holding cFS application source template" + -- | Argument variable list to cFS app generation command strCFSAppVarListArgDesc :: String strCFSAppVarListArgDesc = From ee6e14495ea996c07f3e3721f15e0c934abd780f Mon Sep 17 00:00:00 2001 From: Ivan Perez Date: Tue, 12 Nov 2024 04:01:06 +0000 Subject: [PATCH 4/6] ogma-cli: Update README to document cFS application template customization. Refs #157. The cFS application generation backend has a fixed template that it uses to generate the cFS application. That template does not fit all use cases, so we are finding users heavily modifying the output (which is hard to keep up with when there are changes), and or not using ogma altogether for that reason. Prior commits have expanded the command to allow for customization of the template using a user-provided directory and expanding variables in the template using mustache. This commit documents the new feature in the README. --- ogma-cli/README.md | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/ogma-cli/README.md b/ogma-cli/README.md index 6d22e14..b662355 100644 --- a/ogma-cli/README.md +++ b/ogma-cli/README.md @@ -115,9 +115,10 @@ flags to customize the list of known variables, so that projects can maintain their own variable databases beyond what Ogma includes by default. cFS applications are generated using the Ogma command `cfs`, which receives -three main arguments: +four main arguments: - `--app-target-dir DIR`: location where the cFS application files must be stored. +- `--app-template-dir DIR`: location of the cFS application template to use. - `--variable-file FILENAME`: a file containing a list of variables that must be made available to the monitor. - `--variable-db FILENAME`: a file containing a database of known variables, @@ -187,6 +188,45 @@ void COPILOT_ProcessIcarousPosition(void) } ``` +### Template Customization + +By default, Ogma uses a pre-defined template to generate the cFS monitoring +application. It's possible to customize the output by providing a directory +with a set of files with a cFS application template, which Ogma will use +instead. + +To choose this feature, one must call Ogma's `cfs` command with the argument +`--app-template-dir DIR`, where `DIR` is the path to a directory containing a +cFS application template. For example, assuming that the directory +`my_template` contains a custom cFS application template, one can execute: + +``` +$ ogma cfs --app-template-dir my_template/ --variable-db examples/cfs-variable-db --variable-file examples/cfs-variables +``` + +Ogma will copy the files in that directory to the target path, filling in +several holes with specific information: + +- `{{variablesS}}`: this will be replaced by a list of variable declarations, + one for each global variable that holds information read from the cFS +software bus that must be made accessible to the monitoring code. + +- `{{msgSubscriptionsS}}`: this will be replaced by a list of calls to + `CFE_SB_Subscribe`, subscribing to the necessary information coming in the +software bus. + +- `{{msgCasesS}}`: this will be replaced by a switch case statements that match + the ID of an incoming message, to handle information being received that must +be updated and would trigger a re-evaluation of the monitors. + +- `{{msgHandlerS}}`: this will be replaced by function definitions of the + functions that will be called to actually update the variables with +information coming from the software bus, and re-evaluate the monitors. + +We understand that this level of customization may be insufficient for your +application. If that is the case, feel free to reach out to our team to discuss +how we could make the template expansion system more versatile. + ## ROS Application Generation The Robot Operating System (ROS) is a framework to build robot applications. From 30f7a1693e27d01b7668409558a42907b284e80f Mon Sep 17 00:00:00 2001 From: Ivan Perez Date: Tue, 12 Nov 2024 04:58:28 +0000 Subject: [PATCH 5/6] ogma-core: Document changes in CHANGELOG. Refs #157. --- ogma-core/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ogma-core/CHANGELOG.md b/ogma-core/CHANGELOG.md index 6744d41..00dca11 100644 --- a/ogma-core/CHANGELOG.md +++ b/ogma-core/CHANGELOG.md @@ -3,6 +3,7 @@ ## [1.X.Y] - 2024-11-13 * Fix incorrect path when using Space ROS humble-2024.10.0 (#158). +* Use template expansion system to generate cFS monitoring application (#157). ## [1.4.1] - 2024-09-21 From a1a96818e5a60420c233a3a00d4117e95dfbb1b0 Mon Sep 17 00:00:00 2001 From: Ivan Perez Date: Tue, 12 Nov 2024 04:58:59 +0000 Subject: [PATCH 6/6] ogma-cli: Document changes in CHANGELOG. Refs #157. --- ogma-cli/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ogma-cli/CHANGELOG.md b/ogma-cli/CHANGELOG.md index 00e5d43..5b17da9 100644 --- a/ogma-cli/CHANGELOG.md +++ b/ogma-cli/CHANGELOG.md @@ -1,5 +1,9 @@ # Revision history for ogma-cli +## [1.X.Y] - 2024-11-11 + +* Provide ability to customize template in cfs command (#157). + ## [1.4.1] - 2024-09-21 * Version bump 1.4.1 (#155).