Ewm3143 apply diffcal during norm workflow

src/snapred/backend/dao/WorkspaceMetadata.py

@@ -1,38 +1,42 @@
-from typing import Literal, get_args
-
 from pydantic import BaseModel, ConfigDict
 
-# possible states for diffraction calibration
+from snapred.meta.Enum import StrEnum
 
 UNSET: str = "unset"
 
-DIFFCAL_METADATA_STATE = Literal[
-    UNSET,  # the default condition before any settings
-    "exists",  # the state exists and the corresponding .h5 file located
-    "alternate",  # the user supplied an alternate .h5 that they believe is usable.
-    "none",  # proceed using the defaul (IDF) geometry.
-]
 
-diffcal_metadata_state_list = list(get_args(DIFFCAL_METADATA_STATE))
+class DiffcalStateMetadata(StrEnum):
+    """Class to hold tags related to a workspace"""
+
+    UNSET: str = UNSET  # the default condition before any settings
+    DEFAULT: str = "default"  # the default condition before any settings
+    EXISTS: str = "exists"  # the state exists and the corresponding .h5 file located
+    ALTERNATE: str = "alternate"  # the user supplied an alternate .h5 that they believe is usable.
+    NONE: str = "none"  # proceed using the defaul (IDF) geometry.
+
 
+# TODO: Remove
+diffcal_metadata_state_list = [e.value for e in DiffcalStateMetadata]
+
+
+class NormalizationStateMetadata(StrEnum):
+    """Class to hold tags related to a workspace"""
 
-# possible states for normalization
+    UNSET: str = UNSET  # the default condition before any settings
+    EXISTS: str = "exists"  # the state exists and the corresponding .h5 file located
+    ALTERNATE: str = "alternate"  # the user supplied an alternate .h5 that they believe is usable
+    NONE: str = "none"  # proceed without applying any normalization
+    FAKE: str = "fake"  # proceed by creating a "fake vanadium"
 
-NORMCAL_METADATA_STATE = Literal[
-    UNSET,  # the default condition before any settings
-    "exists",  # the state exists and the corresponding .h5 file located
-    "alternate",  # the user supplied an alternate .h5 that they believe is usable
-    "none",  # proceed without applying any normalization
-    "fake",  # proceed by creating a "fake vanadium"
-]
 
-normcal_metadata_state_list = list(get_args(NORMCAL_METADATA_STATE))
+# TODO: rename this to normalization_metadata_state_list or remove it
+normcal_metadata_state_list = [e.value for e in NormalizationStateMetadata]
 
 
 class WorkspaceMetadata(BaseModel):
     """Class to hold tags related to a workspace"""
 
-    diffcalState: DIFFCAL_METADATA_STATE = UNSET
-    normalizationState: NORMCAL_METADATA_STATE = UNSET
+    diffcalState: DiffcalStateMetadata = UNSET
+    normalizationState: NormalizationStateMetadata = UNSET
 
-    model_config = ConfigDict(extra="forbid")
+    model_config = ConfigDict(extra="forbid", use_enum_values=True)

src/snapred/backend/data/DataFactoryService.py

@@ -54,6 +54,9 @@
     def constructStateId(self, runId: str):
         return self.lookupService.generateStateId(runId)
 
+    def stateExists(self, runId: str):
+        return self.lookupService.stateExists(runId)
+
     def getCalibrantSample(self, filePath):
         return self.lookupService.readCalibrantSample(filePath)
 

src/snapred/backend/data/GroceryService.py

@@ -17,6 +17,7 @@
 from snapred.backend.dao.indexing.Versioning import VERSION_DEFAULT
 from snapred.backend.dao.ingredients import GroceryListItem
 from snapred.backend.dao.state import DetectorState
+from snapred.backend.dao.WorkspaceMetadata import WorkspaceMetadata
 from snapred.backend.data.LocalDataService import LocalDataService
 from snapred.backend.log.logger import snapredLogger
 from snapred.backend.recipe.algorithm.MantidSnapper import MantidSnapper
@@ -462,6 +463,19 @@
             ws = None
         return ws
 
+    def writeWorkspaceMetadataAsTags(self, workspaceName: WorkspaceName, workspaceMetadata: WorkspaceMetadata):
+        """
+        Write workspace metadata to the workspace as tags.
+
+        :param workspaceName: the name of the workspace to write the metadata to
+        :type workspaceName: WorkspaceName
+        :param workspaceMetadata: the metadata to write to the workspace
+        :type workspaceMetadata: WorkspaceMetadata
+        """
+        metadata = workspaceMetadata.dict()
+        for logname in metadata.keys():
+            self.setWorkspaceTag(workspaceName, logname, metadata[logname])
+
     def getWorkspaceTag(self, workspaceName: str, logname: str):
         """
         Simple wrapper to get a workspace metadata tag, for the service layer.

src/snapred/backend/data/Indexer.py

@@ -20,7 +20,7 @@
 from snapred.backend.dao.normalization.NormalizationRecord import NormalizationRecord
 from snapred.backend.dao.reduction.ReductionRecord import ReductionRecord
 from snapred.backend.log.logger import snapredLogger
-from snapred.meta.mantid.AllowedPeakTypes import StrEnum
+from snapred.meta.Enum import StrEnum
 from snapred.meta.mantid.WorkspaceNameGenerator import ValueFormatter as wnvf
 from snapred.meta.redantic import parse_file_as, write_model_list_pretty, write_model_pretty
 

src/snapred/backend/data/LocalDataService.py

@@ -239,6 +239,12 @@
         ipts = GetIPTS(RunNumber=runNumber, Instrument=instrumentName)
         return str(ipts)
 
+    def stateExists(self, runId: str) -> bool:
+        stateId, _ = self.generateStateId(runId)
+        statePath = self.constructCalibrationStateRoot(stateId)
+        # Shouldnt need to check lite as we init both at the same time
+        return statePath.exists()
+
     def workspaceIsInstance(self, wsName: str, wsType: Any) -> bool:
         # Is the workspace an instance of the specified type.
         if not mtd.doesExist(wsName):
@@ -327,14 +333,20 @@
     def constructCalibrationStateRoot(self, stateId) -> Path:
         return Path(Config["instrument.calibration.powder.home"], str(stateId))
 
+    def _getLiteModeString(self, useLiteMode: bool) -> str:
+        return "lite" if useLiteMode else "native"
+
     def _constructCalibrationStatePath(self, stateId, useLiteMode) -> Path:
-        mode = "lite" if useLiteMode else "native"
+        mode = self._getLiteModeString(useLiteMode)
         return self.constructCalibrationStateRoot(stateId) / mode / "diffraction"
 
     def _constructNormalizationStatePath(self, stateId, useLiteMode) -> Path:
-        mode = "lite" if useLiteMode else "native"
+        mode = self._getLiteModeString(useLiteMode)
         return self.constructCalibrationStateRoot(stateId) / mode / "normalization"
 
+    def _hasWritePermissionsCalibrationStateRoot(self) -> bool:
+        return self.checkWritePermissions(Path(Config["instrument.calibration.powder.home"]))
+
     # reduction paths #
 
     @validate_call
@@ -745,7 +757,12 @@
     @validate_call
     def readCalibrationState(self, runId: str, useLiteMode: bool, version: Optional[int] = None):
         if not self.calibrationExists(runId, useLiteMode):
-            raise RecoverableException.stateUninitialized(runId, useLiteMode)
+            if self._hasWritePermissionsCalibrationStateRoot():
+                raise RecoverableException.stateUninitialized(runId, useLiteMode)
+            else:
+                raise RuntimeError(
+                    "No calibration exists, and you lack permissions to create one, please contact your CIS."
+                )
 
         indexer = self.calibrationIndexer(runId, useLiteMode)
         # NOTE if we prefer latest version in index, uncomment below

src/snapred/backend/error/ContinueWarning.py

@@ -12,6 +12,7 @@ class ContinueWarning(Exception):
     class Type(Flag):
         UNSET = 0
         MISSING_DIFFRACTION_CALIBRATION = auto()
+        DEFAULT_DIFFRACTION_CALIBRATION = auto()
         MISSING_NORMALIZATION = auto()
         LOW_PEAK_COUNT = auto()
         NO_WRITE_PERMISSIONS = auto()

src/snapred/backend/error/RecoverableException.py

@@ -4,6 +4,7 @@
 from pydantic import BaseModel
 
 from snapred.backend.log.logger import snapredLogger
+from snapred.meta.decorators.ExceptionHandler import extractTrueStacktrace
 
 logger = snapredLogger.getLogger(__name__)
 
@@ -39,6 +40,7 @@ def __init__(self, message: str, flags: "Type" = 0, data: Optional[Any] = None):
         RecoverableException.Model.update_forward_refs()
         RecoverableException.Model.model_rebuild(force=True)
         self.model = RecoverableException.Model(message=message, flags=flags, data=data)
+        logger.error(f"{extractTrueStacktrace()}")
         super().__init__(message)
 
     def parse_raw(raw):

src/snapred/backend/recipe/PreprocessReductionRecipe.py

@@ -31,23 +31,37 @@
         self.sampleWs = groceries["inputWorkspace"]
         self.diffcalWs = groceries.get("diffcalWorkspace", "")
         self.maskWs = groceries.get("maskWorkspace", "")
+        self.outputWs = groceries.get("outputWorkspace", groceries["inputWorkspace"])
+
+    def _validateGrocery(self, key, ws):
+        # Skip validation for outputWorkspace
+        # It is either the inputWorkspace or a new workspace created he
+        if key != "outputWorkspace":
+            super()._validateGrocery(key, ws)
 
     def queueAlgos(self):
         """
         Queues up the processing algorithms for the recipe.
         Requires: unbagged groceries and chopped ingredients.
         """
 
+        if self.outputWs != self.sampleWs:
+            self.mantidSnapper.CloneWorkspace(
+                "Cloning workspace...",
+                InputWorkspace=self.sampleWs,
+                OutputWorkspace=self.outputWs,
+            )
+
         if self.maskWs:
             self.mantidSnapper.MaskDetectorFlags(
                 "Applying pixel mask...",
                 MaskWorkspace=self.maskWs,
-                OutputWorkspace=self.sampleWs,
+                OutputWorkspace=self.outputWs,
             )
 
         if self.diffcalWs:
             self.mantidSnapper.ApplyDiffCal(
-                "Applying diffcal..", InstrumentWorkspace=self.sampleWs, CalibrationWorkspace=self.diffcalWs
+                "Applying diffcal..", InstrumentWorkspace=self.outputWs, CalibrationWorkspace=self.diffcalWs
             )
 
     def cook(self, ingredients: Ingredients, groceries: Dict[str, str]) -> Dict[str, Any]:

src/snapred/backend/recipe/Recipe.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Dict, Generic, List, Set, Tuple, TypeVar, get_args
+from typing import Any, Dict, Generic, List, Set, Tuple, TypeVar, get_args
 
 from pydantic import BaseModel, ValidationError
 
@@ -15,6 +15,8 @@
 
 
 class Recipe(ABC, Generic[Ingredients]):
+    _Ingredients: Any
+
     def __init__(self, utensils: Utensils = None):
         """
         Sets up the recipe with the necessary utensils.
@@ -25,6 +27,9 @@
             utensils.PyInit()
         self.mantidSnapper = utensils.mantidSnapper
 
+    def __init_subclass__(cls) -> None:
+        cls._Ingredients = get_args(cls.__orig_bases__[0])[0]  # type: ignore
+
     ## Abstract methods which MUST be implemented
 
     @abstractmethod
@@ -54,31 +59,52 @@
         """
         return {}
 
-    def validateInputs(self, ingredients: Ingredients, groceries: Dict[str, WorkspaceName]):
+    @classmethod
+    def Ingredients(cls, **kwargs):
+        return cls._Ingredients(**kwargs)
+
+    def _validateGrocery(self, key, ws):
         """
-        Validate the input properties before chopping or unbagging
+        Validate the given grocery
+        """
+        if ws is None:
+            raise RuntimeError(f"The workspace property {key} was not found in the groceries")
+
+        if not isinstance(key, str):
+            raise ValueError("The key for the grocery must be a string.")
+
+        if not isinstance(ws, list):
+            ws = [ws]
+
+        for wsStr in ws:
+            if isinstance(wsStr, str) and not self.mantidSnapper.mtd.doesExist(wsStr):
+                raise RuntimeError(f"The indicated workspace {wsStr} not found in Mantid ADS.")
+
+    def _validateIngredients(self, ingredients: Ingredients):
+        """
+        Validate the ingredients before chopping
         """
         if ingredients is not None and not isinstance(ingredients, BaseModel):
             raise ValueError("Ingredients must be a Pydantic BaseModel.")
         # cast the ingredients into the Ingredients type
         try:
             # to run the same as Ingredients.model_validate(ingredients)
             if ingredients is not None:
-                get_args(self.__orig_bases__[0])[0].model_validate(ingredients.dict())
+                self._Ingredients.model_validate(ingredients.dict())
         except ValidationError as e:
-            raise e
+            raise ValueError(f"Invalid ingredients: {e}")
+
+    def validateInputs(self, ingredients: Ingredients, groceries: Dict[str, WorkspaceName]):
+        """
+        Validate the input properties before chopping or unbagging
+        """
+        self._validateIngredients(ingredients)
         # ensure all of the given workspaces exist
         # NOTE may need to be tweaked to ignore output workspaces...
         logger.info(f"Validating the given workspaces: {groceries.values()}")
         for key in self.mandatoryInputWorkspaces():
             ws = groceries.get(key)
-            if ws is None:
-                raise RuntimeError(f"The workspace property {key} was not found in the groceries")
-            if not isinstance(ws, list):
-                ws = [ws]
-            for wsStr in ws:
-                if isinstance(wsStr, str) and not self.mantidSnapper.mtd.doesExist(wsStr):
-                    raise RuntimeError(f"The indicated workspace {wsStr} not found in Mantid ADS.")
+            self._validateGrocery(key, ws)
 
     def stirInputs(self):
         """

src/snapred/backend/recipe/ReductionGroupProcessingRecipe.py

@@ -1,11 +1,10 @@
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List, Set, Tuple
 
 from snapred.backend.dao.ingredients import ReductionGroupProcessingIngredients as Ingredients
 from snapred.backend.error.AlgorithmException import AlgorithmException
 from snapred.backend.log.logger import snapredLogger
-from snapred.backend.recipe.Recipe import Recipe
+from snapred.backend.recipe.Recipe import Recipe, WorkspaceName
 from snapred.meta.decorators.Singleton import Singleton
-from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName
 
 logger = snapredLogger.getLogger(__name__)
 
@@ -70,8 +69,8 @@
         )
         self.outputWS = self.rawInput
 
-    def validateInputs(self, ingredients: Ingredients, groceries: Dict[str, WorkspaceName]):
-        pass
+    def mandatoryInputWorkspaces(self) -> Set[WorkspaceName]:
+        return {"inputWorkspace", "groupingWorkspace"}
 
     def execute(self):
         """

src/snapred/backend/recipe/ReductionRecipe.py

@@ -117,11 +117,20 @@
     def _applyRecipe(self, recipe: Type[Recipe], ingredients_, **kwargs):
         if "inputWorkspace" in kwargs:
             inputWorkspace = kwargs["inputWorkspace"]
+            if not inputWorkspace:
+                self.logger().debug(f"{recipe.__name__} :: Skipping recipe with default empty input workspace")
+                return
             if self.mantidSnapper.mtd.doesExist(inputWorkspace):
                 self.groceries.update(kwargs)
                 recipe().cook(ingredients_, self.groceries)
             else:
-                self.logger().warning(f"Skipping {type(recipe)} as {inputWorkspace} does not exist.")
+                raise RuntimeError(
+                    (
+                        f"{recipe.__name__} ::"
+                        " Missing non-default input workspace with groceries:"
+                        f" {self.groceries} and kwargs: {kwargs}"
+                    )
+                )
 
     def _prepGroupingWorkspaces(self, groupingIndex: int):
         # TODO:  We need the wng to be able to deconstruct the workspace name