From aa8a11caf6e1049ca6738eb8222f7f6af53b3159 Mon Sep 17 00:00:00 2001 From: Darsh Date: Wed, 2 Oct 2024 10:52:57 -0400 Subject: [PATCH 01/23] Story deliverables. --- .../recipe/ArtificialNormalizationRecipe.py | 70 +++++++++++++++++++ src/snapred/backend/recipe/ReductionRecipe.py | 12 ++++ .../backend/service/ReductionService.py | 22 +++++- 3 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 src/snapred/backend/recipe/ArtificialNormalizationRecipe.py diff --git a/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py b/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py new file mode 100644 index 000000000..44c630641 --- /dev/null +++ b/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py @@ -0,0 +1,70 @@ +import json +from typing import Any, Dict, List, Tuple + +from snapred.backend.dao.ingredients import ArtificialNormalizationIngredients as Ingredients +from snapred.backend.log.logger import snapredLogger +from snapred.backend.recipe.Recipe import Recipe +from snapred.meta.decorators.Singleton import Singleton + +logger = snapredLogger.getLogger(__name__) + +Pallet = Tuple[Ingredients, Dict[str, str]] + + +@Singleton +class ArtificialNormalizationRecipe(Recipe[Ingredients]): + """ + The purpose of this recipe is to apply artificial normalization + using the CreateArtificialNormalizationAlgo algorithm. + """ + + def chopIngredients(self, ingredients: Ingredients): + self.peakWindowClippingSize = ingredients.peakWindowClippingSize + self.smoothingParameter = ingredients.smoothingParameter + self.decreaseParameter = ingredients.decreaseParameter + self.lss = ingredients.lss + + def unbagGroceries(self, groceries: Dict[str, Any]): + self.inputWS = groceries["inputWorkspace"] + self.outputWS = groceries.get("outputWorkspace", groceries["inputWorkspace"]) + + def queueAlgos(self): + """ + Queues up the processing algorithms for the recipe. + Requires: unbagged groceries and chopped ingredients. + """ + ingredients = { + "peakWindowClippingSize": self.peakWindowClippingSize, + "smoothingParameter": self.smoothingParameter, + "decreaseParameter": self.decreaseParameter, + "lss": self.lss, + } + ingredientsStr = json.dumps(ingredients) + + self.mantidSnapper.CreateArtificialNormalizationAlgo( + "Creating artificial normalization...", + InputWorkspace=self.inputWS, + OutputWorkspace=self.outputWS, + Ingredients=ingredientsStr, + ) + + def cook(self, ingredients: Ingredients, groceries: Dict[str, str]) -> Dict[str, Any]: + """ + Main interface method for the recipe. + Given the ingredients and groceries, it prepares, executes, and returns the final workspace. + """ + self.prep(ingredients, groceries) + self.execute() + return self.outputWS + + def cater(self, shipment: List[Pallet]) -> List[Dict[str, Any]]: + """ + A secondary interface method for the recipe. + It is a batched version of cook. + Given a shipment of ingredients and groceries, it prepares, executes and returns the final workspaces. + """ + output = [] + for ingredient, grocery in shipment: + output.append(self.cook(ingredient, grocery)) + self.execute() + return output diff --git a/src/snapred/backend/recipe/ReductionRecipe.py b/src/snapred/backend/recipe/ReductionRecipe.py index 551cda0ba..7fb1b9f02 100644 --- a/src/snapred/backend/recipe/ReductionRecipe.py +++ b/src/snapred/backend/recipe/ReductionRecipe.py @@ -1,8 +1,10 @@ from typing import Any, Dict, List, Tuple, Type +from snapred.backend.dao.ingredients import ArtificialNormalizationIngredients from snapred.backend.dao.ingredients import ReductionIngredients as Ingredients from snapred.backend.log.logger import snapredLogger from snapred.backend.recipe.ApplyNormalizationRecipe import ApplyNormalizationRecipe +from snapred.backend.recipe.ArtificialNormalizationRecipe import ArtificialNormalizationRecipe from snapred.backend.recipe.GenerateFocussedVanadiumRecipe import GenerateFocussedVanadiumRecipe from snapred.backend.recipe.PreprocessReductionRecipe import PreprocessReductionRecipe from snapred.backend.recipe.Recipe import Recipe, WorkspaceName @@ -225,6 +227,16 @@ def cook(self, ingredients: Ingredients, groceries: Dict[str, str]) -> Dict[str, Given the ingredients and groceries, it prepares, executes and returns the final workspace. """ self.prep(ingredients, groceries) + if not self.normalizationWs and self.sampleWs: + logger.info("Normalization is missing, applying artificial normalization...") + artificialIngredients = ArtificialNormalizationIngredients( + peakWindowClippingSize=5, # Replace with appropriate value or user input + smoothingParameter=0.5, # Replace with appropriate value or user input + decreaseParameter=True, + lss=True, + ) + artificialNormRecipe = ArtificialNormalizationRecipe() + self.normalizationWs = artificialNormRecipe.cook(artificialIngredients, groceries) return self.execute() def cater(self, shipment: List[Pallet]) -> List[Dict[str, Any]]: diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index d88a0d726..75ebb5498 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -3,7 +3,11 @@ from pathlib import Path from typing import Any, Dict, List -from snapred.backend.dao.ingredients import GroceryListItem, ReductionIngredients +from snapred.backend.dao.ingredients import ( + ArtificialNormalizationIngredients, + GroceryListItem, + ReductionIngredients, +) from snapred.backend.dao.reduction.ReductionRecord import ReductionRecord from snapred.backend.dao.request import ( FarmFreshIngredients, @@ -20,6 +24,7 @@ from snapred.backend.error.StateValidationException import StateValidationException from snapred.backend.log.logger import snapredLogger from snapred.backend.recipe.algorithm.MantidSnapper import MantidSnapper +from snapred.backend.recipe.ArtificialNormalizationRecipe import ArtificialNormalizationRecipe from snapred.backend.recipe.ReductionRecipe import ReductionRecipe from snapred.backend.service.Service import Service from snapred.backend.service.SousChef import SousChef @@ -308,8 +313,9 @@ def fetchReductionGroceries(self, request: ReductionRequest) -> Dict[str, Any]: :rtype: Dict[str, Any] """ # Fetch pixel masks - residentMasks = {} combinedMask = None + residentMasks = {} + # Check for existing pixel masks if request.pixelMasks: for mask in request.pixelMasks: match mask.tokens("workspaceType"): @@ -358,6 +364,18 @@ def fetchReductionGroceries(self, request: ReductionRequest) -> Dict[str, Any]: self.groceryClerk.name("normalizationWorkspace").normalization(request.runNumber, normVersion).useLiteMode( request.useLiteMode ).add() + elif calVersion: + ingredients = ArtificialNormalizationIngredients( + peakWindowClippingSize=5, # Replace with user input or default + smoothingParameter=0.5, # + decreaseParameter=True, # + lss=True, # + ) + groceries = {"inputWorkspace": "sample_WS"} + artificialNormRecipe = ArtificialNormalizationRecipe() + normWs = artificialNormRecipe.cook(ingredients, groceries) + + self.groceryClerk.name("normalizationWorkspace").set(normWs) request.versions = Versions( calVersion, From f0b05e566e42e3fd35bf7aa39793808c5e1e090e Mon Sep 17 00:00:00 2001 From: Darsh Date: Wed, 2 Oct 2024 15:07:21 -0400 Subject: [PATCH 02/23] Few updates --- .../recipe/algorithm/CreateArtificialNormalizationAlgo.py | 3 ++- src/snapred/backend/service/ReductionService.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/snapred/backend/recipe/algorithm/CreateArtificialNormalizationAlgo.py b/src/snapred/backend/recipe/algorithm/CreateArtificialNormalizationAlgo.py index 3d6263f14..387d8ca3b 100644 --- a/src/snapred/backend/recipe/algorithm/CreateArtificialNormalizationAlgo.py +++ b/src/snapred/backend/recipe/algorithm/CreateArtificialNormalizationAlgo.py @@ -9,6 +9,7 @@ WorkspaceUnitValidator, ) from mantid.kernel import Direction +from mantid.plots.axesfunctions import plot from snapred.backend.log.logger import snapredLogger from snapred.backend.recipe.algorithm.MantidSnapper import MantidSnapper @@ -135,7 +136,7 @@ def PyExec(self): smoothing=self.smoothingParameter, ) self.outputWorkspace.setY(i, clippedData) - + ax = plot(self.outputWorkspace, distribution=True) # noqa: F841 # Set the output workspace property self.setProperty("OutputWorkspace", self.outputWorkspaceName) diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index 75ebb5498..ab8abb8b3 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -371,7 +371,12 @@ def fetchReductionGroceries(self, request: ReductionRequest) -> Dict[str, Any]: decreaseParameter=True, # lss=True, # ) - groceries = {"inputWorkspace": "sample_WS"} + groceries = { + "inputWorkspace": self.groceryClerk.name("normalizationWorkspace") + .normalization(request.runNumber, normVersion) + .useLiteMode(request.useLiteMode) + .add() + } artificialNormRecipe = ArtificialNormalizationRecipe() normWs = artificialNormRecipe.cook(ingredients, groceries) From 7bd709d8529ca778b570dbc7ae9062353bd92be3 Mon Sep 17 00:00:00 2001 From: Darsh Date: Thu, 3 Oct 2024 16:31:07 -0400 Subject: [PATCH 03/23] Many tests. --- .../recipe/ArtificialNormalizationRecipe.py | 13 +-- src/snapred/backend/recipe/ReductionRecipe.py | 7 ++ .../CreateArtificialNormalizationAlgo.py | 4 +- .../outputs/APIServicePaths.json.new | 7 ++ ... test_CreateArtificialNormaliztionAlgo.py} | 0 .../test_ArtificialNormalizationRecipe.py | 94 +++++++++++++++++++ .../backend/recipe/test_ReductionRecipe.py | 23 ++++- 7 files changed, 139 insertions(+), 9 deletions(-) create mode 100644 tests/resources/outputs/APIServicePaths.json.new rename tests/unit/backend/recipe/algorithm/{test_CreateArtifificialNormaliztionAlgo.py => test_CreateArtificialNormaliztionAlgo.py} (100%) create mode 100644 tests/unit/backend/recipe/test_ArtificialNormalizationRecipe.py diff --git a/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py b/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py index 44c630641..381cbea73 100644 --- a/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py +++ b/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py @@ -19,10 +19,11 @@ class ArtificialNormalizationRecipe(Recipe[Ingredients]): """ def chopIngredients(self, ingredients: Ingredients): - self.peakWindowClippingSize = ingredients.peakWindowClippingSize - self.smoothingParameter = ingredients.smoothingParameter - self.decreaseParameter = ingredients.decreaseParameter - self.lss = ingredients.lss + self.ingredients = ingredients.copy() + self.peakWindowClippingSize = self.ingredients.peakWindowClippingSize + self.smoothingParameter = self.ingredients.smoothingParameter + self.decreaseParameter = self.ingredients.decreaseParameter + self.lss = self.ingredients.lss def unbagGroceries(self, groceries: Dict[str, Any]): self.inputWS = groceries["inputWorkspace"] @@ -54,8 +55,8 @@ def cook(self, ingredients: Ingredients, groceries: Dict[str, str]) -> Dict[str, Given the ingredients and groceries, it prepares, executes, and returns the final workspace. """ self.prep(ingredients, groceries) - self.execute() - return self.outputWS + return self.execute() + # return {"outputWorkspace": self.outputWS} def cater(self, shipment: List[Pallet]) -> List[Dict[str, Any]]: """ diff --git a/src/snapred/backend/recipe/ReductionRecipe.py b/src/snapred/backend/recipe/ReductionRecipe.py index 7fb1b9f02..353882c71 100644 --- a/src/snapred/backend/recipe/ReductionRecipe.py +++ b/src/snapred/backend/recipe/ReductionRecipe.py @@ -39,6 +39,13 @@ class ReductionRecipe(Recipe[Ingredients]): self.groupingWorkspaces = groceries["groupingWorkspaces"] """ + def __init__(self): + super().__init__() + self.sampleWs = None + self.normalizationWs = None + self.maskWs = None + self.groupingWorkspaces = [] + def logger(self): return logger diff --git a/src/snapred/backend/recipe/algorithm/CreateArtificialNormalizationAlgo.py b/src/snapred/backend/recipe/algorithm/CreateArtificialNormalizationAlgo.py index 387d8ca3b..7ca2287be 100644 --- a/src/snapred/backend/recipe/algorithm/CreateArtificialNormalizationAlgo.py +++ b/src/snapred/backend/recipe/algorithm/CreateArtificialNormalizationAlgo.py @@ -9,7 +9,6 @@ WorkspaceUnitValidator, ) from mantid.kernel import Direction -from mantid.plots.axesfunctions import plot from snapred.backend.log.logger import snapredLogger from snapred.backend.recipe.algorithm.MantidSnapper import MantidSnapper @@ -136,7 +135,8 @@ def PyExec(self): smoothing=self.smoothingParameter, ) self.outputWorkspace.setY(i, clippedData) - ax = plot(self.outputWorkspace, distribution=True) # noqa: F841 + self.outputWorkspace.setDistribution(True) + # Set the output workspace property self.setProperty("OutputWorkspace", self.outputWorkspaceName) diff --git a/tests/resources/outputs/APIServicePaths.json.new b/tests/resources/outputs/APIServicePaths.json.new new file mode 100644 index 000000000..126e21be3 --- /dev/null +++ b/tests/resources/outputs/APIServicePaths.json.new @@ -0,0 +1,7 @@ +{ + "mockService": { + "": { + "mockObject": "{\n \"properties\": {\n \"m_list\": {\n \"items\": {\n \"type\": \"number\"\n },\n \"title\": \"M List\",\n \"type\": \"array\"\n },\n \"m_float\": {\n \"title\": \"M Float\",\n \"type\": \"number\"\n },\n \"m_int\": {\n \"title\": \"M Int\",\n \"type\": \"integer\"\n },\n \"m_string\": {\n \"title\": \"M String\",\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"m_list\",\n \"m_float\",\n \"m_int\",\n \"m_string\"\n ],\n \"title\": \".MockObject'>\",\n \"type\": \"object\"\n}" + } + } +} diff --git a/tests/unit/backend/recipe/algorithm/test_CreateArtifificialNormaliztionAlgo.py b/tests/unit/backend/recipe/algorithm/test_CreateArtificialNormaliztionAlgo.py similarity index 100% rename from tests/unit/backend/recipe/algorithm/test_CreateArtifificialNormaliztionAlgo.py rename to tests/unit/backend/recipe/algorithm/test_CreateArtificialNormaliztionAlgo.py diff --git a/tests/unit/backend/recipe/test_ArtificialNormalizationRecipe.py b/tests/unit/backend/recipe/test_ArtificialNormalizationRecipe.py new file mode 100644 index 000000000..62d2b937b --- /dev/null +++ b/tests/unit/backend/recipe/test_ArtificialNormalizationRecipe.py @@ -0,0 +1,94 @@ +import json +import unittest +from unittest import mock + +import pytest +from mantid.simpleapi import ( + CreateSingleValuedWorkspace, + mtd, +) +from snapred.backend.dao.ingredients import ArtificialNormalizationIngredients as Ingredients +from snapred.backend.recipe.ArtificialNormalizationRecipe import ArtificialNormalizationRecipe as Recipe +from util.SculleryBoy import SculleryBoy + + +class TestArtificialNormalizationRecipe(unittest.TestCase): + sculleryBoy = SculleryBoy() + + def __make_groceries(self): + inputWorkspace = mtd.unique_name(prefix="inputworkspace") + outputWorkspace = "outputworkspace" # noqa: F841 + CreateSingleValuedWorkspace(OutputWorkspace=inputWorkspace) + + def test_chopIngredients(self): + recipe = Recipe() + ingredients = mock.Mock() + recipe.chopIngredients(ingredients) + assert ingredients.copy() == recipe.ingredients + + def test_unbagGroceries(self): + recipe = Recipe() + + groceries = { + "inputWorkspace": "sample", + "outputWorkspace": "output", + } + recipe.unbagGroceries(groceries) + assert recipe.inputWS == groceries["inputWorkspace"] + assert recipe.outputWS == groceries["outputWorkspace"] + + groceries = { + "inputWorkspace": "sample", + "outputWorkspace": "output", + } + recipe.unbagGroceries(groceries) + assert recipe.inputWS == groceries["inputWorkspace"] + assert recipe.outputWS == groceries["outputWorkspace"] + groceries = {} + with pytest.raises(KeyError): + recipe.unbagGroceries(groceries) + + def test_queueAlgos(self): + recipe = Recipe() + recipe.mantidSnapper = mock.Mock() + ingredients = Ingredients(peakWindowClippingSize=10, smoothingParameter=0.5, decreaseParameter=True, lss=True) + groceries = {"inputWorkspace": "inputworkspace", "outputWorkspace": "outputworkspace"} + + recipe.chopIngredients(ingredients) + recipe.unbagGroceries(groceries) + + recipe.queueAlgos() + + expected_ingredients_dict = { + "peakWindowClippingSize": 10, + "smoothingParameter": 0.5, + "decreaseParameter": True, + "lss": True, + } + expected_ingredients_str = json.dumps(expected_ingredients_dict) + + recipe.mantidSnapper.CreateArtificialNormalizationAlgo.assert_called_once_with( + "Creating artificial normalization...", + InputWorkspace="inputworkspace", + OutputWorkspace="outputworkspace", + Ingredients=expected_ingredients_str, + ) + + def test_cook(self): + recipe = Recipe() + recipe.prep = mock.Mock() + recipe.execute = mock.Mock() + recipe.cook(None, None) + recipe.prep.assert_called() + recipe.execute.assert_called() + + def test_cater(self): + recipe = Recipe() + recipe.cook = mock.Mock() + mockIngredients = mock.Mock() + mockGroceries = mock.Mock() + pallet = (mockIngredients, mockGroceries) + shipment = [pallet] + output = recipe.cater(shipment) + recipe.cook.assert_called_once_with(mockIngredients, mockGroceries) + assert output[0] == recipe.cook.return_value diff --git a/tests/unit/backend/recipe/test_ReductionRecipe.py b/tests/unit/backend/recipe/test_ReductionRecipe.py index ddd5b0cb4..f4253e09d 100644 --- a/tests/unit/backend/recipe/test_ReductionRecipe.py +++ b/tests/unit/backend/recipe/test_ReductionRecipe.py @@ -3,7 +3,7 @@ import pytest from mantid.simpleapi import CreateSingleValuedWorkspace, mtd -from snapred.backend.dao.ingredients import ReductionIngredients +from snapred.backend.dao.ingredients import ArtificialNormalizationIngredients, ReductionIngredients from snapred.backend.recipe.ReductionRecipe import ( ApplyNormalizationRecipe, GenerateFocussedVanadiumRecipe, @@ -289,6 +289,27 @@ def test_cloneIntermediateWorkspace(self): mock.ANY, InputWorkspace="input", OutputWorkspace="output" ) + @mock.patch("snapred.backend.recipe.ReductionRecipe.ArtificialNormalizationRecipe") + def test_apply_artificial_normalization(self, artificialNorm): + recipe = ReductionRecipe() + recipe.prep = mock.Mock() + recipe.execute = mock.Mock() + recipe.sampleWs = "sample_ws" + recipe.normalizationWs = None + groceries = {"inputWorkspace": "sample_ws"} + recipe.groceries = groceries + + artificialNorm().cook.return_value = "artificial_normalization_ws" + result = recipe.cook(mock.Mock(), groceries) # noqa: F841 + artificialNorm().cook.assert_called_once_with( + ArtificialNormalizationIngredients( + peakWindowClippingSize=5, smoothingParameter=0.5, decreaseParameter=True, lss=True + ), + groceries, + ) + assert recipe.normalizationWs == "artificial_normalization_ws" + recipe.execute.assert_called_once() + def test_execute(self): recipe = ReductionRecipe() recipe.groceries = {} From 87f8b056fbc9dd6ec4550808c9a2b4bd9921f7f5 Mon Sep 17 00:00:00 2001 From: Darsh Date: Tue, 8 Oct 2024 09:08:11 -0400 Subject: [PATCH 04/23] Delete extra test. --- .../test_CreateArtificialNormaliztionAlgo.py | 93 ------------------- 1 file changed, 93 deletions(-) delete mode 100644 tests/unit/backend/recipe/algorithm/test_CreateArtificialNormaliztionAlgo.py diff --git a/tests/unit/backend/recipe/algorithm/test_CreateArtificialNormaliztionAlgo.py b/tests/unit/backend/recipe/algorithm/test_CreateArtificialNormaliztionAlgo.py deleted file mode 100644 index 817d8ab1d..000000000 --- a/tests/unit/backend/recipe/algorithm/test_CreateArtificialNormaliztionAlgo.py +++ /dev/null @@ -1,93 +0,0 @@ -import unittest - -import numpy as np -from mantid.simpleapi import ( - ConvertUnits, - mtd, -) -from snapred.backend.dao.ingredients.ArtificialNormalizationIngredients import ArtificialNormalizationIngredients -from snapred.backend.recipe.algorithm.CreateArtificialNormalizationAlgo import CreateArtificialNormalizationAlgo as Algo -from util.diffraction_calibration_synthetic_data import SyntheticData - - -class TestCreateArtificialNormalizationAlgo(unittest.TestCase): - def setUp(self): - self.inputs = SyntheticData() - self.fakeIngredients = self.inputs.ingredients - - self.fakeRawData = "test_data_ws" - self.fakeGroupingWorkspace = "test_grouping_ws" - self.fakeMaskWorkspace = "test_mask_ws" - self.inputs.generateWorkspaces(self.fakeRawData, self.fakeGroupingWorkspace, self.fakeMaskWorkspace) - - ConvertUnits( - InputWorkspace=self.fakeRawData, - OutputWorkspace=self.fakeRawData, - Target="dSpacing", - ) - - def tearDown(self) -> None: - mtd.clear() - assert len(mtd.getObjectNames()) == 0 - return super().tearDown() - - def test_chop_ingredients(self): - self.fakeIngredients = ArtificialNormalizationIngredients( - peakWindowClippingSize=10, - smoothingParameter=0.5, - decreaseParameter=True, - lss=True, - ) - algo = Algo() - algo.initialize() - algo.setProperty("InputWorkspace", self.fakeRawData) - algo.setProperty("Ingredients", self.fakeIngredients.json()) - algo.setProperty("OutputWorkspace", "test_output_ws") - originalData = [] - inputWs = mtd[self.fakeRawData] - for i in range(inputWs.getNumberHistograms()): - originalData.append(inputWs.readY(i).copy()) - algo.execute() - self.assertTrue(mtd.doesExist("test_output_ws")) # noqa: PT009 - output_ws = mtd["test_output_ws"] - for i in range(output_ws.getNumberHistograms()): - dataY = output_ws.readY(i) - self.assertNotEqual(list(dataY), list(originalData[i]), f"Data for histogram {i} was not modified") # noqa: PT009 - self.assertTrue(np.any(dataY)) # noqa: PT009 - self.assertEqual(algo.peakWindowClippingSize, 10) # noqa: PT009 - self.assertEqual(algo.smoothingParameter, 0.5) # noqa: PT009 - self.assertTrue(algo.decreaseParameter) # noqa: PT009 - self.assertTrue(algo.LSS) # noqa: PT009 - - def test_execute(self): - self.fakeIngredients = ArtificialNormalizationIngredients( - peakWindowClippingSize=10, - smoothingParameter=0.5, - decreaseParameter=True, - lss=True, - ) - algo = Algo() - algo.initialize() - algo.setProperty("InputWorkspace", self.fakeRawData) - algo.setProperty("Ingredients", self.fakeIngredients.json()) - algo.setProperty("OutputWorkspace", "test_output_ws") - assert algo.execute() - - def test_output_data_characteristics(self): - self.fakeIngredients = ArtificialNormalizationIngredients( - peakWindowClippingSize=10, - smoothingParameter=0.5, - decreaseParameter=True, - lss=True, - ) - algo = Algo() - algo.initialize() - algo.setProperty("InputWorkspace", self.fakeRawData) - algo.setProperty("Ingredients", self.fakeIngredients.json()) - algo.setProperty("OutputWorkspace", "test_output_ws") - algo.execute() - output_ws = mtd["test_output_ws"] - for i in range(output_ws.getNumberHistograms()): - dataY = output_ws.readY(i) - self.assertFalse(np.isnan(dataY).any(), f"Histogram {i} contains NaN values") # noqa: PT009 - self.assertFalse(np.isinf(dataY).any(), f"Histogram {i} contains infinite values") # noqa: PT009 From b10fa8f6e1aa8e324da41ceda70d20ebd6284183 Mon Sep 17 00:00:00 2001 From: Darsh Date: Wed, 9 Oct 2024 13:40:07 -0400 Subject: [PATCH 05/23] Many updates along with UI stuff. --- .../CreateArtificialNormalizationRequest.py | 13 ++ .../backend/dao/request/ReductionRequest.py | 1 + .../recipe/ArtificialNormalizationRecipe.py | 4 +- .../backend/service/ReductionService.py | 195 ++++++++++++------ src/snapred/ui/model/WorkflowNodeModel.py | 8 + src/snapred/ui/view/BackendRequestView.py | 4 + .../reduction/ArtificialNormalizationView.py | 172 +++++++++++++++ src/snapred/ui/widget/TrueFalseDropDown.py | 33 +++ src/snapred/ui/workflow/ReductionWorkflow.py | 113 +++++++++- src/snapred/ui/workflow/WorkflowBuilder.py | 58 ++++-- 10 files changed, 512 insertions(+), 89 deletions(-) create mode 100644 src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py create mode 100644 src/snapred/ui/view/reduction/ArtificialNormalizationView.py create mode 100644 src/snapred/ui/widget/TrueFalseDropDown.py diff --git a/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py b/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py new file mode 100644 index 000000000..5bbfad484 --- /dev/null +++ b/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py @@ -0,0 +1,13 @@ +from pydantic import BaseModel + +from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName + + +class CreateArtificialNormalizationRequest(BaseModel): + runNumber: str + useLiteMode: bool + peakWindowClippingSize: int + smoothingParameter: float + decreaseParameter: bool = True + lss: bool = True + diffractionWorkspace: WorkspaceName diff --git a/src/snapred/backend/dao/request/ReductionRequest.py b/src/snapred/backend/dao/request/ReductionRequest.py index dd2c1099d..cb82e272b 100644 --- a/src/snapred/backend/dao/request/ReductionRequest.py +++ b/src/snapred/backend/dao/request/ReductionRequest.py @@ -23,6 +23,7 @@ class ReductionRequest(BaseModel): versions: Versions = Versions(None, None) pixelMasks: List[WorkspaceName] = [] + artificialNormalization: Optional[WorkspaceName] = None # TODO: Move to SNAPRequest continueFlags: Optional[ContinueWarning.Type] = ContinueWarning.Type.UNSET diff --git a/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py b/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py index 381cbea73..cdb584159 100644 --- a/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py +++ b/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py @@ -26,8 +26,8 @@ def chopIngredients(self, ingredients: Ingredients): self.lss = self.ingredients.lss def unbagGroceries(self, groceries: Dict[str, Any]): - self.inputWS = groceries["inputWorkspace"] - self.outputWS = groceries.get("outputWorkspace", groceries["inputWorkspace"]) + self.inputWS = groceries["diffractionWorkspace"] + self.outputWS = groceries["artificalNormWorkspace"] def queueAlgos(self): """ diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index 875b326db..b37b1a1b5 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -10,6 +10,7 @@ ) from snapred.backend.dao.reduction.ReductionRecord import ReductionRecord from snapred.backend.dao.request import ( + CreateArtificialNormalizationRequest, FarmFreshIngredients, ReductionExportRequest, ReductionRequest, @@ -17,6 +18,7 @@ from snapred.backend.dao.request.ReductionRequest import Versions from snapred.backend.dao.response.ReductionResponse import ReductionResponse from snapred.backend.dao.SNAPRequest import SNAPRequest +from snapred.backend.dao.SNAPResponse import ResponseCode, SNAPResponse from snapred.backend.data.DataExportService import DataExportService from snapred.backend.data.DataFactoryService import DataFactoryService from snapred.backend.data.GroceryService import GroceryService @@ -77,6 +79,7 @@ def __init__(self): self.registerPath("checkWritePermissions", self.checkWritePermissions) self.registerPath("getSavePath", self.getSavePath) self.registerPath("getStateIds", self.getStateIds) + self.registerPath("artificialNormalization", self.artificialNormalization) return @staticmethod @@ -90,42 +93,67 @@ def validateReduction(self, request: ReductionRequest): :param request: a reduction request :type request: ReductionRequest """ - continueFlags = ContinueWarning.Type.UNSET - # check if a normalization is present - if not self.dataFactoryService.normalizationExists(request.runNumber, request.useLiteMode): - continueFlags |= ContinueWarning.Type.MISSING_NORMALIZATION - # check if a diffraction calibration is present - if not self.dataFactoryService.calibrationExists(request.runNumber, request.useLiteMode): - continueFlags |= ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION - - # remove any continue flags that are present in the request by xor-ing with the flags - if request.continueFlags: - continueFlags = continueFlags ^ (request.continueFlags & continueFlags) - - if continueFlags: - raise ContinueWarning( - "Reduction is missing calibration data, continue in uncalibrated mode?", continueFlags - ) + if request.artificialNormalization is not None: + continueFlags = ContinueWarning.Type.UNSET + + # check that the user has write permissions to the save directory + if not self.checkWritePermissions(request.runNumber): + continueFlags |= ContinueWarning.Type.NO_WRITE_PERMISSIONS + + # remove any continue flags that are present in the request by xor-ing with the flags + if request.continueFlags: + continueFlags = continueFlags ^ (request.continueFlags & continueFlags) + + if continueFlags: + raise ContinueWarning( + f"

Remeber, you don't have permissions to write to " + f"
{self.getSavePath(request.runNumber)},
" + + "but you can still save using the workbench tools.

" + + "

Would you like to continue anyway?

", + continueFlags, + ) + else: + continueFlags = ContinueWarning.Type.UNSET + warningMessages = [] + + # check if a normalization is present + if not self.dataFactoryService.normalizationExists(request.runNumber, request.useLiteMode): + continueFlags |= ContinueWarning.Type.MISSING_NORMALIZATION + warningMessages.append("Normalization is missing, continuing with artificial normalization step.") + # check if a diffraction calibration is present + if not self.dataFactoryService.calibrationExists(request.runNumber, request.useLiteMode): + continueFlags |= ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION + warningMessages.append("Diffraction calibration is missing, continuing with uncalibrated mode.") + + # remove any continue flags that are present in the request by xor-ing with the flags + if request.continueFlags: + continueFlags = continueFlags ^ (request.continueFlags & continueFlags) + + if continueFlags: + detailedMessage = "\n".join(warningMessages) + raise ContinueWarning( + f"The reduction cannot proceed due to missing data:\n{detailedMessage}\n", continueFlags + ) - # ... ensure separate continue warnings ... - continueFlags = ContinueWarning.Type.UNSET + # ... ensure separate continue warnings ... + continueFlags = ContinueWarning.Type.UNSET - # check that the user has write permissions to the save directory - if not self.checkWritePermissions(request.runNumber): - continueFlags |= ContinueWarning.Type.NO_WRITE_PERMISSIONS + # check that the user has write permissions to the save directory + if not self.checkWritePermissions(request.runNumber): + continueFlags |= ContinueWarning.Type.NO_WRITE_PERMISSIONS - # remove any continue flags that are present in the request by xor-ing with the flags - if request.continueFlags: - continueFlags = continueFlags ^ (request.continueFlags & continueFlags) + # remove any continue flags that are present in the request by xor-ing with the flags + if request.continueFlags: + continueFlags = continueFlags ^ (request.continueFlags & continueFlags) - if continueFlags: - raise ContinueWarning( - f"

It looks like you don't have permissions to write to " - f"
{self.getSavePath(request.runNumber)},
" - + "but you can still save using the workbench tools.

" - + "

Would you like to continue anyway?

", - continueFlags, - ) + if continueFlags: + raise ContinueWarning( + f"

It looks like you don't have permissions to write to " + f"
{self.getSavePath(request.runNumber)},
" + + "but you can still save using the workbench tools.

" + + "

Would you like to continue anyway?

", + continueFlags, + ) @FromString def reduction(self, request: ReductionRequest): @@ -347,47 +375,63 @@ def fetchReductionGroceries(self, request: ReductionRequest) -> Dict[str, Any]: # As an interim solution: set the request "versions" field to the latest calibration and normalization versions. # TODO: set these when the request is initially generated. - calVersion = None - normVersion = None - calVersion = self.dataFactoryService.getThisOrLatestCalibrationVersion(request.runNumber, request.useLiteMode) - self.groceryClerk.name("diffcalWorkspace").diffcal_table(request.runNumber, calVersion).useLiteMode( - request.useLiteMode - ).add() - - if ContinueWarning.Type.MISSING_NORMALIZATION not in request.continueFlags: - normVersion = self.dataFactoryService.getThisOrLatestNormalizationVersion( + if request.artificialNormalization is not None: + calVersion = None + normVersion = 0 + calVersion = self.dataFactoryService.getThisOrLatestCalibrationVersion( request.runNumber, request.useLiteMode ) - self.groceryClerk.name("normalizationWorkspace").normalization(request.runNumber, normVersion).useLiteMode( + self.groceryClerk.name("diffcalWorkspace").diffcal_table(request.runNumber, calVersion).useLiteMode( request.useLiteMode ).add() - elif calVersion: - ingredients = ArtificialNormalizationIngredients( - peakWindowClippingSize=5, # Replace with user input or default - smoothingParameter=0.5, # - decreaseParameter=True, # - lss=True, # + return self.groceryService.fetchGroceryDict( + groceryDict=self.groceryClerk.buildDict(), + normalizationWorkspace=request.artificialNormalization, + **({"maskWorkspace": combinedMask} if combinedMask else {}), ) - groceries = { - "inputWorkspace": self.groceryClerk.name("normalizationWorkspace") - .normalization(request.runNumber, normVersion) - .useLiteMode(request.useLiteMode) - .add() - } - artificialNormRecipe = ArtificialNormalizationRecipe() - normWs = artificialNormRecipe.cook(ingredients, groceries) - - self.groceryClerk.name("normalizationWorkspace").set(normWs) - - request.versions = Versions( - calVersion, - normVersion, - ) - return self.groceryService.fetchGroceryDict( - groceryDict=self.groceryClerk.buildDict(), - **({"maskWorkspace": combinedMask} if combinedMask else {}), - ) + else: + calVersion = None + normVersion = None + calVersion = self.dataFactoryService.getThisOrLatestCalibrationVersion( + request.runNumber, request.useLiteMode + ) + self.groceryClerk.name("diffcalWorkspace").diffcal_table(request.runNumber, calVersion).useLiteMode( + request.useLiteMode + ).add() + + if ContinueWarning.Type.MISSING_NORMALIZATION not in request.continueFlags: + normVersion = self.dataFactoryService.getThisOrLatestNormalizationVersion( + request.runNumber, request.useLiteMode + ) + self.groceryClerk.name("normalizationWorkspace").normalization( + request.runNumber, normVersion + ).useLiteMode(request.useLiteMode).add() + elif calVersion and normVersion is None: + groceryList = ( + self.groceryClerk.name("diffractionWorkspace") + .diffcal_output(request.runNumber, calVersion) + .useLiteMode(request.useLiteMode) + .buildDict() + ) + + groceries = self.groceryService.fetchGroceryDict(groceryList) + + return SNAPResponse( + code=ResponseCode.CONTINUE_WARNING, + message="missing normalization", + data=groceries, + ) + + request.versions = Versions( + calVersion, + normVersion, + ) + + return self.groceryService.fetchGroceryDict( + groceryDict=self.groceryClerk.buildDict(), + **({"maskWorkspace": combinedMask} if combinedMask else {}), + ) def saveReduction(self, request: ReductionExportRequest): self.dataExportService.exportReductionRecord(request.record) @@ -447,3 +491,20 @@ def _groupByVanadiumVersion(self, requests: List[SNAPRequest]): def getCompatibleMasks(self, request: ReductionRequest) -> List[WorkspaceName]: runNumber, useLiteMode = request.runNumber, request.useLiteMode return self.dataFactoryService.getCompatibleReductionMasks(runNumber, useLiteMode) + + def artificialNormalization(self, request: CreateArtificialNormalizationRequest): + ingredients = ArtificialNormalizationIngredients( + peakWindowClippingSize=request.peakWindowClippingSize, + smoothingParameter=request.smoothingParameter, + decreaseParameter=request.decreaseParameter, + lss=request.lss, + ) + self.groceryClerk.name("artificalNormWorkspace").ingredients(ingredients).useLiteMode(request.useLiteMode).add() + groceryList = self.groceryClerk.buildDict() + groceries = self.groceryService.fetchGroceryDict( + groceryList, + inputWorkspace=request.diffractionWorkspace, + ) + + data = ArtificialNormalizationRecipe().cook(ingredients, groceries) + return data diff --git a/src/snapred/ui/model/WorkflowNodeModel.py b/src/snapred/ui/model/WorkflowNodeModel.py index e483974bf..c70751394 100644 --- a/src/snapred/ui/model/WorkflowNodeModel.py +++ b/src/snapred/ui/model/WorkflowNodeModel.py @@ -18,6 +18,14 @@ class WorkflowNodeModel(object): iterate: bool = False continueAnywayHandler: Callable[[ContinueWarning.Model], None] = None + def hide(self): + """Hide the node's view.""" + self.view.setVisible(False) + + def show(self): + """Show the node's view.""" + self.view.setVisible(True) + def __iter__(self): return _WorkflowModelIterator(self) diff --git a/src/snapred/ui/view/BackendRequestView.py b/src/snapred/ui/view/BackendRequestView.py index 6def779e2..7e1066740 100644 --- a/src/snapred/ui/view/BackendRequestView.py +++ b/src/snapred/ui/view/BackendRequestView.py @@ -6,6 +6,7 @@ from snapred.ui.widget.LabeledField import LabeledField from snapred.ui.widget.MultiSelectDropDown import MultiSelectDropDown from snapred.ui.widget.SampleDropDown import SampleDropDown +from snapred.ui.widget.TrueFalseDropDown import TrueFalseDropDown class BackendRequestView(QWidget): @@ -33,6 +34,9 @@ def _labeledCheckBox(self, label): def _sampleDropDown(self, label, items=[]): return SampleDropDown(label, items, self) + def _trueFalseDropDown(self, label, items=[]): + return TrueFalseDropDown(label, items, self) + def _multiSelectDropDown(self, label, items=[]): return MultiSelectDropDown(label, items, self) diff --git a/src/snapred/ui/view/reduction/ArtificialNormalizationView.py b/src/snapred/ui/view/reduction/ArtificialNormalizationView.py new file mode 100644 index 000000000..c5be2bc94 --- /dev/null +++ b/src/snapred/ui/view/reduction/ArtificialNormalizationView.py @@ -0,0 +1,172 @@ +import matplotlib.pyplot as plt +from mantid.plots.datafunctions import get_spectrum +from mantid.simpleapi import mtd +from qtpy.QtCore import Signal, Slot +from qtpy.QtWidgets import ( + QHBoxLayout, + QLineEdit, + QMessageBox, + QPushButton, +) +from snapred.meta.Config import Config +from snapred.meta.decorators.Resettable import Resettable +from snapred.ui.view.BackendRequestView import BackendRequestView +from snapred.ui.widget.SmoothingSlider import SmoothingSlider +from workbench.plotting.figuremanager import MantidFigureCanvas +from workbench.plotting.toolbar import WorkbenchNavigationToolbar + + +@Resettable +class ArtificialNormalizationView(BackendRequestView): + signalRunNumberUpdate = Signal(str) + signalValueChanged = Signal(float, bool, bool, int) + signalUpdateRecalculationButton = Signal(bool) + signalUpdateFields = Signal(float, bool, bool) + + def __init__(self, parent=None): + super().__init__(parent=parent) + + # create the run number fields + self.fieldRunNumber = self._labeledField("Run Number", " ") + + # create the graph elements + self.figure = plt.figure(constrained_layout=True) + self.canvas = MantidFigureCanvas(self.figure) + self.navigationBar = WorkbenchNavigationToolbar(self.canvas, self) + + # create the other specification elements + self.lssDropdown = self._trueFalseDropDown("LSS") + self.decreaseParameterDropdown = self._trueFalseDropDown("Decrease Parameter") + + # disable run number + for x in [self.fieldRunNumber]: + x.setEnable(False) + + # create the adjustment controls + self.smoothingSlider = self._labeledField("Smoothing", SmoothingSlider()) + self.peakWindowClippingSize = self._labeledField( + "Peak Window Clipping Size", + QLineEdit(str(Config["constants.ArtificialNormalization.peakWindowClippingSize"])), + ) + + peakControlLayout = QHBoxLayout() + peakControlLayout.addWidget(self.smoothingSlider, 2) + peakControlLayout.addWidget(self.peakWindowClippingSize) + + # a big ol recalculate button + self.recalculationButton = QPushButton("Recalculate") + self.recalculationButton.clicked.connect(self.emitValueChange) + + # add all elements to the grid layout + self.layout.addWidget(self.fieldRunNumber, 0, 0) + self.layout.addWidget(self.navigationBar, 1, 0) + self.layout.addWidget(self.canvas, 2, 0, 1, -1) + self.layout.addLayout(peakControlLayout, 3, 0, 1, 2) + self.layout.addWidget(self.lssDropdown, 4, 0) + self.layout.addWidget(self.decreaseParameterDropdown, 4, 1) + self.layout.addWidget(self.recalculationButton, 5, 0, 1, 2) + + self.layout.setRowStretch(2, 10) + + # store the initial layout without graphs + self.initialLayoutHeight = self.size().height() + + self.signalUpdateRecalculationButton.connect(self.setEnableRecalculateButton) + self.signalUpdateFields.connect(self._updateFields) + + @Slot(str) + def _updateRunNumber(self, runNumber): + self.fieldRunNumber.setText(runNumber) + + def updateRunNumber(self, runNumber): + self.signalRunNumberUpdate.emit(runNumber) + + @Slot(int, int, float) + def _updateFields(self, smoothingParameter, lss, decreaseParameter): + self.smoothingSlider.field.setValue(smoothingParameter) + self.lssDropdown.setCurrentIndex(lss) + self.decreaseParameterDropdown.setCurrentIndex(decreaseParameter) + + def updateFields(self, smoothingParameter, lss, decreaseParameter): + self.signalUpdateFields.emit(smoothingParameter, lss, decreaseParameter) + + @Slot() + def emitValueChange(self): + # verify the fields before recalculation + try: + smoothingValue = self.smoothingSlider.field.value() + lss = self.lssDropdown.currentIndex() == "True" + decreaseParameter = self.decreaseParameterDropdown.currentIndex == "True" + peakWindowClippingSize = int(self.peakWindowClippingSize.field.text()) + except ValueError as e: + QMessageBox.warning( + self, + "Invalid Peak Parameters", + f"Smoothing or peak window clipping size is invalid: {str(e)}", + QMessageBox.Ok, + ) + return + self.signalValueChanged.emit(smoothingValue, lss, decreaseParameter, peakWindowClippingSize) + + def updateWorkspaces(self, diffractionWorkspace, artificialNormWorkspace): + self.diffractionWorkspace = diffractionWorkspace + self.artificialNormWorkspace = artificialNormWorkspace + self._updateGraphs() + + def _updateGraphs(self): + # get the updated workspaces and optimal graph grid + diffractionWorkspace = mtd[self.diffractionWorkspace] + artificialNormWorkspace = mtd[self.artificialNormWorkspace] + numGraphs = diffractionWorkspace.getNumberHistograms() + nrows, ncols = self._optimizeRowsAndCols(numGraphs) + + # now re-draw the figure + self.figure.clear() + for i in range(numGraphs): + ax = self.figure.add_subplot(nrows, ncols, i + 1, projection="mantid") + ax.plot(diffractionWorkspace, wkspIndex=i, label="Diffcal Data", normalize_by_bin_width=True) + ax.plot( + artificialNormWorkspace, + wkspIndex=i, + label="Artificial Normalization Data", + normalize_by_bin_width=True, + linestyle="--", + ) + ax.legend() + ax.tick_params(direction="in") + ax.set_title(f"Group ID: {i + 1}") + # fill in the discovered peaks for easier viewing + x, y, _, _ = get_spectrum(diffractionWorkspace, i, normalize_by_bin_width=True) + # for each detected peak in this group, shade in the peak region + + # resize window and redraw + self.setMinimumHeight(self.initialLayoutHeight + int(self.figure.get_size_inches()[1] * self.figure.dpi)) + self.canvas.draw() + + def _optimizeRowsAndCols(self, numGraphs): + # Get best size for layout + sqrtSize = int(numGraphs**0.5) + if sqrtSize == numGraphs**0.5: + rowSize = sqrtSize + colSize = sqrtSize + elif numGraphs <= ((sqrtSize + 1) * sqrtSize): + rowSize = sqrtSize + colSize = sqrtSize + 1 + else: + rowSize = sqrtSize + 1 + colSize = sqrtSize + 1 + return rowSize, colSize + + @Slot(bool) + def setEnableRecalculateButton(self, enable): + self.recalculationButton.setEnabled(enable) + + def disableRecalculateButton(self): + self.signalUpdateRecalculationButton.emit(False) + + def enableRecalculateButton(self): + self.signalUpdateRecalculationButton.emit(True) + + def verify(self): + # TODO what needs to be verified? + return True diff --git a/src/snapred/ui/widget/TrueFalseDropDown.py b/src/snapred/ui/widget/TrueFalseDropDown.py new file mode 100644 index 000000000..9a0fe6594 --- /dev/null +++ b/src/snapred/ui/widget/TrueFalseDropDown.py @@ -0,0 +1,33 @@ +from qtpy.QtWidgets import QComboBox, QVBoxLayout, QWidget + + +class TrueFalseDropDown(QWidget): + def __init__(self, label, parent=None): + super(TrueFalseDropDown, self).__init__(parent) + self.setStyleSheet("background-color: #F5E9E2;") + self._label = label + + self.dropDown = QComboBox() + self._initItems() + + layout = QVBoxLayout() + layout.addWidget(self.dropDown) + self.setLayout(layout) + + def _initItems(self): + self.dropDown.clear() + self.dropDown.addItem(self._label) + self.dropDown.addItems(["True", "False"]) + self.dropDown.model().item(0).setEnabled(False) + self.dropDown.setCurrentIndex(1) + + def currentIndex(self): + # Subtract 1 because the label is considered an index + return self.dropDown.currentIndex() - 1 + + def setCurrentIndex(self, index): + # Add 1 to skip the label + self.dropDown.setCurrentIndex(index + 1) + + def currentText(self): + return self.dropDown.currentText() diff --git a/src/snapred/ui/workflow/ReductionWorkflow.py b/src/snapred/ui/workflow/ReductionWorkflow.py index 2d28bbc2b..07fd91fb7 100644 --- a/src/snapred/ui/workflow/ReductionWorkflow.py +++ b/src/snapred/ui/workflow/ReductionWorkflow.py @@ -3,6 +3,7 @@ from qtpy.QtCore import Slot from snapred.backend.dao.request import ( + CreateArtificialNormalizationRequest, ReductionExportRequest, ReductionRequest, ) @@ -11,6 +12,7 @@ from snapred.backend.log.logger import snapredLogger from snapred.meta.decorators.ExceptionToErrLog import ExceptionToErrLog from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName +from snapred.ui.view.reduction.ArtificialNormalizationView import ArtificialNormalizationView from snapred.ui.view.reduction.ReductionRequestView import ReductionRequestView from snapred.ui.view.reduction.ReductionSaveView import ReductionSaveView from snapred.ui.workflow.WorkflowBuilder import WorkflowBuilder @@ -22,7 +24,7 @@ class ReductionWorkflow(WorkflowImplementer): def __init__(self, parent=None): super().__init__(parent) - + self.artificialNormComplete = False self._reductionRequestView = ReductionRequestView( parent=parent, populatePixelMaskDropdown=self._populatePixelMaskDropdown, @@ -33,6 +35,10 @@ def __init__(self, parent=None): self._reductionRequestView.enterRunNumberButton.clicked.connect(lambda: self._populatePixelMaskDropdown()) self._reductionRequestView.pixelMaskDropdown.dropDown.view().pressed.connect(self._onPixelMaskSelection) + self._artificialNormalizationView = ArtificialNormalizationView( + parent=parent, + ) + self._reductionSaveView = ReductionSaveView( parent=parent, ) @@ -50,11 +56,18 @@ def __init__(self, parent=None): "Reduction", continueAnywayHandler=self._continueAnywayHandler, ) + .addNode( + self._artificialNormalization, + self._artificialNormalizationView, + "Artificial Normalization", + visible=False, + ) .addNode(self._nothing, self._reductionSaveView, "Save") .build() ) self._reductionRequestView.retainUnfocusedDataCheckbox.checkedChanged.connect(self._enableConvertToUnits) + self._artificialNormalizationView.signalValueChanged.connect(self.onArtificialNormalizationValueChange) def _enableConvertToUnits(self): state = self._reductionRequestView.retainUnfocusedDataCheckbox.isChecked() @@ -171,6 +184,47 @@ def _triggerReduction(self, workflowPresenter): if unfocusedData is not None: self.outputs.append(unfocusedData) + elif ( + response.message == "missing normalization" + and response.code == ResponseCode.CONTINUE_WARNING + and response.data is not None + ): + self._artificialNormalizationView.updateRunNumber(runNumber) + workflowPresenter.presenter.widget.showTab("Artificial Normalization") + self._artificialNormalization(workflowPresenter, response.data, runNumber) + + if self.artificialNormComplete: + artificialNormWorkspace = self._artificialNormalizationView.artificialNormWorkspace + + # Modify the request to use the artificial normalization workspace + request_ = ReductionRequest( + runNumber=str(runNumber), + useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(), + timestamp=timestamp, + continueFlags=self.continueAnywayFlags, + pixelMasks=pixelMasks, + keepUnfocused=self._reductionRequestView.retainUnfocusedDataCheckbox.isChecked(), + convertUnitsTo=self._reductionRequestView.convertUnitsDropdown.currentText(), + normalizationWorkspace=artificialNormWorkspace, + ) + + # Re-trigger reduction + response = self.request(path="reduction/", payload=request_) + + if response.code == ResponseCode.OK: + # Continue to the save step as before + record, unfocusedData = response.data.record, response.data.unfocusedData + savePath = self.request(path="reduction/getSavePath", payload=record.runNumber).data + self._reductionSaveView.updateContinueAnyway(self.continueAnywayFlags) + self._reductionSaveView.updateSavePath(savePath) + + if ContinueWarning.Type.NO_WRITE_PERMISSIONS not in self.continueAnywayFlags: + self.request(path="reduction/save", payload=ReductionExportRequest(record=record)) + + # Handle output workspaces + self.outputs.extend(record.workspaceNames) + if unfocusedData is not None: + self.outputs.append(unfocusedData) # Note that the run number is deliberately not deleted from the run numbers list. # Almost certainly it should be moved to a "completed run numbers" list. @@ -181,6 +235,63 @@ def _triggerReduction(self, workflowPresenter): return self.responses[-1] + def _artificialNormalization(self, workflowPresenter, responseData, runNumber): + view = workflowPresenter.widget.tabView # noqa: F841 + + try: + # Handle artificial normalization request here + request_ = CreateArtificialNormalizationRequest( + runNumber=runNumber, + useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(), + peakWindowClippingSize=int(self._artificialNormalizationView.peakWindowClippingSize.field.text()), + smoothingParameter=self._artificialNormalizationView.smoothingSlider.field.value(), + decreaseParameter=self._artificialNormalizationView.decreaseParameterDropdown.currentIndex() == 1, + lss=self._artificialNormalizationView.lssDropdown.currentIndex() == 1, + diffcalWorkspace=responseData.get("diffractionWorkspace"), + ) + response = self.request(path="reduction/artificialNormalization", payload=request_.json()) + + # Update workspaces in the artificial normalization view + diffractionWorkspace = response.data.get("diffractionWorkspace") + artificialNormWorkspace = response.data.get("artificialNormWorkspace") + + if diffractionWorkspace and artificialNormWorkspace: + self._artificialNormalizationView.updateWorkspaces(diffractionWorkspace, artificialNormWorkspace) + self.artificialNormComplete = True + else: + print(f"Error: Workspaces not found in the response: {response.data}") + except Exception as e: # noqa: BLE001 + print(f"Error during artificial normalization request: {e}") + + @Slot(float, bool, bool, int) + def onArtificialNormalizationValueChange(self, smoothingValue, lss, decreaseParameter, peakWindowClippingSize): + self._artificialNormalizationView.disableRecalculateButton() + # Recalculate normalization based on updated values + runNumber = self._artificialNormalizationView.fieldRunNumber.text() + diffractionWorkspace = self._artificialNormalizationView.diffractionWorkspace + try: + request_ = CreateArtificialNormalizationRequest( + runNumber=runNumber, + useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(), + peakWindowClippingSize=peakWindowClippingSize, + smoothingParameter=smoothingValue, + decreaseParameter=decreaseParameter, + lss=lss, + diffractionWorkspace=WorkspaceName(diffractionWorkspace), + ) + + response = self.request(path="reduction/artificialNormalization", payload=request_.json()) + diffractionWorkspace = response.data["diffractionWorkspace"] + artificialNormWorkspace = response.data["artificialNormWorkspace"] + + # Update the view with new workspaces + self._artificialNormalizationView.updateWorkspaces(diffractionWorkspace, artificialNormWorkspace) + + except Exception as e: # noqa: BLE001 + print(f"Error during recalculation: {e}") + + self._artificialNormalizationView.enableRecalculateButton() + @property def widget(self): return self.workflow.presenter.widget diff --git a/src/snapred/ui/workflow/WorkflowBuilder.py b/src/snapred/ui/workflow/WorkflowBuilder.py index 4e2dde6ea..dc7c038bb 100644 --- a/src/snapred/ui/workflow/WorkflowBuilder.py +++ b/src/snapred/ui/workflow/WorkflowBuilder.py @@ -10,35 +10,55 @@ def __init__(self, *, startLambda=None, iterateLambda=None, resetLambda=None, ca self._resetLambda = resetLambda self._cancelLambda = cancelLambda self._workflow = None + self._invisibleNodes = [] def addNode( - self, continueAction, subview, name="Unnamed", required=True, iterate=False, continueAnywayHandler=None + self, + continueAction, + subview, + name="Unnamed", + required=True, + iterate=False, + continueAnywayHandler=None, + visible=True, ): + """ + Adds a node to the workflow. If visible=False, the node is initially hidden. + """ + node = WorkflowNodeModel( + continueAction=continueAction, + view=subview, + nextModel=None, + name=name, + required=required, + iterate=iterate, + continueAnywayHandler=continueAnywayHandler, + ) + + # If the node is invisible, add it to the invisible nodes list + if not visible: + self._invisibleNodes.append(node) + if self._workflow is None: - self._workflow = WorkflowNodeModel( - continueAction=continueAction, - view=subview, - nextModel=None, - name=name, - required=required, - iterate=iterate, - continueAnywayHandler=continueAnywayHandler, - ) + self._workflow = node else: currentWorkflow = self._workflow while currentWorkflow.nextModel is not None: currentWorkflow = currentWorkflow.nextModel - currentWorkflow.nextModel = WorkflowNodeModel( - continueAction=continueAction, - view=subview, - nextModel=None, - name=name, - required=required, - iterate=iterate, - continueAnywayHandler=continueAnywayHandler, - ) + currentWorkflow.nextModel = node + return self + def makeNodeVisible(self, nodeName): + """ + Makes an invisible node visible by name. + """ + for node in self._invisibleNodes: + if node.name == nodeName: + node.view.setVisible(True) # Set the node's view to be visible + self._invisibleNodes.remove(node) # Remove from the invisible list + break + def build(self): return Workflow( self._workflow, From 415a9a80c5a3540be5758ca0c9fe2625bfd98957 Mon Sep 17 00:00:00 2001 From: Darsh Date: Wed, 9 Oct 2024 14:53:26 -0400 Subject: [PATCH 06/23] Fixes. --- .../dao/request/CreateArtificialNormalizationRequest.py | 4 ++++ src/snapred/backend/service/SousChef.py | 2 +- src/snapred/ui/view/BackendRequestView.py | 4 ++-- .../ui/view/reduction/ArtificialNormalizationView.py | 6 +++--- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py b/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py index 5bbfad484..7c94d89de 100644 --- a/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py +++ b/src/snapred/backend/dao/request/CreateArtificialNormalizationRequest.py @@ -11,3 +11,7 @@ class CreateArtificialNormalizationRequest(BaseModel): decreaseParameter: bool = True lss: bool = True diffractionWorkspace: WorkspaceName + + class Config: + arbitrary_types_allowed = True # Allow arbitrary types like WorkspaceName + extra = "forbid" # Forbid extra fields diff --git a/src/snapred/backend/service/SousChef.py b/src/snapred/backend/service/SousChef.py index a290ec637..ab20d49b7 100644 --- a/src/snapred/backend/service/SousChef.py +++ b/src/snapred/backend/service/SousChef.py @@ -239,7 +239,7 @@ def _pullNormalizationRecordFFI( calibrantSamplePath = None if normalizationRecord is not None: smoothingParameter = normalizationRecord.smoothingParameter - calibrantSamplePath = normalizationRecord.calibrantSamplePath + calibrantSamplePath = normalizationRecord.normalizationCalibrantSamplePath # TODO: Should smoothing parameter be an ingredient? return ingredients, smoothingParameter, calibrantSamplePath diff --git a/src/snapred/ui/view/BackendRequestView.py b/src/snapred/ui/view/BackendRequestView.py index 7e1066740..1644b17f0 100644 --- a/src/snapred/ui/view/BackendRequestView.py +++ b/src/snapred/ui/view/BackendRequestView.py @@ -34,8 +34,8 @@ def _labeledCheckBox(self, label): def _sampleDropDown(self, label, items=[]): return SampleDropDown(label, items, self) - def _trueFalseDropDown(self, label, items=[]): - return TrueFalseDropDown(label, items, self) + def _trueFalseDropDown(self, label): + return TrueFalseDropDown(label, self) def _multiSelectDropDown(self, label, items=[]): return MultiSelectDropDown(label, items, self) diff --git a/src/snapred/ui/view/reduction/ArtificialNormalizationView.py b/src/snapred/ui/view/reduction/ArtificialNormalizationView.py index c5be2bc94..7f5ad4ff3 100644 --- a/src/snapred/ui/view/reduction/ArtificialNormalizationView.py +++ b/src/snapred/ui/view/reduction/ArtificialNormalizationView.py @@ -27,7 +27,7 @@ def __init__(self, parent=None): super().__init__(parent=parent) # create the run number fields - self.fieldRunNumber = self._labeledField("Run Number", " ") + self.fieldRunNumber = self._labeledField("Run Number", QLineEdit()) # create the graph elements self.figure = plt.figure(constrained_layout=True) @@ -40,7 +40,7 @@ def __init__(self, parent=None): # disable run number for x in [self.fieldRunNumber]: - x.setEnable(False) + x.setEnabled(False) # create the adjustment controls self.smoothingSlider = self._labeledField("Smoothing", SmoothingSlider()) @@ -81,7 +81,7 @@ def _updateRunNumber(self, runNumber): def updateRunNumber(self, runNumber): self.signalRunNumberUpdate.emit(runNumber) - @Slot(int, int, float) + @Slot(float, bool, bool) def _updateFields(self, smoothingParameter, lss, decreaseParameter): self.smoothingSlider.field.setValue(smoothingParameter) self.lssDropdown.setCurrentIndex(lss) From 43fd73a37401bd5de36862090a69365d21c93234 Mon Sep 17 00:00:00 2001 From: Darsh Date: Wed, 9 Oct 2024 15:19:20 -0400 Subject: [PATCH 07/23] Fix --- src/snapred/backend/service/ReductionService.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index b37b1a1b5..831b05165 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -408,14 +408,11 @@ def fetchReductionGroceries(self, request: ReductionRequest) -> Dict[str, Any]: request.runNumber, normVersion ).useLiteMode(request.useLiteMode).add() elif calVersion and normVersion is None: - groceryList = ( - self.groceryClerk.name("diffractionWorkspace") - .diffcal_output(request.runNumber, calVersion) - .useLiteMode(request.useLiteMode) - .buildDict() - ) + self.groceryClerk.name("diffractionWorkspace").diffcal_output( + request.runNumber, calVersion + ).useLiteMode(request.useLiteMode).add() - groceries = self.groceryService.fetchGroceryDict(groceryList) + groceries = self.groceryService.fetchGroceryDict(self.groceryClerk.buildDict()) return SNAPResponse( code=ResponseCode.CONTINUE_WARNING, From 204b7ce0e90bd428c1e2139aac16c64ba1de896a Mon Sep 17 00:00:00 2001 From: Darsh Date: Wed, 9 Oct 2024 15:29:08 -0400 Subject: [PATCH 08/23] More fixes. --- src/snapred/backend/service/ReductionService.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index 831b05165..09d0559a6 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -408,11 +408,16 @@ def fetchReductionGroceries(self, request: ReductionRequest) -> Dict[str, Any]: request.runNumber, normVersion ).useLiteMode(request.useLiteMode).add() elif calVersion and normVersion is None: - self.groceryClerk.name("diffractionWorkspace").diffcal_output( - request.runNumber, calVersion - ).useLiteMode(request.useLiteMode).add() + groceryList = ( + self.groceryClerk.name("diffractionWorkspace") + .diffcal_output(request.runNumber, calVersion) + .useLiteMode(request.useLiteMode) + .setUnit(wng.UNITS.DSP) + .setGroupingScheme("column") + .buildDict() + ) - groceries = self.groceryService.fetchGroceryDict(self.groceryClerk.buildDict()) + groceries = self.groceryService.fetchGroceryDict(groceryList) return SNAPResponse( code=ResponseCode.CONTINUE_WARNING, From 9e14929139e8cb116989b0ab5775ab984f508c80 Mon Sep 17 00:00:00 2001 From: Darsh Date: Fri, 11 Oct 2024 09:16:53 -0400 Subject: [PATCH 09/23] Many more udpates to workflow. --- .../dao/response/ArtificialNormResponse.py | 13 ++ .../recipe/ArtificialNormalizationRecipe.py | 71 ----------- src/snapred/backend/recipe/GenericRecipe.py | 5 + src/snapred/backend/recipe/ReductionRecipe.py | 19 --- .../CreateArtificialNormalizationAlgo.py | 6 +- .../backend/service/ReductionService.py | 33 +++--- src/snapred/ui/model/WorkflowNodeModel.py | 8 -- src/snapred/ui/workflow/ReductionWorkflow.py | 111 +++++++++--------- src/snapred/ui/workflow/WorkflowBuilder.py | 48 +++----- .../test_ArtificialNormalizationRecipe.py | 94 --------------- 10 files changed, 112 insertions(+), 296 deletions(-) create mode 100644 src/snapred/backend/dao/response/ArtificialNormResponse.py delete mode 100644 src/snapred/backend/recipe/ArtificialNormalizationRecipe.py delete mode 100644 tests/unit/backend/recipe/test_ArtificialNormalizationRecipe.py diff --git a/src/snapred/backend/dao/response/ArtificialNormResponse.py b/src/snapred/backend/dao/response/ArtificialNormResponse.py new file mode 100644 index 000000000..78c0f6612 --- /dev/null +++ b/src/snapred/backend/dao/response/ArtificialNormResponse.py @@ -0,0 +1,13 @@ +from pydantic import BaseModel, ConfigDict + +from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName + + +class ArtificialNormResponse(BaseModel): + diffractionWorkspace: WorkspaceName + + model_config = ConfigDict( + extra="forbid", + # required in order to use 'WorkspaceName' + arbitrary_types_allowed=True, + ) diff --git a/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py b/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py deleted file mode 100644 index cdb584159..000000000 --- a/src/snapred/backend/recipe/ArtificialNormalizationRecipe.py +++ /dev/null @@ -1,71 +0,0 @@ -import json -from typing import Any, Dict, List, Tuple - -from snapred.backend.dao.ingredients import ArtificialNormalizationIngredients as Ingredients -from snapred.backend.log.logger import snapredLogger -from snapred.backend.recipe.Recipe import Recipe -from snapred.meta.decorators.Singleton import Singleton - -logger = snapredLogger.getLogger(__name__) - -Pallet = Tuple[Ingredients, Dict[str, str]] - - -@Singleton -class ArtificialNormalizationRecipe(Recipe[Ingredients]): - """ - The purpose of this recipe is to apply artificial normalization - using the CreateArtificialNormalizationAlgo algorithm. - """ - - def chopIngredients(self, ingredients: Ingredients): - self.ingredients = ingredients.copy() - self.peakWindowClippingSize = self.ingredients.peakWindowClippingSize - self.smoothingParameter = self.ingredients.smoothingParameter - self.decreaseParameter = self.ingredients.decreaseParameter - self.lss = self.ingredients.lss - - def unbagGroceries(self, groceries: Dict[str, Any]): - self.inputWS = groceries["diffractionWorkspace"] - self.outputWS = groceries["artificalNormWorkspace"] - - def queueAlgos(self): - """ - Queues up the processing algorithms for the recipe. - Requires: unbagged groceries and chopped ingredients. - """ - ingredients = { - "peakWindowClippingSize": self.peakWindowClippingSize, - "smoothingParameter": self.smoothingParameter, - "decreaseParameter": self.decreaseParameter, - "lss": self.lss, - } - ingredientsStr = json.dumps(ingredients) - - self.mantidSnapper.CreateArtificialNormalizationAlgo( - "Creating artificial normalization...", - InputWorkspace=self.inputWS, - OutputWorkspace=self.outputWS, - Ingredients=ingredientsStr, - ) - - def cook(self, ingredients: Ingredients, groceries: Dict[str, str]) -> Dict[str, Any]: - """ - Main interface method for the recipe. - Given the ingredients and groceries, it prepares, executes, and returns the final workspace. - """ - self.prep(ingredients, groceries) - return self.execute() - # return {"outputWorkspace": self.outputWS} - - def cater(self, shipment: List[Pallet]) -> List[Dict[str, Any]]: - """ - A secondary interface method for the recipe. - It is a batched version of cook. - Given a shipment of ingredients and groceries, it prepares, executes and returns the final workspaces. - """ - output = [] - for ingredient, grocery in shipment: - output.append(self.cook(ingredient, grocery)) - self.execute() - return output diff --git a/src/snapred/backend/recipe/GenericRecipe.py b/src/snapred/backend/recipe/GenericRecipe.py index 437ba3ebf..22284c4ce 100644 --- a/src/snapred/backend/recipe/GenericRecipe.py +++ b/src/snapred/backend/recipe/GenericRecipe.py @@ -7,6 +7,7 @@ from snapred.backend.log.logger import snapredLogger from snapred.backend.recipe.algorithm.BufferMissingColumnsAlgo import BufferMissingColumnsAlgo from snapred.backend.recipe.algorithm.CalibrationMetricExtractionAlgorithm import CalibrationMetricExtractionAlgorithm +from snapred.backend.recipe.algorithm.CreateArtificialNormalizationAlgo import CreateArtificialNormalizationAlgo from snapred.backend.recipe.algorithm.DetectorPeakPredictor import DetectorPeakPredictor from snapred.backend.recipe.algorithm.FitMultiplePeaksAlgorithm import FitMultiplePeaksAlgorithm from snapred.backend.recipe.algorithm.FocusSpectraAlgorithm import FocusSpectraAlgorithm @@ -104,3 +105,7 @@ class ConvertTableToMatrixWorkspaceRecipe(GenericRecipe[ConvertTableToMatrixWork class BufferMissingColumnsRecipe(GenericRecipe[BufferMissingColumnsAlgo]): pass + + +class ArtificialNormalizationRecipe(GenericRecipe[CreateArtificialNormalizationAlgo]): + pass diff --git a/src/snapred/backend/recipe/ReductionRecipe.py b/src/snapred/backend/recipe/ReductionRecipe.py index 353882c71..551cda0ba 100644 --- a/src/snapred/backend/recipe/ReductionRecipe.py +++ b/src/snapred/backend/recipe/ReductionRecipe.py @@ -1,10 +1,8 @@ from typing import Any, Dict, List, Tuple, Type -from snapred.backend.dao.ingredients import ArtificialNormalizationIngredients from snapred.backend.dao.ingredients import ReductionIngredients as Ingredients from snapred.backend.log.logger import snapredLogger from snapred.backend.recipe.ApplyNormalizationRecipe import ApplyNormalizationRecipe -from snapred.backend.recipe.ArtificialNormalizationRecipe import ArtificialNormalizationRecipe from snapred.backend.recipe.GenerateFocussedVanadiumRecipe import GenerateFocussedVanadiumRecipe from snapred.backend.recipe.PreprocessReductionRecipe import PreprocessReductionRecipe from snapred.backend.recipe.Recipe import Recipe, WorkspaceName @@ -39,13 +37,6 @@ class ReductionRecipe(Recipe[Ingredients]): self.groupingWorkspaces = groceries["groupingWorkspaces"] """ - def __init__(self): - super().__init__() - self.sampleWs = None - self.normalizationWs = None - self.maskWs = None - self.groupingWorkspaces = [] - def logger(self): return logger @@ -234,16 +225,6 @@ def cook(self, ingredients: Ingredients, groceries: Dict[str, str]) -> Dict[str, Given the ingredients and groceries, it prepares, executes and returns the final workspace. """ self.prep(ingredients, groceries) - if not self.normalizationWs and self.sampleWs: - logger.info("Normalization is missing, applying artificial normalization...") - artificialIngredients = ArtificialNormalizationIngredients( - peakWindowClippingSize=5, # Replace with appropriate value or user input - smoothingParameter=0.5, # Replace with appropriate value or user input - decreaseParameter=True, - lss=True, - ) - artificialNormRecipe = ArtificialNormalizationRecipe() - self.normalizationWs = artificialNormRecipe.cook(artificialIngredients, groceries) return self.execute() def cater(self, shipment: List[Pallet]) -> List[Dict[str, Any]]: diff --git a/src/snapred/backend/recipe/algorithm/CreateArtificialNormalizationAlgo.py b/src/snapred/backend/recipe/algorithm/CreateArtificialNormalizationAlgo.py index 7ca2287be..f4a428314 100644 --- a/src/snapred/backend/recipe/algorithm/CreateArtificialNormalizationAlgo.py +++ b/src/snapred/backend/recipe/algorithm/CreateArtificialNormalizationAlgo.py @@ -36,7 +36,7 @@ def PyInit(self): "OutputWorkspace", "", Direction.Output, - PropertyMode.Mandatory, + PropertyMode.Optional, validator=WorkspaceUnitValidator("dSpacing"), ), doc="Workspace that contains artificial normalization.", @@ -58,7 +58,9 @@ def chopInredients(self, ingredientsStr: str): def unbagGroceries(self): self.inputWorkspaceName = self.getPropertyValue("InputWorkspace") - self.outputWorkspaceName = self.getPropertyValue("OutputWorkspace") + self.outputWorkspaceName = ( + self.getPropertyValue("OutputWorkspace") or self.inputWorkspaceName + "_artificial_norm" + ) def peakClip(self, data, winSize: int, decrese: bool, LLS: bool, smoothing: float): # Clipping peaks from the data with optional smoothing and transformations diff --git a/src/snapred/backend/service/ReductionService.py b/src/snapred/backend/service/ReductionService.py index 09d0559a6..13f7856d1 100644 --- a/src/snapred/backend/service/ReductionService.py +++ b/src/snapred/backend/service/ReductionService.py @@ -16,9 +16,9 @@ ReductionRequest, ) from snapred.backend.dao.request.ReductionRequest import Versions +from snapred.backend.dao.response.ArtificialNormResponse import ArtificialNormResponse from snapred.backend.dao.response.ReductionResponse import ReductionResponse from snapred.backend.dao.SNAPRequest import SNAPRequest -from snapred.backend.dao.SNAPResponse import ResponseCode, SNAPResponse from snapred.backend.data.DataExportService import DataExportService from snapred.backend.data.DataFactoryService import DataFactoryService from snapred.backend.data.GroceryService import GroceryService @@ -26,7 +26,9 @@ from snapred.backend.error.StateValidationException import StateValidationException from snapred.backend.log.logger import snapredLogger from snapred.backend.recipe.algorithm.MantidSnapper import MantidSnapper -from snapred.backend.recipe.ArtificialNormalizationRecipe import ArtificialNormalizationRecipe +from snapred.backend.recipe.GenericRecipe import ( + ArtificialNormalizationRecipe, +) from snapred.backend.recipe.ReductionRecipe import ReductionRecipe from snapred.backend.service.Service import Service from snapred.backend.service.SousChef import SousChef @@ -170,6 +172,9 @@ def reduction(self, request: ReductionRequest): ingredients = self.prepReductionIngredients(request) groceries = self.fetchReductionGroceries(request) + if isinstance(groceries, ArtificialNormResponse): + return groceries + # attach the list of grouping workspaces to the grocery dictionary groceries["groupingWorkspaces"] = groupingResults["groupingWorkspaces"] @@ -412,18 +417,14 @@ def fetchReductionGroceries(self, request: ReductionRequest) -> Dict[str, Any]: self.groceryClerk.name("diffractionWorkspace") .diffcal_output(request.runNumber, calVersion) .useLiteMode(request.useLiteMode) - .setUnit(wng.UNITS.DSP) - .setGroupingScheme("column") + .unit(wng.Units.DSP) + .group("column") .buildDict() ) groceries = self.groceryService.fetchGroceryDict(groceryList) - - return SNAPResponse( - code=ResponseCode.CONTINUE_WARNING, - message="missing normalization", - data=groceries, - ) + diffractionWorkspace = groceries.get("diffractionWorkspace") + return ArtificialNormResponse(diffractionWorkspace=diffractionWorkspace) request.versions = Versions( calVersion, @@ -501,12 +502,8 @@ def artificialNormalization(self, request: CreateArtificialNormalizationRequest) decreaseParameter=request.decreaseParameter, lss=request.lss, ) - self.groceryClerk.name("artificalNormWorkspace").ingredients(ingredients).useLiteMode(request.useLiteMode).add() - groceryList = self.groceryClerk.buildDict() - groceries = self.groceryService.fetchGroceryDict( - groceryList, - inputWorkspace=request.diffractionWorkspace, + artificialNormWorkspace = ArtificialNormalizationRecipe().executeRecipe( + InputWorkspace=request.diffractionWorkspace, + Ingredients=ingredients, ) - - data = ArtificialNormalizationRecipe().cook(ingredients, groceries) - return data + return artificialNormWorkspace diff --git a/src/snapred/ui/model/WorkflowNodeModel.py b/src/snapred/ui/model/WorkflowNodeModel.py index c70751394..e483974bf 100644 --- a/src/snapred/ui/model/WorkflowNodeModel.py +++ b/src/snapred/ui/model/WorkflowNodeModel.py @@ -18,14 +18,6 @@ class WorkflowNodeModel(object): iterate: bool = False continueAnywayHandler: Callable[[ContinueWarning.Model], None] = None - def hide(self): - """Hide the node's view.""" - self.view.setVisible(False) - - def show(self): - """Show the node's view.""" - self.view.setVisible(True) - def __iter__(self): return _WorkflowModelIterator(self) diff --git a/src/snapred/ui/workflow/ReductionWorkflow.py b/src/snapred/ui/workflow/ReductionWorkflow.py index 07fd91fb7..83a4c55f9 100644 --- a/src/snapred/ui/workflow/ReductionWorkflow.py +++ b/src/snapred/ui/workflow/ReductionWorkflow.py @@ -7,6 +7,8 @@ ReductionExportRequest, ReductionRequest, ) +from snapred.backend.dao.response.ArtificialNormResponse import ArtificialNormResponse +from snapred.backend.dao.response.ReductionResponse import ReductionResponse from snapred.backend.dao.SNAPResponse import ResponseCode, SNAPResponse from snapred.backend.error.ContinueWarning import ContinueWarning from snapred.backend.log.logger import snapredLogger @@ -24,7 +26,6 @@ class ReductionWorkflow(WorkflowImplementer): def __init__(self, parent=None): super().__init__(parent) - self.artificialNormComplete = False self._reductionRequestView = ReductionRequestView( parent=parent, populatePixelMaskDropdown=self._populatePixelMaskDropdown, @@ -57,10 +58,9 @@ def __init__(self, parent=None): continueAnywayHandler=self._continueAnywayHandler, ) .addNode( - self._artificialNormalization, + self._continueWithNormalization, self._artificialNormalizationView, "Artificial Normalization", - visible=False, ) .addNode(self._nothing, self._reductionSaveView, "Save") .build() @@ -162,7 +162,7 @@ def _triggerReduction(self, workflowPresenter): ) response = self.request(path="reduction/", payload=request_) - if response.code == ResponseCode.OK: + if isinstance(response.data, ReductionResponse): record, unfocusedData = response.data.record, response.data.unfocusedData # .. update "save" panel message: @@ -184,47 +184,10 @@ def _triggerReduction(self, workflowPresenter): if unfocusedData is not None: self.outputs.append(unfocusedData) - elif ( - response.message == "missing normalization" - and response.code == ResponseCode.CONTINUE_WARNING - and response.data is not None - ): + elif isinstance(response.data, ArtificialNormResponse): self._artificialNormalizationView.updateRunNumber(runNumber) - workflowPresenter.presenter.widget.showTab("Artificial Normalization") self._artificialNormalization(workflowPresenter, response.data, runNumber) - - if self.artificialNormComplete: - artificialNormWorkspace = self._artificialNormalizationView.artificialNormWorkspace - - # Modify the request to use the artificial normalization workspace - request_ = ReductionRequest( - runNumber=str(runNumber), - useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(), - timestamp=timestamp, - continueFlags=self.continueAnywayFlags, - pixelMasks=pixelMasks, - keepUnfocused=self._reductionRequestView.retainUnfocusedDataCheckbox.isChecked(), - convertUnitsTo=self._reductionRequestView.convertUnitsDropdown.currentText(), - normalizationWorkspace=artificialNormWorkspace, - ) - - # Re-trigger reduction - response = self.request(path="reduction/", payload=request_) - - if response.code == ResponseCode.OK: - # Continue to the save step as before - record, unfocusedData = response.data.record, response.data.unfocusedData - savePath = self.request(path="reduction/getSavePath", payload=record.runNumber).data - self._reductionSaveView.updateContinueAnyway(self.continueAnywayFlags) - self._reductionSaveView.updateSavePath(savePath) - - if ContinueWarning.Type.NO_WRITE_PERMISSIONS not in self.continueAnywayFlags: - self.request(path="reduction/save", payload=ReductionExportRequest(record=record)) - - # Handle output workspaces - self.outputs.extend(record.workspaceNames) - if unfocusedData is not None: - self.outputs.append(unfocusedData) + return self.responses[-1] # Note that the run number is deliberately not deleted from the run numbers list. # Almost certainly it should be moved to a "completed run numbers" list. @@ -233,11 +196,10 @@ def _triggerReduction(self, workflowPresenter): # TODO: make '_clearWorkspaces' a public method (i.e make this combination a special `cleanup` method). self._clearWorkspaces(exclude=self.outputs, clearCachedWorkspaces=True) - return self.responses[-1] + return self.responses[-2] def _artificialNormalization(self, workflowPresenter, responseData, runNumber): view = workflowPresenter.widget.tabView # noqa: F841 - try: # Handle artificial normalization request here request_ = CreateArtificialNormalizationRequest( @@ -247,17 +209,15 @@ def _artificialNormalization(self, workflowPresenter, responseData, runNumber): smoothingParameter=self._artificialNormalizationView.smoothingSlider.field.value(), decreaseParameter=self._artificialNormalizationView.decreaseParameterDropdown.currentIndex() == 1, lss=self._artificialNormalizationView.lssDropdown.currentIndex() == 1, - diffcalWorkspace=responseData.get("diffractionWorkspace"), + diffractionWorkspace=responseData.diffractionWorkspace, ) - response = self.request(path="reduction/artificialNormalization", payload=request_.json()) - + response = self.request(path="reduction/artificialNormalization", payload=request_) # Update workspaces in the artificial normalization view - diffractionWorkspace = response.data.get("diffractionWorkspace") - artificialNormWorkspace = response.data.get("artificialNormWorkspace") + diffractionWorkspace = responseData.diffractionWorkspace + artificialNormWorkspace = response.data if diffractionWorkspace and artificialNormWorkspace: self._artificialNormalizationView.updateWorkspaces(diffractionWorkspace, artificialNormWorkspace) - self.artificialNormComplete = True else: print(f"Error: Workspaces not found in the response: {response.data}") except Exception as e: # noqa: BLE001 @@ -277,12 +237,11 @@ def onArtificialNormalizationValueChange(self, smoothingValue, lss, decreasePara smoothingParameter=smoothingValue, decreaseParameter=decreaseParameter, lss=lss, - diffractionWorkspace=WorkspaceName(diffractionWorkspace), + diffractionWorkspace=diffractionWorkspace, ) - response = self.request(path="reduction/artificialNormalization", payload=request_.json()) - diffractionWorkspace = response.data["diffractionWorkspace"] - artificialNormWorkspace = response.data["artificialNormWorkspace"] + response = self.request(path="reduction/artificialNormalization", payload=request_) + artificialNormWorkspace = response.data # Update the view with new workspaces self._artificialNormalizationView.updateWorkspaces(diffractionWorkspace, artificialNormWorkspace) @@ -292,6 +251,48 @@ def onArtificialNormalizationValueChange(self, smoothingValue, lss, decreasePara self._artificialNormalizationView.enableRecalculateButton() + def _continueWithNormalization(self, workflowPresenter): + # Get the updated normalization workspace from the ArtificialNormalizationView + view = workflowPresenter.widget.tabView # noqa: F841 + artificialNormWorkspace = self._artificialNormalizationView.artificialNormWorkspace + + # Now modify the request to use the artificial normalization workspace and continue the workflow + pixelMasks = self._reconstructPixelMaskNames(self._reductionRequestView.getPixelMasks()) + timestamp = self.request(path="reduction/getUniqueTimestamp").data + + request_ = ReductionRequest( + runNumber=str(self._artificialNormalizationView.fieldRunNumber.text()), + useLiteMode=self._reductionRequestView.liteModeToggle.field.getState(), + timestamp=timestamp, + continueFlags=self.continueAnywayFlags, + pixelMasks=pixelMasks, + keepUnfocused=self._reductionRequestView.retainUnfocusedDataCheckbox.isChecked(), + convertUnitsTo=self._reductionRequestView.convertUnitsDropdown.currentText(), + normalizationWorkspace=artificialNormWorkspace, + ) + + # Re-trigger reduction with the artificial normalization workspace + response = self.request(path="reduction/", payload=request_) + + if response.code == ResponseCode.OK: + # Continue to the save step as before + record, unfocusedData = response.data.record, response.data.unfocusedData + savePath = self.request(path="reduction/getSavePath", payload=record.runNumber).data + self._reductionSaveView.updateContinueAnyway(self.continueAnywayFlags) + self._reductionSaveView.updateSavePath(savePath) + + if ContinueWarning.Type.NO_WRITE_PERMISSIONS not in self.continueAnywayFlags: + self.request(path="reduction/save", payload=ReductionExportRequest(record=record)) + + # Handle output workspaces + self.outputs.extend(record.workspaceNames) + if unfocusedData is not None: + self.outputs.append(unfocusedData) + + # Clear workspaces except the output ones before transitioning to the save panel + self._clearWorkspaces(exclude=self.outputs, clearCachedWorkspaces=True) + return self.responses[-1] + @property def widget(self): return self.workflow.presenter.widget diff --git a/src/snapred/ui/workflow/WorkflowBuilder.py b/src/snapred/ui/workflow/WorkflowBuilder.py index dc7c038bb..5fc46e554 100644 --- a/src/snapred/ui/workflow/WorkflowBuilder.py +++ b/src/snapred/ui/workflow/WorkflowBuilder.py @@ -10,43 +10,33 @@ def __init__(self, *, startLambda=None, iterateLambda=None, resetLambda=None, ca self._resetLambda = resetLambda self._cancelLambda = cancelLambda self._workflow = None - self._invisibleNodes = [] def addNode( - self, - continueAction, - subview, - name="Unnamed", - required=True, - iterate=False, - continueAnywayHandler=None, - visible=True, + self, continueAction, subview, name="Unnamed", required=True, iterate=False, continueAnywayHandler=None ): - """ - Adds a node to the workflow. If visible=False, the node is initially hidden. - """ - node = WorkflowNodeModel( - continueAction=continueAction, - view=subview, - nextModel=None, - name=name, - required=required, - iterate=iterate, - continueAnywayHandler=continueAnywayHandler, - ) - - # If the node is invisible, add it to the invisible nodes list - if not visible: - self._invisibleNodes.append(node) - if self._workflow is None: - self._workflow = node + self._workflow = WorkflowNodeModel( + continueAction=continueAction, + view=subview, + nextModel=None, + name=name, + required=required, + iterate=iterate, + continueAnywayHandler=continueAnywayHandler, + ) else: currentWorkflow = self._workflow while currentWorkflow.nextModel is not None: currentWorkflow = currentWorkflow.nextModel - currentWorkflow.nextModel = node - + currentWorkflow.nextModel = WorkflowNodeModel( + continueAction=continueAction, + view=subview, + nextModel=None, + name=name, + required=required, + iterate=iterate, + continueAnywayHandler=continueAnywayHandler, + ) return self def makeNodeVisible(self, nodeName): diff --git a/tests/unit/backend/recipe/test_ArtificialNormalizationRecipe.py b/tests/unit/backend/recipe/test_ArtificialNormalizationRecipe.py deleted file mode 100644 index 62d2b937b..000000000 --- a/tests/unit/backend/recipe/test_ArtificialNormalizationRecipe.py +++ /dev/null @@ -1,94 +0,0 @@ -import json -import unittest -from unittest import mock - -import pytest -from mantid.simpleapi import ( - CreateSingleValuedWorkspace, - mtd, -) -from snapred.backend.dao.ingredients import ArtificialNormalizationIngredients as Ingredients -from snapred.backend.recipe.ArtificialNormalizationRecipe import ArtificialNormalizationRecipe as Recipe -from util.SculleryBoy import SculleryBoy - - -class TestArtificialNormalizationRecipe(unittest.TestCase): - sculleryBoy = SculleryBoy() - - def __make_groceries(self): - inputWorkspace = mtd.unique_name(prefix="inputworkspace") - outputWorkspace = "outputworkspace" # noqa: F841 - CreateSingleValuedWorkspace(OutputWorkspace=inputWorkspace) - - def test_chopIngredients(self): - recipe = Recipe() - ingredients = mock.Mock() - recipe.chopIngredients(ingredients) - assert ingredients.copy() == recipe.ingredients - - def test_unbagGroceries(self): - recipe = Recipe() - - groceries = { - "inputWorkspace": "sample", - "outputWorkspace": "output", - } - recipe.unbagGroceries(groceries) - assert recipe.inputWS == groceries["inputWorkspace"] - assert recipe.outputWS == groceries["outputWorkspace"] - - groceries = { - "inputWorkspace": "sample", - "outputWorkspace": "output", - } - recipe.unbagGroceries(groceries) - assert recipe.inputWS == groceries["inputWorkspace"] - assert recipe.outputWS == groceries["outputWorkspace"] - groceries = {} - with pytest.raises(KeyError): - recipe.unbagGroceries(groceries) - - def test_queueAlgos(self): - recipe = Recipe() - recipe.mantidSnapper = mock.Mock() - ingredients = Ingredients(peakWindowClippingSize=10, smoothingParameter=0.5, decreaseParameter=True, lss=True) - groceries = {"inputWorkspace": "inputworkspace", "outputWorkspace": "outputworkspace"} - - recipe.chopIngredients(ingredients) - recipe.unbagGroceries(groceries) - - recipe.queueAlgos() - - expected_ingredients_dict = { - "peakWindowClippingSize": 10, - "smoothingParameter": 0.5, - "decreaseParameter": True, - "lss": True, - } - expected_ingredients_str = json.dumps(expected_ingredients_dict) - - recipe.mantidSnapper.CreateArtificialNormalizationAlgo.assert_called_once_with( - "Creating artificial normalization...", - InputWorkspace="inputworkspace", - OutputWorkspace="outputworkspace", - Ingredients=expected_ingredients_str, - ) - - def test_cook(self): - recipe = Recipe() - recipe.prep = mock.Mock() - recipe.execute = mock.Mock() - recipe.cook(None, None) - recipe.prep.assert_called() - recipe.execute.assert_called() - - def test_cater(self): - recipe = Recipe() - recipe.cook = mock.Mock() - mockIngredients = mock.Mock() - mockGroceries = mock.Mock() - pallet = (mockIngredients, mockGroceries) - shipment = [pallet] - output = recipe.cater(shipment) - recipe.cook.assert_called_once_with(mockIngredients, mockGroceries) - assert output[0] == recipe.cook.return_value From 2b25773448842b7a3bf77d143913ded6bd30e5d4 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 14 Oct 2024 08:08:24 -0400 Subject: [PATCH 10/23] More updates. Now for some verfication and fixing some tests. --- src/snapred/ui/workflow/ReductionWorkflow.py | 4 ++-- src/snapred/ui/workflow/WorkflowBuilder.py | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/snapred/ui/workflow/ReductionWorkflow.py b/src/snapred/ui/workflow/ReductionWorkflow.py index 83a4c55f9..62f4e841e 100644 --- a/src/snapred/ui/workflow/ReductionWorkflow.py +++ b/src/snapred/ui/workflow/ReductionWorkflow.py @@ -195,8 +195,8 @@ def _triggerReduction(self, workflowPresenter): # _before_ transitioning to the "save" panel. # TODO: make '_clearWorkspaces' a public method (i.e make this combination a special `cleanup` method). self._clearWorkspaces(exclude=self.outputs, clearCachedWorkspaces=True) - - return self.responses[-2] + workflowPresenter.advanceWorkflow() + return self.responses[-1] def _artificialNormalization(self, workflowPresenter, responseData, runNumber): view = workflowPresenter.widget.tabView # noqa: F841 diff --git a/src/snapred/ui/workflow/WorkflowBuilder.py b/src/snapred/ui/workflow/WorkflowBuilder.py index 5fc46e554..4e2dde6ea 100644 --- a/src/snapred/ui/workflow/WorkflowBuilder.py +++ b/src/snapred/ui/workflow/WorkflowBuilder.py @@ -39,16 +39,6 @@ def addNode( ) return self - def makeNodeVisible(self, nodeName): - """ - Makes an invisible node visible by name. - """ - for node in self._invisibleNodes: - if node.name == nodeName: - node.view.setVisible(True) # Set the node's view to be visible - self._invisibleNodes.remove(node) # Remove from the invisible list - break - def build(self): return Workflow( self._workflow, From 8c27c74b0078e2c6f054508fc94f374817584434 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 14 Oct 2024 08:37:14 -0400 Subject: [PATCH 11/23] Updates to tests, checking code cov now --- src/snapred/backend/service/SousChef.py | 2 +- .../backend/recipe/test_ReductionRecipe.py | 23 +------------------ 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/src/snapred/backend/service/SousChef.py b/src/snapred/backend/service/SousChef.py index ab20d49b7..a290ec637 100644 --- a/src/snapred/backend/service/SousChef.py +++ b/src/snapred/backend/service/SousChef.py @@ -239,7 +239,7 @@ def _pullNormalizationRecordFFI( calibrantSamplePath = None if normalizationRecord is not None: smoothingParameter = normalizationRecord.smoothingParameter - calibrantSamplePath = normalizationRecord.normalizationCalibrantSamplePath + calibrantSamplePath = normalizationRecord.calibrantSamplePath # TODO: Should smoothing parameter be an ingredient? return ingredients, smoothingParameter, calibrantSamplePath diff --git a/tests/unit/backend/recipe/test_ReductionRecipe.py b/tests/unit/backend/recipe/test_ReductionRecipe.py index f4253e09d..ddd5b0cb4 100644 --- a/tests/unit/backend/recipe/test_ReductionRecipe.py +++ b/tests/unit/backend/recipe/test_ReductionRecipe.py @@ -3,7 +3,7 @@ import pytest from mantid.simpleapi import CreateSingleValuedWorkspace, mtd -from snapred.backend.dao.ingredients import ArtificialNormalizationIngredients, ReductionIngredients +from snapred.backend.dao.ingredients import ReductionIngredients from snapred.backend.recipe.ReductionRecipe import ( ApplyNormalizationRecipe, GenerateFocussedVanadiumRecipe, @@ -289,27 +289,6 @@ def test_cloneIntermediateWorkspace(self): mock.ANY, InputWorkspace="input", OutputWorkspace="output" ) - @mock.patch("snapred.backend.recipe.ReductionRecipe.ArtificialNormalizationRecipe") - def test_apply_artificial_normalization(self, artificialNorm): - recipe = ReductionRecipe() - recipe.prep = mock.Mock() - recipe.execute = mock.Mock() - recipe.sampleWs = "sample_ws" - recipe.normalizationWs = None - groceries = {"inputWorkspace": "sample_ws"} - recipe.groceries = groceries - - artificialNorm().cook.return_value = "artificial_normalization_ws" - result = recipe.cook(mock.Mock(), groceries) # noqa: F841 - artificialNorm().cook.assert_called_once_with( - ArtificialNormalizationIngredients( - peakWindowClippingSize=5, smoothingParameter=0.5, decreaseParameter=True, lss=True - ), - groceries, - ) - assert recipe.normalizationWs == "artificial_normalization_ws" - recipe.execute.assert_called_once() - def test_execute(self): recipe = ReductionRecipe() recipe.groceries = {} From 7ee1dd99e7b0215f00394b96bd37b19edb2e9254 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 14 Oct 2024 12:21:09 -0400 Subject: [PATCH 12/23] Fix tests and satisfy code cov --- .../backend/error/StateValidationException.py | 15 +- .../ui/view/reduction/ReductionRequestView.py | 4 + .../backend/service/test_ReductionService.py | 155 ++++++++++++++++++ 3 files changed, 169 insertions(+), 5 deletions(-) diff --git a/src/snapred/backend/error/StateValidationException.py b/src/snapred/backend/error/StateValidationException.py index ba1b6f28e..6c11fd1db 100644 --- a/src/snapred/backend/error/StateValidationException.py +++ b/src/snapred/backend/error/StateValidationException.py @@ -11,9 +11,14 @@ class StateValidationException(Exception): "Raised when an Instrument State is invalid" - def __init__(self, exception: Exception): - exceptionStr = str(exception) - tb = exception.__traceback__ + def __init__(self, exception): + # Handle both string and Exception types for 'exception' + if isinstance(exception, Exception): + exceptionStr = str(exception) + tb = exception.__traceback__ + else: + exceptionStr = str(exception) + tb = None if tb is not None: tb_info = traceback.extract_tb(tb) @@ -22,9 +27,9 @@ def __init__(self, exception: Exception): lineNumber = tb_info[-1].lineno functionName = tb_info[-1].name else: - filePath, lineNumber, functionName = None, lineNumber, functionName + filePath, lineNumber, functionName = None, None, None else: - filePath, lineNumber, functionName = None, None, None + filePath, lineNumber, functionName = None, None, None # noqa: F841 doesFileExist, hasWritePermission = self._checkFileAndPermissions(filePath) diff --git a/src/snapred/ui/view/reduction/ReductionRequestView.py b/src/snapred/ui/view/reduction/ReductionRequestView.py index be181e2b4..03d1e40b6 100644 --- a/src/snapred/ui/view/reduction/ReductionRequestView.py +++ b/src/snapred/ui/view/reduction/ReductionRequestView.py @@ -136,6 +136,10 @@ def clearRunNumbers(self): def verify(self): currentText = self.runNumberDisplay.toPlainText() runNumbers = [num.strip() for num in currentText.split("\n") if num.strip()] + + if not runNumbers: + raise ValueError("Please enter at least one run number.") + for runNumber in runNumbers: if not runNumber.isdigit(): raise ValueError( diff --git a/tests/unit/backend/service/test_ReductionService.py b/tests/unit/backend/service/test_ReductionService.py index c8a157f9c..d8de21ac5 100644 --- a/tests/unit/backend/service/test_ReductionService.py +++ b/tests/unit/backend/service/test_ReductionService.py @@ -7,6 +7,8 @@ import pydantic import pytest from mantid.simpleapi import ( + ConvertUnits, + CreateWorkspace, DeleteWorkspace, mtd, ) @@ -14,6 +16,7 @@ from snapred.backend.dao.ingredients.ReductionIngredients import ReductionIngredients from snapred.backend.dao.reduction.ReductionRecord import ReductionRecord from snapred.backend.dao.request import ( + CreateArtificialNormalizationRequest, ReductionExportRequest, ReductionRequest, ) @@ -21,6 +24,7 @@ from snapred.backend.dao.SNAPRequest import SNAPRequest from snapred.backend.dao.state import DetectorState from snapred.backend.dao.state.FocusGroup import FocusGroup +from snapred.backend.dao.state.PixelGroupingParameters import PixelGroupingParameters from snapred.backend.error.ContinueWarning import ContinueWarning from snapred.backend.error.StateValidationException import StateValidationException from snapred.backend.service.ReductionService import ReductionService @@ -350,6 +354,157 @@ def test_validateReduction_no_permissions_and_no_calibrations_second_reentry(sel # and in addition, re-entry for the second continue-anyway check. self.instance.validateReduction(self.request) + def test_validateReduction_with_continueFlags(self): + self.request.continueFlags = None + + self.instance.dataFactoryService.normalizationExists = mock.Mock(return_value=True) + self.instance.dataFactoryService.calibrationExists = mock.Mock(return_value=True) + self.instance.checkWritePermissions = mock.Mock(return_value=False) + + with pytest.raises(ContinueWarning) as excInfo: + self.instance.validateReduction(self.request) + + assert excInfo.value.model.flags == ContinueWarning.Type.NO_WRITE_PERMISSIONS + + def test_validateReduction_with_warningMessages(self): + self.instance.dataFactoryService.normalizationExists = mock.Mock(return_value=False) + self.instance.dataFactoryService.calibrationExists = mock.Mock(return_value=False) + + with pytest.raises(ContinueWarning) as excInfo: + self.instance.validateReduction(self.request) + + assert "Normalization is missing" in str(excInfo.value) + assert "Diffraction calibration is missing" in str(excInfo.value) + + def test_artificialNormalization(self): + mockAlgo = mock.Mock() + len_wksp = 6 + input_ws_name = mtd.unique_name(prefix="input_ws_") + + mockAlgo.executeRecipe.return_value = input_ws_name + "_artificial_norm" + CreateWorkspace( + OutputWorkspace=input_ws_name, + DataX=[1] * len_wksp, + DataY=[1] * len_wksp, + NSpec=len_wksp, + UnitX="dSpacing", + ) + ConvertUnits( + InputWorkspace=input_ws_name, + OutputWorkspace=input_ws_name, + Target="dSpacing", + ) + with mock.patch( + "snapred.backend.service.ReductionService.ArtificialNormalizationRecipe", return_value=mockAlgo + ): + request = CreateArtificialNormalizationRequest( + runNumber="12345", + useLiteMode=True, + peakWindowClippingSize=1.0, + smoothingParameter=0.5, + decreaseParameter=True, + lss=True, + diffractionWorkspace=input_ws_name, + ) + + response = self.instance.artificialNormalization(request) + + assert response == input_ws_name + "_artificial_norm" + + mockAlgo.executeRecipe.assert_called_once_with(InputWorkspace=input_ws_name, Ingredients=mock.ANY) + + def test_loadAllGroupings_with_exception(self): + mock_grouping_map = mock.Mock() + mock_grouping_map.getMap.return_value = {} + + self.instance.dataFactoryService.getGroupingMap = mock.Mock( + side_effect=StateValidationException("Invalid State") + ) + + self.instance.dataFactoryService.getDefaultGroupingMap = mock.Mock(return_value=mock_grouping_map) + + result = self.instance.loadAllGroupings(self.request.runNumber, self.request.useLiteMode) + + mock_grouping_map.getMap.assert_called_once_with(self.request.useLiteMode) + + assert result == {"focusGroups": [], "groupingWorkspaces": []} + + def test_saveReductionPath(self): + mockPixelGroupingParams = { + "group1": [mock.Mock(spec=PixelGroupingParameters), mock.Mock(spec=PixelGroupingParameters)] + } + + mockRecord = ReductionRecord( + runNumber="123456", + useLiteMode=True, + timestamp=123456.789, + pixelGroupingParameters=mockPixelGroupingParams, + workspaceNames=["workspace1", "workspace2"], + ) + + mockRequest = ReductionExportRequest(record=mockRecord) + + with ( + mock.patch.object(self.instance.dataExportService, "exportReductionRecord") as mockExportRecord, + mock.patch.object(self.instance.dataExportService, "exportReductionData") as mockExportData, + ): + self.instance.saveReduction(mockRequest) + + mockExportRecord.assert_called_once_with(mockRecord) + mockExportData.assert_called_once_with(mockRecord) + + def test_loadReductionPath(self): + with pytest.raises(NotImplementedError): + self.instance.loadReduction("someState", 123456.789) + + def test_prepCombinedMask(self): + self.maskWS1 = mock.Mock() + self.maskWS2 = mock.Mock() + + masks = [self.maskWS1, self.maskWS2] + + combinedMaskName = ( # noqa: F841 + wng.reductionPixelMask() + .runNumber(self.request.runNumber) + .timestamp(self.instance.getUniqueTimestamp()) + .build() + ) + + with mock.patch.object(self.instance.mantidSnapper, "BinaryOperateMasks") as mockBinaryOperateMasks: + self.instance.prepCombinedMask( + self.request.runNumber, self.request.useLiteMode, self.request.timestamp, masks + ) + + assert mockBinaryOperateMasks.call_count == len(masks) + + def test_checkWritePermissions_fails(self): + with mock.patch.object(self.instance.dataExportService, "checkWritePermissions", return_value=False): + assert not self.instance.checkWritePermissions("123456") + + def test_createReductionRecord_missing_normalization_and_calibration(self): + self.request.continueFlags = ( + ContinueWarning.Type.MISSING_DIFFRACTION_CALIBRATION | ContinueWarning.Type.MISSING_NORMALIZATION + ) + + mockIngredients = ReductionIngredients( + runNumber="12345", + useLiteMode=True, + timestamp=123456789.0, + pixelGroups=[], + smoothingParameter=0.1, + calibrantSamplePath="path/to/calibrant", + peakIntensityThreshold=0.05, + keepUnfocused=True, + convertUnitsTo="TOF", + ) + mockWorkspaceNames = ["ws1", "ws2"] + + result = self.instance._createReductionRecord(self.request, mockIngredients, mockWorkspaceNames) + + assert result.normalization is None + assert result.calibration is None + assert result.workspaceNames == mockWorkspaceNames + class TestReductionServiceMasks: @pytest.fixture(autouse=True, scope="class") From 6fe5a280be6610d5e845b92f0af8cf42e42e2d54 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 14 Oct 2024 12:36:39 -0400 Subject: [PATCH 13/23] More tests... --- .../backend/service/test_ReductionService.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/unit/backend/service/test_ReductionService.py b/tests/unit/backend/service/test_ReductionService.py index d8de21ac5..c5eb441b3 100644 --- a/tests/unit/backend/service/test_ReductionService.py +++ b/tests/unit/backend/service/test_ReductionService.py @@ -157,6 +157,34 @@ def test_saveReduction(self): mockExportRecord.assert_called_once_with(record) mockExportData.assert_called_once_with(record) + def test_validateReduction_with_artificialNormalization_and_no_permissions(self): + self.request.artificialNormalization = mock.Mock() + self.instance.checkWritePermissions = mock.Mock(return_value=False) + + with pytest.raises(ContinueWarning) as excInfo: + self.instance.validateReduction(self.request) + + assert excInfo.value.model.flags == ContinueWarning.Type.NO_WRITE_PERMISSIONS + + def test_validateReduction_with_continueFlags_xor_operation(self): + self.request.artificialNormalization = mock.Mock() + self.request.continueFlags = ContinueWarning.Type.NO_WRITE_PERMISSIONS + + self.instance.checkWritePermissions = mock.Mock(return_value=True) + + self.instance.validateReduction(self.request) + + def test_fetchReductionGroceries_with_artificialNormalization(self): + self.request.artificialNormalization = "artificial_norm_ws" + + self.instance.dataFactoryService.getThisOrLatestCalibrationVersion = mock.Mock(return_value=1) + + self.instance.fetchReductionGroceries(self.request) + + self.instance.groceryClerk.name("diffcalWorkspace").diffcal_table.assert_called_once_with( + self.request.runNumber, 1 + ) + def test_loadReduction(self): ## this makes codecov happy with pytest.raises(NotImplementedError): From 4dda2b6206c1e3685a7c7c65687e1201478ad352 Mon Sep 17 00:00:00 2001 From: Darsh Dinger <106342815+darshdinger@users.noreply.github.com> Date: Mon, 14 Oct 2024 12:54:33 -0400 Subject: [PATCH 14/23] Delete tests/resources/outputs/APIServicePaths.json.new --- tests/resources/outputs/APIServicePaths.json.new | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 tests/resources/outputs/APIServicePaths.json.new diff --git a/tests/resources/outputs/APIServicePaths.json.new b/tests/resources/outputs/APIServicePaths.json.new deleted file mode 100644 index 126e21be3..000000000 --- a/tests/resources/outputs/APIServicePaths.json.new +++ /dev/null @@ -1,7 +0,0 @@ -{ - "mockService": { - "": { - "mockObject": "{\n \"properties\": {\n \"m_list\": {\n \"items\": {\n \"type\": \"number\"\n },\n \"title\": \"M List\",\n \"type\": \"array\"\n },\n \"m_float\": {\n \"title\": \"M Float\",\n \"type\": \"number\"\n },\n \"m_int\": {\n \"title\": \"M Int\",\n \"type\": \"integer\"\n },\n \"m_string\": {\n \"title\": \"M String\",\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"m_list\",\n \"m_float\",\n \"m_int\",\n \"m_string\"\n ],\n \"title\": \".MockObject'>\",\n \"type\": \"object\"\n}" - } - } -} From 19979ee6c8812d5ef5cb03e21407bc0529b882a2 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 14 Oct 2024 13:02:06 -0400 Subject: [PATCH 15/23] more code cov... --- .../backend/service/test_ReductionService.py | 71 +++++++++++-------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/tests/unit/backend/service/test_ReductionService.py b/tests/unit/backend/service/test_ReductionService.py index c5eb441b3..a87de4959 100644 --- a/tests/unit/backend/service/test_ReductionService.py +++ b/tests/unit/backend/service/test_ReductionService.py @@ -157,34 +157,6 @@ def test_saveReduction(self): mockExportRecord.assert_called_once_with(record) mockExportData.assert_called_once_with(record) - def test_validateReduction_with_artificialNormalization_and_no_permissions(self): - self.request.artificialNormalization = mock.Mock() - self.instance.checkWritePermissions = mock.Mock(return_value=False) - - with pytest.raises(ContinueWarning) as excInfo: - self.instance.validateReduction(self.request) - - assert excInfo.value.model.flags == ContinueWarning.Type.NO_WRITE_PERMISSIONS - - def test_validateReduction_with_continueFlags_xor_operation(self): - self.request.artificialNormalization = mock.Mock() - self.request.continueFlags = ContinueWarning.Type.NO_WRITE_PERMISSIONS - - self.instance.checkWritePermissions = mock.Mock(return_value=True) - - self.instance.validateReduction(self.request) - - def test_fetchReductionGroceries_with_artificialNormalization(self): - self.request.artificialNormalization = "artificial_norm_ws" - - self.instance.dataFactoryService.getThisOrLatestCalibrationVersion = mock.Mock(return_value=1) - - self.instance.fetchReductionGroceries(self.request) - - self.instance.groceryClerk.name("diffcalWorkspace").diffcal_table.assert_called_once_with( - self.request.runNumber, 1 - ) - def test_loadReduction(self): ## this makes codecov happy with pytest.raises(NotImplementedError): @@ -533,6 +505,49 @@ def test_createReductionRecord_missing_normalization_and_calibration(self): assert result.calibration is None assert result.workspaceNames == mockWorkspaceNames + def test_validateReduction_with_artificialNormalization_and_no_permissions(self): + self.request.artificialNormalization = mock.Mock() + self.instance.checkWritePermissions = mock.Mock(return_value=False) + + with pytest.raises(ContinueWarning) as excInfo: + self.instance.validateReduction(self.request) + + assert excInfo.value.model.flags == ContinueWarning.Type.NO_WRITE_PERMISSIONS + + def test_validateReduction_with_continueFlags_xor_operation(self): + self.request.artificialNormalization = mock.Mock() + self.request.continueFlags = ContinueWarning.Type.NO_WRITE_PERMISSIONS + + self.instance.checkWritePermissions = mock.Mock(return_value=True) + + self.instance.validateReduction(self.request) + + def test_fetchReductionGroceries_with_artificialNormalization(self): + self.request.artificialNormalization = "artificial_norm_ws" + + self.instance.dataFactoryService.getThisOrLatestCalibrationVersion = mock.Mock(return_value=1) + + mock_grocery_clerk = mock.Mock() + self.instance.groceryClerk = mock_grocery_clerk + + mock_grocery_clerk.name.return_value = mock_grocery_clerk + mock_grocery_clerk.diffcal_table.return_value = mock_grocery_clerk + mock_grocery_clerk.useLiteMode.return_value = mock_grocery_clerk + mock_grocery_clerk.add.return_value = mock_grocery_clerk + mock_grocery_clerk.buildDict.return_value = {"key1": "value1"} + + mock_grocery_service = mock.Mock() + self.instance.groceryService = mock_grocery_service + mock_grocery_service.fetchGroceryList.return_value = ["workspace1"] + + self.instance.fetchReductionGroceries(self.request) + + mock_grocery_clerk.name.assert_any_call("inputWorkspace") + mock_grocery_clerk.name.assert_any_call("diffcalWorkspace") + mock_grocery_clerk.diffcal_table.assert_called_once_with(self.request.runNumber, 1) + mock_grocery_clerk.useLiteMode.assert_called_once_with(self.request.useLiteMode) + mock_grocery_clerk.add.assert_called_once() + class TestReductionServiceMasks: @pytest.fixture(autouse=True, scope="class") From a731d37b304c0de32fdbfc3608ace6345b1fdd63 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 14 Oct 2024 13:14:38 -0400 Subject: [PATCH 16/23] Possibly now? --- .../backend/service/test_ReductionService.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tests/unit/backend/service/test_ReductionService.py b/tests/unit/backend/service/test_ReductionService.py index a87de4959..6f7222ea2 100644 --- a/tests/unit/backend/service/test_ReductionService.py +++ b/tests/unit/backend/service/test_ReductionService.py @@ -21,6 +21,7 @@ ReductionRequest, ) from snapred.backend.dao.request.ReductionRequest import Versions +from snapred.backend.dao.response.ArtificialNormResponse import ArtificialNormResponse from snapred.backend.dao.SNAPRequest import SNAPRequest from snapred.backend.dao.state import DetectorState from snapred.backend.dao.state.FocusGroup import FocusGroup @@ -548,6 +549,89 @@ def test_fetchReductionGroceries_with_artificialNormalization(self): mock_grocery_clerk.useLiteMode.assert_called_once_with(self.request.useLiteMode) mock_grocery_clerk.add.assert_called_once() + def test_fetchReductionGroceries_with_missing_normalization(self): + self.request.continueFlags = ContinueWarning.Type.UNSET + self.request.artificialNormalization = None + + self.instance.dataFactoryService.getThisOrLatestCalibrationVersion = mock.Mock(return_value=1) + self.instance.dataFactoryService.getThisOrLatestNormalizationVersion = mock.Mock(return_value=2) + + mock_grocery_clerk = mock.Mock() + self.instance.groceryClerk = mock_grocery_clerk + mock_grocery_clerk.name.return_value = mock_grocery_clerk + mock_grocery_clerk.normalization.return_value = mock_grocery_clerk + mock_grocery_clerk.useLiteMode.return_value = mock_grocery_clerk + mock_grocery_clerk.add.return_value = mock_grocery_clerk + mock_grocery_clerk.buildDict.return_value = {"key1": "value1"} + + mock_grocery_service = mock.Mock() + self.instance.groceryService = mock_grocery_service + mock_grocery_service.fetchGroceryDict.return_value = {"inputWorkspace": "workspace1"} + + self.instance.fetchReductionGroceries(self.request) + + self.instance.dataFactoryService.getThisOrLatestNormalizationVersion.assert_called_once_with( + self.request.runNumber, self.request.useLiteMode + ) + + mock_grocery_clerk.name.assert_any_call("normalizationWorkspace") + mock_grocery_clerk.normalization.assert_called_once_with(self.request.runNumber, 2) + mock_grocery_clerk.useLiteMode.assert_called_once_with(self.request.useLiteMode) + mock_grocery_clerk.add.assert_called_once() + + def test_fetchReductionGroceries_with_calibration_and_missing_normalization(self): + self.request.continueFlags = ContinueWarning.Type.MISSING_NORMALIZATION + self.request.artificialNormalization = None + + self.instance.dataFactoryService.getThisOrLatestCalibrationVersion = mock.Mock(return_value=1) + self.instance.dataFactoryService.getThisOrLatestNormalizationVersion = mock.Mock(return_value=None) + + mock_grocery_clerk = mock.Mock() + self.instance.groceryClerk = mock_grocery_clerk + mock_grocery_clerk.name.return_value = mock_grocery_clerk + mock_grocery_clerk.diffcal_output.return_value = mock_grocery_clerk + mock_grocery_clerk.useLiteMode.return_value = mock_grocery_clerk + mock_grocery_clerk.unit.return_value = mock_grocery_clerk + mock_grocery_clerk.group.return_value = mock_grocery_clerk + mock_grocery_clerk.buildDict.return_value = {"key1": "value1"} + + mock_grocery_service = mock.Mock() + self.instance.groceryService = mock_grocery_service + mock_grocery_service.fetchGroceryDict.return_value = {"diffractionWorkspace": "diffraction_ws"} + result = self.instance.fetchReductionGroceries(self.request) + + mock_grocery_clerk.name.assert_called_with("diffractionWorkspace") + mock_grocery_clerk.diffcal_output.assert_called_once_with(self.request.runNumber, 1) + mock_grocery_clerk.unit.assert_called_once_with(wng.Units.DSP) + mock_grocery_clerk.group.assert_called_once_with("column") + mock_grocery_clerk.buildDict.assert_called_once() + + assert isinstance(result, ArtificialNormResponse) + assert result.diffractionWorkspace == "diffraction_ws" + + def test_fetchReductionGroceries_creates_versions(self): + self.request.continueFlags = ContinueWarning.Type.UNSET + self.request.artificialNormalization = None + + self.instance.dataFactoryService.getThisOrLatestCalibrationVersion = mock.Mock(return_value=1) + self.instance.dataFactoryService.getThisOrLatestNormalizationVersion = mock.Mock(return_value=2) + + mock_grocery_clerk = mock.Mock() + self.instance.groceryClerk = mock_grocery_clerk + mock_grocery_clerk.name.return_value = mock_grocery_clerk + mock_grocery_clerk.useLiteMode.return_value = mock_grocery_clerk + mock_grocery_clerk.add.return_value = mock_grocery_clerk + mock_grocery_clerk.buildDict.return_value = {"key1": "value1"} + + mock_grocery_service = mock.Mock() + self.instance.groceryService = mock_grocery_service + mock_grocery_service.fetchGroceryDict.return_value = {"inputWorkspace": "workspace1"} + + self.instance.fetchReductionGroceries(self.request) + + assert self.request.versions.calibration == 1 + assert self.request.versions.normalization == 2 + class TestReductionServiceMasks: @pytest.fixture(autouse=True, scope="class") From fab6f905be47318f573a889683d2c10feae026ba Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 14 Oct 2024 13:30:40 -0400 Subject: [PATCH 17/23] Plz, hope this is the last one for code cov --- tests/unit/backend/service/test_ReductionService.py | 12 ++++++++++++ tests/unit/meta/test_Decorators.py | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/tests/unit/backend/service/test_ReductionService.py b/tests/unit/backend/service/test_ReductionService.py index 6f7222ea2..511015810 100644 --- a/tests/unit/backend/service/test_ReductionService.py +++ b/tests/unit/backend/service/test_ReductionService.py @@ -632,6 +632,18 @@ def test_fetchReductionGroceries_creates_versions(self): assert self.request.versions.calibration == 1 assert self.request.versions.normalization == 2 + def test_reduction_with_artificial_norm_response(self): + artificial_response = ArtificialNormResponse(diffractionWorkspace="mock_diffraction_ws") + self.instance.fetchReductionGroceries = mock.Mock(return_value=artificial_response) + + self.instance.dataFactoryService.calibrationExists = mock.Mock(return_value=True) + self.instance.dataFactoryService.normalizationExists = mock.Mock(return_value=True) + + result = self.instance.reduction(self.request) + + assert result == artificial_response + self.instance.fetchReductionGroceries.assert_called_once_with(self.request) + class TestReductionServiceMasks: @pytest.fixture(autouse=True, scope="class") diff --git a/tests/unit/meta/test_Decorators.py b/tests/unit/meta/test_Decorators.py index 551a73e65..ea2f0b76c 100644 --- a/tests/unit/meta/test_Decorators.py +++ b/tests/unit/meta/test_Decorators.py @@ -134,6 +134,19 @@ def test_stateValidationExceptionWritePerms(): assert "The following error occurred:Test Exception\n\nPlease contact your CIS." in str(excinfo.value) +def test_stateValidationExceptionNoTraceback(): + exception = Exception("Test Exception without traceback") + + with patch("snapred.backend.error.StateValidationException.logger") as logger_mock: + with patch("traceback.extract_tb", return_value=None): + with pytest.raises(StateValidationException) as excinfo: + raise StateValidationException(exception) + + assert "Instrument State for given Run Number is invalid! (see logs for details.)" in str(excinfo.value) + + logger_mock.error.assert_called_once_with("Test Exception without traceback") + + @ExceptionHandler(StateValidationException) def throwsStateException(): raise RuntimeError("I love exceptions!!! Ah ha ha!") From c9f5ad47db6a77f294b86ec846c8c5869a29f0f9 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 14 Oct 2024 13:34:52 -0400 Subject: [PATCH 18/23] Now? --- src/snapred/backend/error/StateValidationException.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snapred/backend/error/StateValidationException.py b/src/snapred/backend/error/StateValidationException.py index 6c11fd1db..3e3d13097 100644 --- a/src/snapred/backend/error/StateValidationException.py +++ b/src/snapred/backend/error/StateValidationException.py @@ -27,7 +27,7 @@ def __init__(self, exception): lineNumber = tb_info[-1].lineno functionName = tb_info[-1].name else: - filePath, lineNumber, functionName = None, None, None + filePath, lineNumber, functionName = None, lineNumber, functionName else: filePath, lineNumber, functionName = None, None, None # noqa: F841 From 7954024d7148140d53329d18db22d08d3f8af8a9 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 14 Oct 2024 13:40:16 -0400 Subject: [PATCH 19/23] ? --- tests/unit/meta/test_Decorators.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/unit/meta/test_Decorators.py b/tests/unit/meta/test_Decorators.py index ea2f0b76c..551a73e65 100644 --- a/tests/unit/meta/test_Decorators.py +++ b/tests/unit/meta/test_Decorators.py @@ -134,19 +134,6 @@ def test_stateValidationExceptionWritePerms(): assert "The following error occurred:Test Exception\n\nPlease contact your CIS." in str(excinfo.value) -def test_stateValidationExceptionNoTraceback(): - exception = Exception("Test Exception without traceback") - - with patch("snapred.backend.error.StateValidationException.logger") as logger_mock: - with patch("traceback.extract_tb", return_value=None): - with pytest.raises(StateValidationException) as excinfo: - raise StateValidationException(exception) - - assert "Instrument State for given Run Number is invalid! (see logs for details.)" in str(excinfo.value) - - logger_mock.error.assert_called_once_with("Test Exception without traceback") - - @ExceptionHandler(StateValidationException) def throwsStateException(): raise RuntimeError("I love exceptions!!! Ah ha ha!") From 40c0587528b9855cf621436e52a866d78ab21ecf Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 14 Oct 2024 13:50:12 -0400 Subject: [PATCH 20/23] ?? --- tests/unit/meta/test_Decorators.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/unit/meta/test_Decorators.py b/tests/unit/meta/test_Decorators.py index 551a73e65..92922bd13 100644 --- a/tests/unit/meta/test_Decorators.py +++ b/tests/unit/meta/test_Decorators.py @@ -134,6 +134,19 @@ def test_stateValidationExceptionWritePerms(): assert "The following error occurred:Test Exception\n\nPlease contact your CIS." in str(excinfo.value) +def test_stateValidationExceptionNoTracebackDetails(): + exception = Exception("Test Exception without valid traceback") + + with patch("snapred.backend.error.StateValidationException.logger") as logger_mock: + with patch("traceback.extract_tb", return_value=[]): + with pytest.raises(StateValidationException) as excinfo: + raise StateValidationException(exception) + + assert str(excinfo.value) == "Instrument State for given Run Number is invalid! (see logs for details.)" + + logger_mock.error.assert_called_with("Test Exception without valid traceback") + + @ExceptionHandler(StateValidationException) def throwsStateException(): raise RuntimeError("I love exceptions!!! Ah ha ha!") From f3da7ebeb0a9c095539bd6baa6f977a4c493d7d2 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 14 Oct 2024 13:55:02 -0400 Subject: [PATCH 21/23] update to test_Decorators --- tests/unit/meta/test_Decorators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/meta/test_Decorators.py b/tests/unit/meta/test_Decorators.py index 92922bd13..507fdfdd5 100644 --- a/tests/unit/meta/test_Decorators.py +++ b/tests/unit/meta/test_Decorators.py @@ -138,7 +138,7 @@ def test_stateValidationExceptionNoTracebackDetails(): exception = Exception("Test Exception without valid traceback") with patch("snapred.backend.error.StateValidationException.logger") as logger_mock: - with patch("traceback.extract_tb", return_value=[]): + with patch("traceback.extract_tb", return_value=None): with pytest.raises(StateValidationException) as excinfo: raise StateValidationException(exception) From 3b8f5e1b4e86c3af57215db5a046f647f48e9d20 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 14 Oct 2024 14:01:43 -0400 Subject: [PATCH 22/23] Plzz. --- tests/unit/meta/test_Decorators.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/unit/meta/test_Decorators.py b/tests/unit/meta/test_Decorators.py index 507fdfdd5..324e22d6a 100644 --- a/tests/unit/meta/test_Decorators.py +++ b/tests/unit/meta/test_Decorators.py @@ -147,6 +147,21 @@ def test_stateValidationExceptionNoTracebackDetails(): logger_mock.error.assert_called_with("Test Exception without valid traceback") +def test_stateValidationExceptionWithPartialTracebackDetails(): + exception = Exception("Test Exception with incomplete traceback") + + mock_tb_info = [traceback.FrameSummary(filename=None, lineno=42, name="testFunction")] + + with patch("snapred.backend.error.StateValidationException.logger") as logger_mock: + with patch("traceback.extract_tb", return_value=mock_tb_info): + with pytest.raises(StateValidationException) as excinfo: + raise StateValidationException(exception) + + assert str(excinfo.value) == "Instrument State for given Run Number is invalid! (see logs for details.)" + + logger_mock.error.assert_called_with("Test Exception with incomplete traceback") + + @ExceptionHandler(StateValidationException) def throwsStateException(): raise RuntimeError("I love exceptions!!! Ah ha ha!") From 77a54544bb90b7930f447f733f851eed40860db3 Mon Sep 17 00:00:00 2001 From: Darsh Date: Mon, 14 Oct 2024 14:08:06 -0400 Subject: [PATCH 23/23] Added another test.. --- tests/unit/meta/test_Decorators.py | 71 ++++++++++++++++++------------ 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/tests/unit/meta/test_Decorators.py b/tests/unit/meta/test_Decorators.py index 324e22d6a..5588bbf0f 100644 --- a/tests/unit/meta/test_Decorators.py +++ b/tests/unit/meta/test_Decorators.py @@ -134,34 +134,6 @@ def test_stateValidationExceptionWritePerms(): assert "The following error occurred:Test Exception\n\nPlease contact your CIS." in str(excinfo.value) -def test_stateValidationExceptionNoTracebackDetails(): - exception = Exception("Test Exception without valid traceback") - - with patch("snapred.backend.error.StateValidationException.logger") as logger_mock: - with patch("traceback.extract_tb", return_value=None): - with pytest.raises(StateValidationException) as excinfo: - raise StateValidationException(exception) - - assert str(excinfo.value) == "Instrument State for given Run Number is invalid! (see logs for details.)" - - logger_mock.error.assert_called_with("Test Exception without valid traceback") - - -def test_stateValidationExceptionWithPartialTracebackDetails(): - exception = Exception("Test Exception with incomplete traceback") - - mock_tb_info = [traceback.FrameSummary(filename=None, lineno=42, name="testFunction")] - - with patch("snapred.backend.error.StateValidationException.logger") as logger_mock: - with patch("traceback.extract_tb", return_value=mock_tb_info): - with pytest.raises(StateValidationException) as excinfo: - raise StateValidationException(exception) - - assert str(excinfo.value) == "Instrument State for given Run Number is invalid! (see logs for details.)" - - logger_mock.error.assert_called_with("Test Exception with incomplete traceback") - - @ExceptionHandler(StateValidationException) def throwsStateException(): raise RuntimeError("I love exceptions!!! Ah ha ha!") @@ -232,3 +204,46 @@ def testFunc(): assert mockLogger.debug.call_count == 2 assert mockLogger.debug.call_args_list[0][0][0] == "Entering testFunc" assert mockLogger.debug.call_args_list[1][0][0] == "Exiting testFunc" + + +def test_stateValidationExceptionNoTracebackDetails(): + exception = Exception("Test Exception without valid traceback") + + with patch("snapred.backend.error.StateValidationException.logger") as logger_mock: + with patch("traceback.extract_tb", return_value=None): + with pytest.raises(StateValidationException) as excinfo: + raise StateValidationException(exception) + + assert str(excinfo.value) == "Instrument State for given Run Number is invalid! (see logs for details.)" + + logger_mock.error.assert_called_with("Test Exception without valid traceback") + + +def test_stateValidationExceptionWithPartialTracebackDetails(): + exception = Exception("Test Exception with incomplete traceback") + + mock_tb_info = [traceback.FrameSummary(filename=None, lineno=42, name="testFunction")] + + with patch("snapred.backend.error.StateValidationException.logger") as logger_mock: + with patch("traceback.extract_tb", return_value=mock_tb_info): + with pytest.raises(StateValidationException) as excinfo: + raise StateValidationException(exception) + + assert str(excinfo.value) == "Instrument State for given Run Number is invalid! (see logs for details.)" + + logger_mock.error.assert_called_with("Test Exception with incomplete traceback") + + +def test_stateValidationExceptionWithMissingFilePath(): + exception = Exception("Test Exception with missing file path") + + mock_tb_info = [traceback.FrameSummary(filename=None, lineno=42, name="testFunction")] + + with patch("snapred.backend.error.StateValidationException.logger") as logger_mock: + with patch("traceback.extract_tb", return_value=mock_tb_info): + with pytest.raises(StateValidationException) as excinfo: + raise StateValidationException(exception) + + assert str(excinfo.value) == "Instrument State for given Run Number is invalid! (see logs for details.)" + + logger_mock.error.assert_called_with("Test Exception with missing file path")