Upscaling QC

setup.py

@@ -69,6 +69,7 @@
             "SurfaceWithSeismicCrossSection = webviz_subsurface.plugins:SurfaceWithSeismicCrossSection",
             "TornadoPlotterFMU = webviz_subsurface.plugins:TornadoPlotterFMU",
             "VolumetricAnalysis = webviz_subsurface.plugins:VolumetricAnalysis",
+            "UpscalingQC = webviz_subsurface.plugins:UpscalingQC",
             "WellCrossSection = webviz_subsurface.plugins:WellCrossSection",
             "WellCrossSectionFMU = webviz_subsurface.plugins:WellCrossSectionFMU",
             "AssistedHistoryMatchingAnalysis = webviz_subsurface.plugins:AssistedHistoryMatchingAnalysis",

tests/unit_tests/model_tests/test_upscaling_qc_model.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+
+import pytest
+
+from pandas.api.types import CategoricalDtype
+from webviz_subsurface.plugins._upscaling_qc.models.upscaling_qc_model import (
+    UpscalingQCModel,
+)
+
+
+@pytest.fixture
+def qcm_model() -> Path:
+    model_folder = Path("../upscaling_qc")
+    yield UpscalingQCModel(model_folder=model_folder)
+
+
+def test_upscaling_qc_model_init(qcm_model):
+    assert set(qcm_model.selectors) == set(["ZONE", "FACIES"])
+    assert set(qcm_model.properties) == set(["PHIT", "KLOGH"])
+    assert set(qcm_model.get_unique_selector_values("ZONE")) == set(
+        ["Valysar", "Volon", "Therys"]
+    )
+    assert all(
+        isinstance(qcm_model._grid_df[col].dtype, CategoricalDtype)
+        for col in qcm_model.selectors
+    )
+
+
+def test_upscaling_qc_model_dataframe(qcm_model):
+    df = qcm_model.get_dataframe(
+        selectors=["ZONE"], selector_values=[["Volon"]], responses=["PHIT"]
+    )
+
+    assert (set(df.columns)) == set(["ZONE", "PHIT"])
+    assert df.shape[0] == 204017
+    assert df["PHIT"].mean() == pytest.approx(0.1794, abs=0.0001)
+
+
+def test_upscaling_qc_model_reduce_points(qcm_model):
+    df = qcm_model.get_dataframe(
+        selectors=["ZONE"],
+        selector_values=[["Volon"]],
+        responses=["PHIT"],
+        max_points=100000,
+    )
+    assert df.shape[0] == 103917
+    assert df["PHIT"].mean() == pytest.approx(0.18, abs=0.01)

webviz_subsurface/plugins/__init__.py

@@ -55,6 +55,7 @@
 from ._surface_with_grid_cross_section import SurfaceWithGridCrossSection
 from ._surface_with_seismic_cross_section import SurfaceWithSeismicCrossSection
 from ._tornado_plotter_fmu import TornadoPlotterFMU
+from ._upscaling_qc import UpscalingQC
 from ._volumetric_analysis import VolumetricAnalysis
 from ._well_completions import WellCompletions
 from ._well_cross_section import WellCrossSection

webviz_subsurface/plugins/_upscaling_qc/__init__.py

@@ -0,0 +1 @@
+from .upscaling_qc import UpscalingQC

webviz_subsurface/plugins/_upscaling_qc/callbacks/update_plot.py

@@ -0,0 +1,33 @@
+from typing import Callable
+
+from dash import Input, Output, State, ALL,callback
+from dash.exceptions import PreventUpdate
+import plotly.graph_objs as go
+
+from webviz_subsurface._figures import create_figure
+from ..models import UpscalingQCModel
+from ..layout.sidebar import PlotTypes
+
+def update_plot(get_uuid: Callable, qc_model: UpscalingQCModel) -> None:
+    @callback(
+        Output(get_uuid("plotly-graph"), "figure"),
+        Input(get_uuid("plot-type"), "value"),
+        Input(get_uuid("x"), "value"),
+        Input({"type": get_uuid("selector"), "value": ALL}, "value"),
+        State({"type": get_uuid("selector"), "value": ALL}, "id")
+    )
+    def _update_plotly_graph(plot_type, x_column, selector_values, selector_ids) -> go.Figure:
+        selectors = [id_obj["value"] for id_obj in selector_ids]
+        dframe = qc_model.get_dataframe(selectors=selectors, selector_values=selector_values, responses=[x_column], max_points=100000)
+        plot_type = PlotTypes(plot_type)
+        if plot_type == PlotTypes.HISTOGRAM:
+            return create_figure(
+            plot_type="histogram",
+            data_frame=dframe,
+            color="SOURCE",
+            x=x_column,
+
+        )
+
+        raise PreventUpdate
+

webviz_subsurface/plugins/_upscaling_qc/layout/__init__.py

@@ -0,0 +1,2 @@
+from .sidebar import sidebar
+from .main import main

webviz_subsurface/plugins/_upscaling_qc/layout/main.py

@@ -0,0 +1,8 @@
+from typing import Callable
+
+from dash import html
+import webviz_core_components as wcc
+
+
+def main(get_uuid: Callable) -> html.Div:
+    return html.Div(wcc.Graph(id=get_uuid("plotly-graph")))

webviz_subsurface/plugins/_upscaling_qc/layout/sidebar.py

@@ -0,0 +1,79 @@
+from enum import Enum
+from typing import Dict, List, Optional
+from typing import Callable
+
+from dash import html
+
+import webviz_core_components as wcc
+
+from ..models import UpscalingQCModel
+
+
+class PlotTypes(str, Enum):
+    HISTOGRAM = "Histogram"
+
+
+def sidebar(get_uuid: Callable, qc_model: UpscalingQCModel) -> html.Div:
+    selectors = [
+        _make_selector(
+            uuid={"type": get_uuid("selector"), "value": selector},
+            label=selector,
+            values=qc_model.get_unique_selector_values(selector),
+        )
+        for selector in qc_model.selectors
+    ]
+    return html.Div(
+        [
+            wcc.Selectors(
+                label="Plot:",
+                children=[
+                    wcc.Dropdown(
+                        id=get_uuid("plot-type"),
+                        options=[{"value": val, "label": val} for val in PlotTypes],
+                        value=PlotTypes.HISTOGRAM,
+                        label="Plot type",
+                        clearable=False
+                    ),
+                    wcc.Dropdown(
+                        id=get_uuid("x"),
+                        options=[
+                            {"value": val, "label": val} for val in qc_model.properties
+                        ],
+                        value=qc_model.properties[0],
+                        label="X",
+                        clearable=False
+                    ),
+                    wcc.Dropdown(
+                        id=get_uuid("group"),
+                        options=[
+                            {"value": val, "label": val} for val in qc_model.selectors
+                        ],
+                        value=None,
+                        label="Group by",
+                    ),
+                    wcc.Dropdown(
+                        id=get_uuid("color"),
+                        options=[
+                            {"value": val, "label": val} for val in qc_model.selectors
+                        ],
+                        value=None,
+                        label="Color",
+                    ),
+                ],
+            ),
+            html.Div(selectors),
+        ]
+    )
+
+
+def _make_selector(
+    uuid: Dict[str, str], label: str, values: List[str], default: Optional[str] = None
+) -> html.Div:
+    return wcc.SelectWithLabel(
+        id=uuid,
+        label=label,
+        options=[{"value": value, "label": value} for value in values],
+        value=default if default is not None else values,
+        multi=True,
+        size=min(len(values), 8),
+    )

webviz_subsurface/plugins/_upscaling_qc/models/__init__.py

@@ -0,0 +1 @@
+from .upscaling_qc_model import UpscalingQCModel

webviz_subsurface/plugins/_upscaling_qc/models/upscaling_qc_model.py

@@ -0,0 +1,114 @@
+from typing import List, Optional
+from pathlib import Path
+import json
+
+import pandas as pd
+
+from fmu.tools.rms.upscaling_qc._types import (
+    UpscalingQCFiles,
+    MetaData,
+)
+
+
+class UpscalingQCModel:
+    def __init__(self, model_folder: Path):
+        self._well_df = self._load_well_data(model_folder)
+        self._bw_df = self._load_bw_data(model_folder)
+        self._grid_df = self._load_grid_data(model_folder)
+        self._metadata = self._load_metadata(model_folder)
+        self._validate_input()
+        self._set_selectors_categorical()
+
+    def _load_well_data(self, model_folder: Path) -> pd.DataFrame:
+        return pd.read_csv(model_folder / UpscalingQCFiles.WELLS)
+
+    def _load_bw_data(self, model_folder: Path) -> pd.DataFrame:
+        return pd.read_csv(model_folder / UpscalingQCFiles.BLOCKEDWELLS)
+
+    def _load_grid_data(self, model_folder: Path) -> pd.DataFrame:
+        return pd.read_csv(model_folder / UpscalingQCFiles.GRID)
+
+    def _load_metadata(self, model_folder: Path) -> pd.DataFrame:
+        with open(model_folder / UpscalingQCFiles.METADATA, "r") as fp:
+            return MetaData(**json.load(fp))
+
+    def _validate_input(self) -> None:
+        """Check data for equality"""
+        columns = set(self.selectors + self.properties)
+        if set(self._well_df.columns) != columns:
+            raise KeyError("Well dataframe does not contain expected columns!")
+        if set(self._bw_df.columns) != columns:
+            raise KeyError("Blocked well dataframe does not contain expected columns!")
+        if set(self._grid_df.columns) != columns:
+            raise KeyError("Grid dataframe does not contain expected columns!")
+
+        for selector in self.selectors:
+            self._check_column_value_equality(selector)
+
+    def _check_column_value_equality(self, column) -> None:
+        if (
+            not set(self._well_df[column].unique())
+            == set(self._bw_df[column].unique())
+            == set(self._grid_df[column].unique())
+        ):
+            raise ValueError(
+                f"Data column {column} has different values in dataframes!"
+            )
+
+    def _set_selectors_categorical(self) -> None:
+        """Selector columns will have few unique values.
+        Set to categorical dtype for optimization"""
+
+        for selector in self.selectors:
+            self._well_df[selector] = self._well_df[selector].astype("category")
+            self._bw_df[selector] = self._bw_df[selector].astype("category")
+            self._grid_df[selector] = self._grid_df[selector].astype("category")
+
+    @property
+    def selectors(self) -> List[str]:
+        """Returns the selector column (discrete filters)"""
+        return self._metadata.selectors
+
+    @property
+    def properties(self) -> List[str]:
+        """Returns the property columns (values for plotting)"""
+        return self._metadata.properties
+
+    def get_unique_selector_values(self, selector: str) -> List[str]:
+        """Returns the unique values for a given selector.
+        Use the blocked well data for lookup as it has the smallest size"""
+        if selector not in self.selectors:
+            raise KeyError("{selector} is not a valid selector.")
+        return list(self._bw_df[selector].unique())
+
+    def get_dataframe(
+        self,
+        selectors: List[str],
+        selector_values: List[str],
+        responses: List[str],
+        max_points: Optional[int] = None,
+        drop_na: bool = True,
+    ) -> pd.DataFrame:
+        """Creates a dataframe from a subset of selectors and their value,
+        and a subset of responses. Optionally a max number of points can
+        be given. If any of the data sets have more points after filtering,
+        the dataset will be reduced by sampling a size of points equal
+        to this number."""
+        dfs = []
+        for source, df in zip(
+            ["Wells", "Blocked wells", "Grid"],
+            [self._well_df, self._bw_df, self._grid_df],
+        ):
+            df = df[selectors + responses]
+
+            for selector, value in zip(selectors, selector_values):
+                df = df[df[selector].isin(value)]
+            if max_points is not None and df.shape[0] > max_points:
+                df = df.sample(max_points)
+            df["SOURCE"] = source
+            dfs.append(df)
+
+        combined_df = pd.concat(dfs)
+        if drop_na:
+            combined_df = combined_df.dropna()
+        return combined_df

webviz_subsurface/plugins/_upscaling_qc/upscaling_qc.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+
+from dash import html
+from webviz_config import WebvizPluginABC, WebvizSettings
+from webviz_core_components import FlexBox, Frame
+from .models import UpscalingQCModel
+from .layout import sidebar, main
+from .callbacks.update_plot import update_plot
+
+class UpscalingQC(WebvizPluginABC):
+    def __init__(self, webviz_settings: WebvizSettings, model_folder: Path):
+        super().__init__()
+        self.qc_model = UpscalingQCModel(model_folder=model_folder)
+        self.set_callbacks()
+    @property
+    def layout(self) -> FlexBox:
+        return FlexBox(
+            [
+                Frame(
+                    style={"height": "90vh", "flex": 1},
+                    children=sidebar(get_uuid=self.uuid, qc_model=self.qc_model),
+                ),
+                Frame(style={"flex": 5}, children=main(get_uuid=self.uuid)),
+            ]
+        )
+    def set_callbacks(self):
+        update_plot(get_uuid=self.uuid, qc_model=self.qc_model)