From fb8d87979d9754d64377475861035380d5c0eb39 Mon Sep 17 00:00:00 2001 From: Hans Kallekleiv <16436291+HansKallekleiv@users.noreply.github.com> Date: Wed, 27 Oct 2021 10:27:07 +0200 Subject: [PATCH] Upscaling QC --- setup.py | 1 + .../model_tests/test_upscaling_qc_model.py | 47 ++++++++ webviz_subsurface/plugins/__init__.py | 1 + .../plugins/_upscaling_qc/__init__.py | 1 + .../_upscaling_qc/callbacks/__init__.py | 0 .../_upscaling_qc/callbacks/update_plot.py | 33 +++++ .../plugins/_upscaling_qc/layout/__init__.py | 2 + .../plugins/_upscaling_qc/layout/ids.py | 0 .../plugins/_upscaling_qc/layout/main.py | 8 ++ .../plugins/_upscaling_qc/layout/sidebar.py | 79 ++++++++++++ .../plugins/_upscaling_qc/models/__init__.py | 1 + .../models/upscaling_qc_model.py | 114 ++++++++++++++++++ .../plugins/_upscaling_qc/upscaling_qc.py | 27 +++++ 13 files changed, 314 insertions(+) create mode 100644 tests/unit_tests/model_tests/test_upscaling_qc_model.py create mode 100644 webviz_subsurface/plugins/_upscaling_qc/__init__.py create mode 100644 webviz_subsurface/plugins/_upscaling_qc/callbacks/__init__.py create mode 100644 webviz_subsurface/plugins/_upscaling_qc/callbacks/update_plot.py create mode 100644 webviz_subsurface/plugins/_upscaling_qc/layout/__init__.py create mode 100644 webviz_subsurface/plugins/_upscaling_qc/layout/ids.py create mode 100644 webviz_subsurface/plugins/_upscaling_qc/layout/main.py create mode 100644 webviz_subsurface/plugins/_upscaling_qc/layout/sidebar.py create mode 100644 webviz_subsurface/plugins/_upscaling_qc/models/__init__.py create mode 100644 webviz_subsurface/plugins/_upscaling_qc/models/upscaling_qc_model.py create mode 100644 webviz_subsurface/plugins/_upscaling_qc/upscaling_qc.py diff --git a/setup.py b/setup.py index ea81ca3b6..0945e28b0 100644 --- a/setup.py +++ b/setup.py @@ -69,6 +69,7 @@ "SurfaceWithSeismicCrossSection = webviz_subsurface.plugins:SurfaceWithSeismicCrossSection", "TornadoPlotterFMU = webviz_subsurface.plugins:TornadoPlotterFMU", "VolumetricAnalysis = webviz_subsurface.plugins:VolumetricAnalysis", + "UpscalingQC = webviz_subsurface.plugins:UpscalingQC", "WellCrossSection = webviz_subsurface.plugins:WellCrossSection", "WellCrossSectionFMU = webviz_subsurface.plugins:WellCrossSectionFMU", "AssistedHistoryMatchingAnalysis = webviz_subsurface.plugins:AssistedHistoryMatchingAnalysis", diff --git a/tests/unit_tests/model_tests/test_upscaling_qc_model.py b/tests/unit_tests/model_tests/test_upscaling_qc_model.py new file mode 100644 index 000000000..3d8e4bfc2 --- /dev/null +++ b/tests/unit_tests/model_tests/test_upscaling_qc_model.py @@ -0,0 +1,47 @@ +from pathlib import Path + +import pytest + +from pandas.api.types import CategoricalDtype +from webviz_subsurface.plugins._upscaling_qc.models.upscaling_qc_model import ( + UpscalingQCModel, +) + + +@pytest.fixture +def qcm_model() -> Path: + model_folder = Path("../upscaling_qc") + yield UpscalingQCModel(model_folder=model_folder) + + +def test_upscaling_qc_model_init(qcm_model): + assert set(qcm_model.selectors) == set(["ZONE", "FACIES"]) + assert set(qcm_model.properties) == set(["PHIT", "KLOGH"]) + assert set(qcm_model.get_unique_selector_values("ZONE")) == set( + ["Valysar", "Volon", "Therys"] + ) + assert all( + isinstance(qcm_model._grid_df[col].dtype, CategoricalDtype) + for col in qcm_model.selectors + ) + + +def test_upscaling_qc_model_dataframe(qcm_model): + df = qcm_model.get_dataframe( + selectors=["ZONE"], selector_values=[["Volon"]], responses=["PHIT"] + ) + + assert (set(df.columns)) == set(["ZONE", "PHIT"]) + assert df.shape[0] == 204017 + assert df["PHIT"].mean() == pytest.approx(0.1794, abs=0.0001) + + +def test_upscaling_qc_model_reduce_points(qcm_model): + df = qcm_model.get_dataframe( + selectors=["ZONE"], + selector_values=[["Volon"]], + responses=["PHIT"], + max_points=100000, + ) + assert df.shape[0] == 103917 + assert df["PHIT"].mean() == pytest.approx(0.18, abs=0.01) diff --git a/webviz_subsurface/plugins/__init__.py b/webviz_subsurface/plugins/__init__.py index 8b81491f4..dfe76b798 100644 --- a/webviz_subsurface/plugins/__init__.py +++ b/webviz_subsurface/plugins/__init__.py @@ -55,6 +55,7 @@ from ._surface_with_grid_cross_section import SurfaceWithGridCrossSection from ._surface_with_seismic_cross_section import SurfaceWithSeismicCrossSection from ._tornado_plotter_fmu import TornadoPlotterFMU +from ._upscaling_qc import UpscalingQC from ._volumetric_analysis import VolumetricAnalysis from ._well_completions import WellCompletions from ._well_cross_section import WellCrossSection diff --git a/webviz_subsurface/plugins/_upscaling_qc/__init__.py b/webviz_subsurface/plugins/_upscaling_qc/__init__.py new file mode 100644 index 000000000..6f365d082 --- /dev/null +++ b/webviz_subsurface/plugins/_upscaling_qc/__init__.py @@ -0,0 +1 @@ +from .upscaling_qc import UpscalingQC diff --git a/webviz_subsurface/plugins/_upscaling_qc/callbacks/__init__.py b/webviz_subsurface/plugins/_upscaling_qc/callbacks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/webviz_subsurface/plugins/_upscaling_qc/callbacks/update_plot.py b/webviz_subsurface/plugins/_upscaling_qc/callbacks/update_plot.py new file mode 100644 index 000000000..e3f8e9753 --- /dev/null +++ b/webviz_subsurface/plugins/_upscaling_qc/callbacks/update_plot.py @@ -0,0 +1,33 @@ +from typing import Callable + +from dash import Input, Output, State, ALL,callback +from dash.exceptions import PreventUpdate +import plotly.graph_objs as go + +from webviz_subsurface._figures import create_figure +from ..models import UpscalingQCModel +from ..layout.sidebar import PlotTypes + +def update_plot(get_uuid: Callable, qc_model: UpscalingQCModel) -> None: + @callback( + Output(get_uuid("plotly-graph"), "figure"), + Input(get_uuid("plot-type"), "value"), + Input(get_uuid("x"), "value"), + Input({"type": get_uuid("selector"), "value": ALL}, "value"), + State({"type": get_uuid("selector"), "value": ALL}, "id") + ) + def _update_plotly_graph(plot_type, x_column, selector_values, selector_ids) -> go.Figure: + selectors = [id_obj["value"] for id_obj in selector_ids] + dframe = qc_model.get_dataframe(selectors=selectors, selector_values=selector_values, responses=[x_column], max_points=100000) + plot_type = PlotTypes(plot_type) + if plot_type == PlotTypes.HISTOGRAM: + return create_figure( + plot_type="histogram", + data_frame=dframe, + color="SOURCE", + x=x_column, + + ) + + raise PreventUpdate + diff --git a/webviz_subsurface/plugins/_upscaling_qc/layout/__init__.py b/webviz_subsurface/plugins/_upscaling_qc/layout/__init__.py new file mode 100644 index 000000000..eac6e8264 --- /dev/null +++ b/webviz_subsurface/plugins/_upscaling_qc/layout/__init__.py @@ -0,0 +1,2 @@ +from .sidebar import sidebar +from .main import main diff --git a/webviz_subsurface/plugins/_upscaling_qc/layout/ids.py b/webviz_subsurface/plugins/_upscaling_qc/layout/ids.py new file mode 100644 index 000000000..e69de29bb diff --git a/webviz_subsurface/plugins/_upscaling_qc/layout/main.py b/webviz_subsurface/plugins/_upscaling_qc/layout/main.py new file mode 100644 index 000000000..c816d966a --- /dev/null +++ b/webviz_subsurface/plugins/_upscaling_qc/layout/main.py @@ -0,0 +1,8 @@ +from typing import Callable + +from dash import html +import webviz_core_components as wcc + + +def main(get_uuid: Callable) -> html.Div: + return html.Div(wcc.Graph(id=get_uuid("plotly-graph"))) diff --git a/webviz_subsurface/plugins/_upscaling_qc/layout/sidebar.py b/webviz_subsurface/plugins/_upscaling_qc/layout/sidebar.py new file mode 100644 index 000000000..9b8317178 --- /dev/null +++ b/webviz_subsurface/plugins/_upscaling_qc/layout/sidebar.py @@ -0,0 +1,79 @@ +from enum import Enum +from typing import Dict, List, Optional +from typing import Callable + +from dash import html + +import webviz_core_components as wcc + +from ..models import UpscalingQCModel + + +class PlotTypes(str, Enum): + HISTOGRAM = "Histogram" + + +def sidebar(get_uuid: Callable, qc_model: UpscalingQCModel) -> html.Div: + selectors = [ + _make_selector( + uuid={"type": get_uuid("selector"), "value": selector}, + label=selector, + values=qc_model.get_unique_selector_values(selector), + ) + for selector in qc_model.selectors + ] + return html.Div( + [ + wcc.Selectors( + label="Plot:", + children=[ + wcc.Dropdown( + id=get_uuid("plot-type"), + options=[{"value": val, "label": val} for val in PlotTypes], + value=PlotTypes.HISTOGRAM, + label="Plot type", + clearable=False + ), + wcc.Dropdown( + id=get_uuid("x"), + options=[ + {"value": val, "label": val} for val in qc_model.properties + ], + value=qc_model.properties[0], + label="X", + clearable=False + ), + wcc.Dropdown( + id=get_uuid("group"), + options=[ + {"value": val, "label": val} for val in qc_model.selectors + ], + value=None, + label="Group by", + ), + wcc.Dropdown( + id=get_uuid("color"), + options=[ + {"value": val, "label": val} for val in qc_model.selectors + ], + value=None, + label="Color", + ), + ], + ), + html.Div(selectors), + ] + ) + + +def _make_selector( + uuid: Dict[str, str], label: str, values: List[str], default: Optional[str] = None +) -> html.Div: + return wcc.SelectWithLabel( + id=uuid, + label=label, + options=[{"value": value, "label": value} for value in values], + value=default if default is not None else values, + multi=True, + size=min(len(values), 8), + ) diff --git a/webviz_subsurface/plugins/_upscaling_qc/models/__init__.py b/webviz_subsurface/plugins/_upscaling_qc/models/__init__.py new file mode 100644 index 000000000..b062503e7 --- /dev/null +++ b/webviz_subsurface/plugins/_upscaling_qc/models/__init__.py @@ -0,0 +1 @@ +from .upscaling_qc_model import UpscalingQCModel diff --git a/webviz_subsurface/plugins/_upscaling_qc/models/upscaling_qc_model.py b/webviz_subsurface/plugins/_upscaling_qc/models/upscaling_qc_model.py new file mode 100644 index 000000000..8ab2d5f3a --- /dev/null +++ b/webviz_subsurface/plugins/_upscaling_qc/models/upscaling_qc_model.py @@ -0,0 +1,114 @@ +from typing import List, Optional +from pathlib import Path +import json + +import pandas as pd + +from fmu.tools.rms.upscaling_qc._types import ( + UpscalingQCFiles, + MetaData, +) + + +class UpscalingQCModel: + def __init__(self, model_folder: Path): + self._well_df = self._load_well_data(model_folder) + self._bw_df = self._load_bw_data(model_folder) + self._grid_df = self._load_grid_data(model_folder) + self._metadata = self._load_metadata(model_folder) + self._validate_input() + self._set_selectors_categorical() + + def _load_well_data(self, model_folder: Path) -> pd.DataFrame: + return pd.read_csv(model_folder / UpscalingQCFiles.WELLS) + + def _load_bw_data(self, model_folder: Path) -> pd.DataFrame: + return pd.read_csv(model_folder / UpscalingQCFiles.BLOCKEDWELLS) + + def _load_grid_data(self, model_folder: Path) -> pd.DataFrame: + return pd.read_csv(model_folder / UpscalingQCFiles.GRID) + + def _load_metadata(self, model_folder: Path) -> pd.DataFrame: + with open(model_folder / UpscalingQCFiles.METADATA, "r") as fp: + return MetaData(**json.load(fp)) + + def _validate_input(self) -> None: + """Check data for equality""" + columns = set(self.selectors + self.properties) + if set(self._well_df.columns) != columns: + raise KeyError("Well dataframe does not contain expected columns!") + if set(self._bw_df.columns) != columns: + raise KeyError("Blocked well dataframe does not contain expected columns!") + if set(self._grid_df.columns) != columns: + raise KeyError("Grid dataframe does not contain expected columns!") + + for selector in self.selectors: + self._check_column_value_equality(selector) + + def _check_column_value_equality(self, column) -> None: + if ( + not set(self._well_df[column].unique()) + == set(self._bw_df[column].unique()) + == set(self._grid_df[column].unique()) + ): + raise ValueError( + f"Data column {column} has different values in dataframes!" + ) + + def _set_selectors_categorical(self) -> None: + """Selector columns will have few unique values. + Set to categorical dtype for optimization""" + + for selector in self.selectors: + self._well_df[selector] = self._well_df[selector].astype("category") + self._bw_df[selector] = self._bw_df[selector].astype("category") + self._grid_df[selector] = self._grid_df[selector].astype("category") + + @property + def selectors(self) -> List[str]: + """Returns the selector column (discrete filters)""" + return self._metadata.selectors + + @property + def properties(self) -> List[str]: + """Returns the property columns (values for plotting)""" + return self._metadata.properties + + def get_unique_selector_values(self, selector: str) -> List[str]: + """Returns the unique values for a given selector. + Use the blocked well data for lookup as it has the smallest size""" + if selector not in self.selectors: + raise KeyError("{selector} is not a valid selector.") + return list(self._bw_df[selector].unique()) + + def get_dataframe( + self, + selectors: List[str], + selector_values: List[str], + responses: List[str], + max_points: Optional[int] = None, + drop_na: bool = True, + ) -> pd.DataFrame: + """Creates a dataframe from a subset of selectors and their value, + and a subset of responses. Optionally a max number of points can + be given. If any of the data sets have more points after filtering, + the dataset will be reduced by sampling a size of points equal + to this number.""" + dfs = [] + for source, df in zip( + ["Wells", "Blocked wells", "Grid"], + [self._well_df, self._bw_df, self._grid_df], + ): + df = df[selectors + responses] + + for selector, value in zip(selectors, selector_values): + df = df[df[selector].isin(value)] + if max_points is not None and df.shape[0] > max_points: + df = df.sample(max_points) + df["SOURCE"] = source + dfs.append(df) + + combined_df = pd.concat(dfs) + if drop_na: + combined_df = combined_df.dropna() + return combined_df diff --git a/webviz_subsurface/plugins/_upscaling_qc/upscaling_qc.py b/webviz_subsurface/plugins/_upscaling_qc/upscaling_qc.py new file mode 100644 index 000000000..4d0ec0a98 --- /dev/null +++ b/webviz_subsurface/plugins/_upscaling_qc/upscaling_qc.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from dash import html +from webviz_config import WebvizPluginABC, WebvizSettings +from webviz_core_components import FlexBox, Frame +from .models import UpscalingQCModel +from .layout import sidebar, main +from .callbacks.update_plot import update_plot + +class UpscalingQC(WebvizPluginABC): + def __init__(self, webviz_settings: WebvizSettings, model_folder: Path): + super().__init__() + self.qc_model = UpscalingQCModel(model_folder=model_folder) + self.set_callbacks() + @property + def layout(self) -> FlexBox: + return FlexBox( + [ + Frame( + style={"height": "90vh", "flex": 1}, + children=sidebar(get_uuid=self.uuid, qc_model=self.qc_model), + ), + Frame(style={"flex": 5}, children=main(get_uuid=self.uuid)), + ] + ) + def set_callbacks(self): + update_plot(get_uuid=self.uuid, qc_model=self.qc_model) \ No newline at end of file