From 5fc4dcaac20820fb42c1ebd5ed6acf5f8456fa47 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 12:32:18 +0200 Subject: [PATCH 01/48] Add new config options for plotting --- flixopt/config.py | 74 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/flixopt/config.py b/flixopt/config.py index a7549a3ec..9681dc8b6 100644 --- a/flixopt/config.py +++ b/flixopt/config.py @@ -54,6 +54,17 @@ 'big_binary_bound': 100_000, } ), + 'plotting': MappingProxyType( + { + 'default_show': True, + 'default_save_path': None, + 'default_engine': 'plotly', + 'default_dpi': 300, + 'default_facet_cols': 3, + 'default_sequential_colorscale': 'turbo', + 'default_qualitative_colorscale': 'plotly', + } + ), } ) @@ -185,6 +196,61 @@ class Modeling: epsilon: float = _DEFAULTS['modeling']['epsilon'] big_binary_bound: int = _DEFAULTS['modeling']['big_binary_bound'] + class Plotting: + """Plotting configuration. + + Configure backends via environment variables: + - Matplotlib: Set `MPLBACKEND` environment variable (e.g., 'Agg', 'TkAgg') + - Plotly: Set `PLOTLY_RENDERER` or use `plotly.io.renderers.default` + + Attributes: + default_show: Default value for the `show` parameter in plot methods. + default_engine: Default plotting engine. + default_dpi: Default DPI for saved plots. + default_facet_cols: Default number of columns for faceted plots. + default_sequential_colorscale: Default colorscale for heatmaps and continuous data. + default_qualitative_colorscale: Default colormap for categorical plots (bar/line/area charts). + + Examples: + ```python + # Set consistent theming + CONFIG.Plotting.plotly_template = 'plotly_dark' + CONFIG.apply() + + # Configure default export and color settings + CONFIG.Plotting.default_dpi = 600 + CONFIG.Plotting.default_sequential_colorscale = 'plasma' + CONFIG.Plotting.default_qualitative_colorscale = 'Dark24' + CONFIG.apply() + ``` + """ + + plotly_template: ( + Literal[ + 'plotly', + 'plotly_white', + 'plotly_dark', + 'ggplot2', + 'seaborn', + 'simple_white', + 'none', + 'gridon', + 'presentation', + 'xgridoff', + 'ygridoff', + ] + | None + ) = _DEFAULTS['plotting']['plotly_template'] + default_show: bool = _DEFAULTS['plotting']['default_show'] + default_save_path: str | None = _DEFAULTS['plotting']['default_save_path'] + default_engine: Literal['plotly', 'matplotlib'] = _DEFAULTS['plotting']['default_engine'] + default_dpi: int = _DEFAULTS['plotting']['default_dpi'] + default_figure_width: int | None = _DEFAULTS['plotting']['default_figure_width'] + default_figure_height: int | None = _DEFAULTS['plotting']['default_figure_height'] + default_facet_cols: int = _DEFAULTS['plotting']['default_facet_cols'] + default_sequential_colorscale: str = _DEFAULTS['plotting']['default_sequential_colorscale'] + default_qualitative_colorscale: str = _DEFAULTS['plotting']['default_qualitative_colorscale'] + config_name: str = _DEFAULTS['config_name'] @classmethod @@ -319,6 +385,14 @@ def to_dict(cls) -> dict: 'epsilon': cls.Modeling.epsilon, 'big_binary_bound': cls.Modeling.big_binary_bound, }, + 'plotting': { + 'default_show': cls.Plotting.default_show, + 'default_engine': cls.Plotting.default_engine, + 'default_dpi': cls.Plotting.default_dpi, + 'default_facet_cols': cls.Plotting.default_facet_cols, + 'default_sequential_colorscale': cls.Plotting.default_sequential_colorscale, + 'default_qualitative_colorscale': cls.Plotting.default_qualitative_colorscale, + }, } From 3edcf40f06544fb97bd8b5f03a176c426471cf47 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 12:33:34 +0200 Subject: [PATCH 02/48] Use turbo instead of viridis --- flixopt/aggregation.py | 2 +- flixopt/plotting.py | 32 ++++++++++++++++---------------- flixopt/results.py | 12 ++++++------ tests/test_results_plots.py | 4 ++-- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/flixopt/aggregation.py b/flixopt/aggregation.py index 53770e140..427937596 100644 --- a/flixopt/aggregation.py +++ b/flixopt/aggregation.py @@ -141,7 +141,7 @@ def describe_clusters(self) -> str: def use_extreme_periods(self): return self.time_series_for_high_peaks or self.time_series_for_low_peaks - def plot(self, colormap: str = 'viridis', show: bool = True, save: pathlib.Path | None = None) -> go.Figure: + def plot(self, colormap: str = 'turbo', show: bool = True, save: pathlib.Path | None = None) -> go.Figure: from . import plotting df_org = self.original_data.copy().rename( diff --git a/flixopt/plotting.py b/flixopt/plotting.py index a024c97fc..544dc12ac 100644 --- a/flixopt/plotting.py +++ b/flixopt/plotting.py @@ -74,7 +74,7 @@ Color specifications can take several forms to accommodate different use cases: **Named Colormaps** (str): - - Standard colormaps: 'viridis', 'plasma', 'cividis', 'tab10', 'Set1' + - Standard colormaps: 'turbo', 'plasma', 'cividis', 'tab10', 'Set1' - Energy-focused: 'portland' (custom flixopt colormap for energy systems) - Backend-specific maps available in Plotly and Matplotlib @@ -91,7 +91,7 @@ Examples: ```python # Named colormap - colors = 'viridis' # Automatic color generation + colors = 'turbo' # Automatic color generation # Explicit color list colors = ['red', 'blue', 'green', '#FFD700'] @@ -138,7 +138,7 @@ class ColorProcessor: **Energy System Colors**: Built-in palettes optimized for energy system visualization Color Input Types: - - **Named Colormaps**: 'viridis', 'plasma', 'portland', 'tab10', etc. + - **Named Colormaps**: 'turbo', 'plasma', 'portland', 'tab10', etc. - **Color Lists**: ['red', 'blue', 'green'] or ['#FF0000', '#0000FF', '#00FF00'] - **Label Dictionaries**: {'Generator': 'red', 'Storage': 'blue', 'Load': 'green'} @@ -147,7 +147,7 @@ class ColorProcessor: ```python # Initialize for Plotly backend - processor = ColorProcessor(engine='plotly', default_colormap='viridis') + processor = ColorProcessor(engine='plotly', default_colormap='turbo') # Process different color specifications colors = processor.process_colors('plasma', ['Gen1', 'Gen2', 'Storage']) @@ -179,11 +179,11 @@ class ColorProcessor: Args: engine: Plotting backend ('plotly' or 'matplotlib'). Determines output color format. default_colormap: Fallback colormap when requested palettes are unavailable. - Common options: 'viridis', 'plasma', 'tab10', 'portland'. + Common options: 'turbo', 'plasma', 'tab10', 'portland'. """ - def __init__(self, engine: PlottingEngine = 'plotly', default_colormap: str = 'viridis'): + def __init__(self, engine: PlottingEngine = 'plotly', default_colormap: str = 'turbo'): """Initialize the color processor with specified backend and defaults.""" if engine not in ['plotly', 'matplotlib']: raise TypeError(f'engine must be "plotly" or "matplotlib", but is {engine}') @@ -397,7 +397,7 @@ def resolve_colors( def with_plotly( data: xr.Dataset | pd.DataFrame | pd.Series, mode: Literal['stacked_bar', 'line', 'area', 'grouped_bar'] = 'stacked_bar', - colors: ColorType = 'viridis', + colors: ColorType = 'turbo', title: str = '', ylabel: str = '', xlabel: str = '', @@ -705,7 +705,7 @@ def with_plotly( def with_matplotlib( data: xr.Dataset | pd.DataFrame | pd.Series, mode: Literal['stacked_bar', 'line'] = 'stacked_bar', - colors: ColorType = 'viridis', + colors: ColorType = 'turbo', title: str = '', ylabel: str = '', xlabel: str = 'Time in h', @@ -1199,7 +1199,7 @@ def preprocess_data_for_pie( def dual_pie_with_plotly( data_left: xr.Dataset | pd.DataFrame | pd.Series, data_right: xr.Dataset | pd.DataFrame | pd.Series, - colors: ColorType = 'viridis', + colors: ColorType = 'turbo', title: str = '', subtitles: tuple[str, str] = ('Left Chart', 'Right Chart'), legend_title: str = '', @@ -1294,7 +1294,7 @@ def dual_pie_with_plotly( def dual_pie_with_matplotlib( data_left: xr.Dataset | pd.DataFrame | pd.Series, data_right: xr.Dataset | pd.DataFrame | pd.Series, - colors: ColorType = 'viridis', + colors: ColorType = 'turbo', title: str = '', subtitles: tuple[str, str] = ('Left Chart', 'Right Chart'), legend_title: str = '', @@ -1400,7 +1400,7 @@ def draw_pie(ax, labels, values, subtitle): def heatmap_with_plotly( data: xr.DataArray, - colors: ColorType = 'viridis', + colors: ColorType = 'turbo', title: str = '', facet_by: str | list[str] | None = None, animate_by: str | None = None, @@ -1428,7 +1428,7 @@ def heatmap_with_plotly( data: An xarray DataArray containing the data to visualize. Should have at least 2 dimensions, or a 'time' dimension that can be reshaped into 2D. colors: Color specification (colormap name, list, or dict). Common options: - 'viridis', 'plasma', 'RdBu', 'portland'. + 'turbo', 'plasma', 'RdBu', 'portland'. title: The main title of the heatmap. facet_by: Dimension to create facets for. Creates a subplot grid. Can be a single dimension name or list (only first dimension used). @@ -1577,7 +1577,7 @@ def heatmap_with_plotly( # Create the imshow plot - px.imshow can work directly with xarray DataArrays common_args = { 'img': data, - 'color_continuous_scale': colors if isinstance(colors, str) else 'viridis', + 'color_continuous_scale': colors if isinstance(colors, str) else 'turbo', 'title': title, } @@ -1601,7 +1601,7 @@ def heatmap_with_plotly( # Fallback: create a simple heatmap without faceting fallback_args = { 'img': data.values, - 'color_continuous_scale': colors if isinstance(colors, str) else 'viridis', + 'color_continuous_scale': colors if isinstance(colors, str) else 'turbo', 'title': title, } fallback_args.update(imshow_kwargs) @@ -1612,7 +1612,7 @@ def heatmap_with_plotly( def heatmap_with_matplotlib( data: xr.DataArray, - colors: ColorType = 'viridis', + colors: ColorType = 'turbo', title: str = '', figsize: tuple[float, float] = (12, 6), reshape_time: tuple[Literal['YS', 'MS', 'W', 'D', 'h', '15min', 'min'], Literal['W', 'D', 'h', '15min', 'min']] @@ -1727,7 +1727,7 @@ def heatmap_with_matplotlib( y_labels = 'y' # Process colormap - cmap = colors if isinstance(colors, str) else 'viridis' + cmap = colors if isinstance(colors, str) else 'turbo' # Create the heatmap using imshow with user customizations imshow_defaults = {'cmap': cmap, 'aspect': 'auto', 'origin': 'upper', 'vmin': vmin, 'vmax': vmax} diff --git a/flixopt/results.py b/flixopt/results.py index 576ff9ec1..81bc812f2 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -720,7 +720,7 @@ def plot_heatmap( variable_name: str | list[str], save: bool | pathlib.Path = False, show: bool = True, - colors: plotting.ColorType = 'viridis', + colors: plotting.ColorType = 'turbo', engine: plotting.PlottingEngine = 'plotly', select: dict[FlowSystemDimensions, Any] | None = None, facet_by: str | list[str] | None = 'scenario', @@ -1025,7 +1025,7 @@ def plot_node_balance( self, save: bool | pathlib.Path = False, show: bool = True, - colors: plotting.ColorType = 'viridis', + colors: plotting.ColorType = 'turbo', engine: plotting.PlottingEngine = 'plotly', select: dict[FlowSystemDimensions, Any] | None = None, unit_type: Literal['flow_rate', 'flow_hours'] = 'flow_rate', @@ -1214,7 +1214,7 @@ def plot_node_balance( def plot_node_balance_pie( self, lower_percentage_group: float = 5, - colors: plotting.ColorType = 'viridis', + colors: plotting.ColorType = 'turbo', text_info: str = 'percent+label+value', save: bool | pathlib.Path = False, show: bool = True, @@ -1481,7 +1481,7 @@ def plot_charge_state( self, save: bool | pathlib.Path = False, show: bool = True, - colors: plotting.ColorType = 'viridis', + colors: plotting.ColorType = 'turbo', engine: plotting.PlottingEngine = 'plotly', mode: Literal['area', 'stacked_bar', 'line'] = 'area', select: dict[FlowSystemDimensions, Any] | None = None, @@ -2099,7 +2099,7 @@ def plot_heatmap( data: xr.DataArray | xr.Dataset, name: str | None = None, folder: pathlib.Path | None = None, - colors: plotting.ColorType = 'viridis', + colors: plotting.ColorType = 'turbo', save: bool | pathlib.Path = False, show: bool = True, engine: plotting.PlottingEngine = 'plotly', @@ -2188,7 +2188,7 @@ def plot_heatmap( # Handle deprecated color_map parameter if color_map is not None: # Check for conflict with new parameter - if colors != 'viridis': # User explicitly set colors + if colors != 'turbo': # User explicitly set colors raise ValueError( "Cannot use both deprecated parameter 'color_map' and new parameter 'colors'. Use only 'colors'." ) diff --git a/tests/test_results_plots.py b/tests/test_results_plots.py index 1fd6cf7f5..a656f7c44 100644 --- a/tests/test_results_plots.py +++ b/tests/test_results_plots.py @@ -28,7 +28,7 @@ def plotting_engine(request): @pytest.fixture( params=[ - 'viridis', # Test string colormap + 'turbo', # Test string colormap ['#ff0000', '#00ff00', '#0000ff', '#ffff00', '#ff00ff', '#00ffff'], # Test color list { 'Boiler(Q_th)|flow_rate': '#ff0000', @@ -51,7 +51,7 @@ def test_results_plots(flow_system, plotting_engine, show, save, color_spec): # Matplotlib doesn't support faceting/animation, so disable them for matplotlib engine heatmap_kwargs = { 'reshape_time': ('D', 'h'), - 'colors': 'viridis', # Note: heatmap only accepts string colormap + 'colors': 'turbo', # Note: heatmap only accepts string colormap 'save': save, 'show': show, 'engine': plotting_engine, From 9ce213c7cf059cfa86e2d987a55dbdad9794bb14 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 12:49:53 +0200 Subject: [PATCH 03/48] Update plotting.py to use updated color management --- flixopt/config.py | 19 -- flixopt/plotting.py | 507 +++++++++++++++++++++++++++++++++----------- 2 files changed, 382 insertions(+), 144 deletions(-) diff --git a/flixopt/config.py b/flixopt/config.py index 9681dc8b6..89b9abd78 100644 --- a/flixopt/config.py +++ b/flixopt/config.py @@ -225,28 +225,9 @@ class Plotting: ``` """ - plotly_template: ( - Literal[ - 'plotly', - 'plotly_white', - 'plotly_dark', - 'ggplot2', - 'seaborn', - 'simple_white', - 'none', - 'gridon', - 'presentation', - 'xgridoff', - 'ygridoff', - ] - | None - ) = _DEFAULTS['plotting']['plotly_template'] default_show: bool = _DEFAULTS['plotting']['default_show'] - default_save_path: str | None = _DEFAULTS['plotting']['default_save_path'] default_engine: Literal['plotly', 'matplotlib'] = _DEFAULTS['plotting']['default_engine'] default_dpi: int = _DEFAULTS['plotting']['default_dpi'] - default_figure_width: int | None = _DEFAULTS['plotting']['default_figure_width'] - default_figure_height: int | None = _DEFAULTS['plotting']['default_figure_height'] default_facet_cols: int = _DEFAULTS['plotting']['default_facet_cols'] default_sequential_colorscale: str = _DEFAULTS['plotting']['default_sequential_colorscale'] default_qualitative_colorscale: str = _DEFAULTS['plotting']['default_qualitative_colorscale'] diff --git a/flixopt/plotting.py b/flixopt/plotting.py index 544dc12ac..3a11b24f6 100644 --- a/flixopt/plotting.py +++ b/flixopt/plotting.py @@ -49,7 +49,7 @@ logger = logging.getLogger('flixopt') -# Define the colors for the 'portland' colormap in matplotlib +# Define the colors for the 'portland' colorscale in matplotlib _portland_colors = [ [12 / 255, 51 / 255, 131 / 255], # Dark blue [10 / 255, 136 / 255, 186 / 255], # Light blue @@ -58,7 +58,7 @@ [217 / 255, 30 / 255, 30 / 255], # Red ] -# Check if the colormap already exists before registering it +# Check if the colorscale already exists before registering it if hasattr(plt, 'colormaps'): # Matplotlib >= 3.7 registry = plt.colormaps if 'portland' not in registry: @@ -73,9 +73,9 @@ Color specifications can take several forms to accommodate different use cases: -**Named Colormaps** (str): - - Standard colormaps: 'turbo', 'plasma', 'cividis', 'tab10', 'Set1' - - Energy-focused: 'portland' (custom flixopt colormap for energy systems) +**Named colorscales** (str): + - Standard colorscales: 'turbo', 'plasma', 'cividis', 'tab10', 'Set1' + - Energy-focused: 'portland' (custom flixopt colorscale for energy systems) - Backend-specific maps available in Plotly and Matplotlib **Color Lists** (list[str]): @@ -90,7 +90,7 @@ Examples: ```python - # Named colormap + # Named colorscale colors = 'turbo' # Automatic color generation # Explicit color list @@ -114,7 +114,7 @@ References: - HTML Color Names: https://htmlcolorcodes.com/color-names/ - - Matplotlib Colormaps: https://matplotlib.org/stable/tutorials/colors/colormaps.html + - Matplotlib colorscales: https://matplotlib.org/stable/tutorials/colors/colorscales.html - Plotly Built-in Colorscales: https://plotly.com/python/builtin-colorscales/ """ @@ -127,97 +127,72 @@ class ColorProcessor: This class provides unified color processing across Plotly and Matplotlib backends, ensuring consistent visual appearance regardless of the plotting engine used. - It handles color palette generation, named colormap translation, and intelligent + It handles color palette generation, named colorscale translation, and intelligent color cycling for complex datasets with many categories. Key Features: **Backend Agnostic**: Automatic color format conversion between engines - **Palette Management**: Support for named colormaps, custom palettes, and color lists + **Palette Management**: Support for named colorscales, custom palettes, and color lists **Intelligent Cycling**: Smart color assignment for datasets with many categories - **Fallback Handling**: Graceful degradation when requested colormaps are unavailable + **Fallback Handling**: Graceful degradation when requested colorscales are unavailable **Energy System Colors**: Built-in palettes optimized for energy system visualization Color Input Types: - - **Named Colormaps**: 'turbo', 'plasma', 'portland', 'tab10', etc. + - **Named colorscales**: 'turbo', 'plasma', 'portland', etc. - **Color Lists**: ['red', 'blue', 'green'] or ['#FF0000', '#0000FF', '#00FF00'] - **Label Dictionaries**: {'Generator': 'red', 'Storage': 'blue', 'Load': 'green'} - Examples: - Basic color processing: - + Example: ```python - # Initialize for Plotly backend - processor = ColorProcessor(engine='plotly', default_colormap='turbo') - - # Process different color specifications + processor = ColorProcessor(engine='plotly', default_colorscale='turbo') colors = processor.process_colors('plasma', ['Gen1', 'Gen2', 'Storage']) - colors = processor.process_colors(['red', 'blue', 'green'], ['A', 'B', 'C']) - colors = processor.process_colors({'Wind': 'skyblue', 'Solar': 'gold'}, ['Wind', 'Solar', 'Gas']) - - # Switch to Matplotlib - processor = ColorProcessor(engine='matplotlib') - mpl_colors = processor.process_colors('tab10', component_labels) - ``` - - Energy system visualization: - - ```python - # Specialized energy system palette - energy_colors = { - 'Natural_Gas': '#8B4513', # Brown - 'Electricity': '#FFD700', # Gold - 'Heat': '#FF4500', # Red-orange - 'Cooling': '#87CEEB', # Sky blue - 'Hydrogen': '#E6E6FA', # Lavender - 'Battery': '#32CD32', # Lime green - } - - processor = ColorProcessor('plotly') - flow_colors = processor.process_colors(energy_colors, flow_labels) ``` Args: engine: Plotting backend ('plotly' or 'matplotlib'). Determines output color format. - default_colormap: Fallback colormap when requested palettes are unavailable. - Common options: 'turbo', 'plasma', 'tab10', 'portland'. + default_colorscale: Fallback colorscale when requested palettes are unavailable. + Common options: 'turbo', 'plasma', 'portland'. """ - def __init__(self, engine: PlottingEngine = 'plotly', default_colormap: str = 'turbo'): + def __init__(self, engine: PlottingEngine = 'plotly', default_colorscale: str | None = None): """Initialize the color processor with specified backend and defaults.""" if engine not in ['plotly', 'matplotlib']: raise TypeError(f'engine must be "plotly" or "matplotlib", but is {engine}') self.engine = engine - self.default_colormap = default_colormap + self.default_colorscale = ( + default_colorscale if default_colorscale is not None else CONFIG.Plotting.default_qualitative_colorscale + ) - def _generate_colors_from_colormap(self, colormap_name: str, num_colors: int) -> list[Any]: + def _generate_colors_from_colorscale(self, colorscale_name: str, num_colors: int) -> list[Any]: """ - Generate colors from a named colormap. + Generate colors from a named colorscale. Args: - colormap_name: Name of the colormap + colorscale_name: Name of the colorscale num_colors: Number of colors to generate Returns: list of colors in the format appropriate for the engine """ if self.engine == 'plotly': - try: - colorscale = px.colors.get_colorscale(colormap_name) - except PlotlyError as e: - logger.error(f"Colorscale '{colormap_name}' not found in Plotly. Using {self.default_colormap}: {e}") - colorscale = px.colors.get_colorscale(self.default_colormap) - - # Generate evenly spaced points - color_points = [i / (num_colors - 1) for i in range(num_colors)] if num_colors > 1 else [0] - return px.colors.sample_colorscale(colorscale, color_points) + return self._get_plotly_colormap_robust(colorscale_name, num_colors) else: # matplotlib try: - cmap = plt.get_cmap(colormap_name, num_colors) + cmap = plt.get_cmap(colorscale_name, num_colors) except ValueError as e: - logger.error(f"Colormap '{colormap_name}' not found in Matplotlib. Using {self.default_colormap}: {e}") - cmap = plt.get_cmap(self.default_colormap, num_colors) + logger.warning( + f"Colormap '{colorscale_name}' not found in Matplotlib. Trying default '{self.default_colorscale}': {e}" + ) + try: + cmap = plt.get_cmap(self.default_colorscale, num_colors) + except ValueError: + logger.warning( + f"Default colormap '{self.default_colorscale}' also not found in Matplotlib. " + f"Using hardcoded fallback 'tab10'" + ) + cmap = plt.get_cmap('tab10', num_colors) return [cmap(i) for i in range(num_colors)] @@ -233,8 +208,8 @@ def _handle_color_list(self, colors: list[str], num_labels: int) -> list[str]: list of colors matching the number of labels """ if len(colors) == 0: - logger.error(f'Empty color list provided. Using {self.default_colormap} instead.') - return self._generate_colors_from_colormap(self.default_colormap, num_labels) + logger.error(f'Empty color list provided. Using {self.default_colorscale} instead.') + return self._generate_colors_from_colorscale(self.default_colorscale, num_labels) if len(colors) < num_labels: logger.warning( @@ -263,18 +238,18 @@ def _handle_color_dict(self, colors: dict[str, str], labels: list[str]) -> list[ list of colors in the same order as labels """ if len(colors) == 0: - logger.warning(f'Empty color dictionary provided. Using {self.default_colormap} instead.') - return self._generate_colors_from_colormap(self.default_colormap, len(labels)) + logger.warning(f'Empty color dictionary provided. Using {self.default_colorscale} instead.') + return self._generate_colors_from_colorscale(self.default_colorscale, len(labels)) # Find missing labels missing_labels = sorted(set(labels) - set(colors.keys())) if missing_labels: logger.warning( - f'Some labels have no color specified: {missing_labels}. Using {self.default_colormap} for these.' + f'Some labels have no color specified: {missing_labels}. Using {self.default_colorscale} for these.' ) # Generate colors for missing labels - missing_colors = self._generate_colors_from_colormap(self.default_colormap, len(missing_labels)) + missing_colors = self._generate_colors_from_colorscale(self.default_colorscale, len(missing_labels)) # Create a copy to avoid modifying the original colors_copy = colors.copy() @@ -296,7 +271,7 @@ def process_colors( Process colors for the specified labels. Args: - colors: Color specification (colormap name, list of colors, or label-to-color mapping) + colors: Color specification (colorscale name, list of colors, or label-to-color mapping) labels: list of data labels that need colors assigned return_mapping: If True, returns a dictionary mapping labels to colors; if False, returns a list of colors in the same order as labels @@ -310,16 +285,16 @@ def process_colors( # Process based on type of colors input if isinstance(colors, str): - color_list = self._generate_colors_from_colormap(colors, len(labels)) + color_list = self._generate_colors_from_colorscale(colors, len(labels)) elif isinstance(colors, list): color_list = self._handle_color_list(colors, len(labels)) elif isinstance(colors, dict): color_list = self._handle_color_dict(colors, labels) else: logger.error( - f'Unsupported color specification type: {type(colors)}. Using {self.default_colormap} instead.' + f'Unsupported color specification type: {type(colors)}. Using {self.default_colorscale} instead.' ) - color_list = self._generate_colors_from_colormap(self.default_colormap, len(labels)) + color_list = self._generate_colors_from_colorscale(self.default_colorscale, len(labels)) # Return either a list or a mapping if return_mapping: @@ -328,6 +303,267 @@ def process_colors( return color_list +class ComponentColorManager: + """Manage consistent colors for flow system components. + + Assign direct colors or group components to get shades from colorscales. + Colorscale families: blues, greens, oranges, reds, purples, teals, greys, etc. + + Example: + ```python + manager = ComponentColorManager() + manager.configure( + { + 'Boiler1': '#FF0000', # Direct color + 'oranges': ['Solar1', 'Solar2'], # Group gets orange shades + } + ) + colors = manager.get_variable_colors(['Boiler1(Bus_A)|flow']) + ``` + """ + + # Class-level colorscale family defaults (Plotly sequential palettes, reversed) + # Reversed so darker colors come first when assigning to components + DEFAULT_FAMILIES = { + 'blues': px.colors.sequential.Blues[7:0:-1], + 'greens': px.colors.sequential.Greens[7:0:-1], + 'reds': px.colors.sequential.Reds[7:0:-1], + 'purples': px.colors.sequential.Purples[7:0:-1], + 'oranges': px.colors.sequential.Oranges[7:0:-1], + 'teals': px.colors.sequential.Teal[7:0:-1], + 'greys': px.colors.sequential.Greys[7:0:-1], + 'pinks': px.colors.sequential.Pinkyl[7:0:-1], + 'peach': px.colors.sequential.Peach[7:0:-1], + 'burg': px.colors.sequential.Burg[7:0:-1], + 'sunsetdark': px.colors.sequential.Sunsetdark[7:0:-1], + 'mint': px.colors.sequential.Mint[7:0:-1], + 'emrld': px.colors.sequential.Emrld[7:0:-1], + 'darkmint': px.colors.sequential.Darkmint[7:0:-1], + } + + def __init__( + self, + components: list[str] | None = None, + default_colorscale: str | None = None, + ) -> None: + """Initialize component color manager. + + Args: + components: Optional list of all component names. If not provided, + components will be discovered from configure() calls. + default_colorscale: Default colormap for ungrouped components. + If None, uses CONFIG.Plotting.default_qualitative_colorscale. + """ + self.components = sorted(set(components)) if components else [] + self.default_colorscale = default_colorscale or CONFIG.Plotting.default_qualitative_colorscale + self.color_families = self.DEFAULT_FAMILIES.copy() + + # Computed colors: {component_name: color} + self._component_colors: dict[str, str] = {} + + # Variable color cache for performance: {variable_name: color} + self._variable_cache: dict[str, str] = {} + + # Auto-assign default colors if components provided + if self.components: + self._assign_default_colors() + + def __repr__(self) -> str: + return ( + f'ComponentColorManager(components={len(self.components)}, ' + f'colors_configured={len(self._component_colors)}, ' + f"default_colorscale='{self.default_colorscale}')" + ) + + def __str__(self) -> str: + lines = [ + 'ComponentColorManager', + f' Components: {len(self.components)}', + ] + + # Show first few components as examples + if self.components: + sample = self.components[:5] + if len(self.components) > 5: + sample_str = ', '.join(sample) + f', ... ({len(self.components) - 5} more)' + else: + sample_str = ', '.join(sample) + lines.append(f' [{sample_str}]') + + lines.append(f' Colors configured: {len(self._component_colors)}') + if self._component_colors: + for comp, color in list(self._component_colors.items())[:3]: + lines.append(f' - {comp}: {color}') + if len(self._component_colors) > 3: + lines.append(f' ... and {len(self._component_colors) - 3} more') + + lines.append(f' Default colormap: {self.default_colorscale}') + + return '\n'.join(lines) + + @classmethod + def from_flow_system(cls, flow_system, **kwargs): + """Create ComponentColorManager from a FlowSystem.""" + from .flow_system import FlowSystem + + if not isinstance(flow_system, FlowSystem): + raise TypeError(f'Expected FlowSystem, got {type(flow_system).__name__}') + + # Extract component names + components = list(flow_system.components.keys()) + + return cls(components=components, **kwargs) + + def configure(self, config: dict[str, str | list[str]] | str | pathlib.Path) -> ComponentColorManager: + """Configure component colors from dict or YAML file. + + Args: + config: Dict with 'component': 'color' or 'colorscale': ['comp1', 'comp2'], + or path to YAML file with same format. + """ + # Load from file if path provided + if isinstance(config, (str, pathlib.Path)): + config = self._load_config_from_file(config) + + if not isinstance(config, dict): + raise TypeError(f'Config must be dict or file path, got {type(config).__name__}') + + # Process config: distinguish between direct colors and grouped colors + for key, value in config.items(): + if isinstance(value, str): + # Direct assignment: component → color + self._component_colors[key] = value + # Add to components list if not already there + if key not in self.components: + self.components.append(key) + self.components.sort() + + elif isinstance(value, list): + # Group assignment: colorscale → [components] + colorscale_name = key + components = value + + # Sample N colors from the colorscale + colors = self._sample_colors_from_colorscale(colorscale_name, len(components)) + + # Assign each component a color + for component, color in zip(components, colors, strict=False): + self._component_colors[component] = color + # Add to components list if not already there + if component not in self.components: + self.components.append(component) + self.components.sort() + + else: + raise TypeError( + f'Invalid config value type for key "{key}". ' + f'Expected str (color) or list[str] (components), got {type(value).__name__}' + ) + + # Clear cache since colors changed + self._variable_cache.clear() + + return self + + def get_color(self, component: str) -> str: + """Get color for a component (defaults to grey if unknown).""" + return self._component_colors.get(component, '#808080') + + def extract_component(self, variable: str) -> str: + """Extract component name from variable name (e.g., 'Boiler1(Bus_A)|flow' → 'Boiler1').""" + component, _ = self._extract_component_and_flow(variable) + return component + + def _extract_component_and_flow(self, variable: str) -> tuple[str, str | None]: + # Try "Component(Flow)|attribute" format + if '(' in variable and ')' in variable: + component = variable.split('(')[0] + flow = variable.split('(')[1].split(')')[0] + return component, flow + + # Try "Component|attribute" format (no flow) + if '|' in variable: + return variable.split('|')[0], None + + # Just the component name itself + return variable, None + + def get_variable_color(self, variable: str) -> str: + """Get color for a variable (extracts component automatically).""" + # Check cache first + if variable in self._variable_cache: + return self._variable_cache[variable] + + # Extract component name from variable + component = self.extract_component(variable) + + # Get color for component + color = self.get_color(component) + + # Cache and return + self._variable_cache[variable] = color + return color + + def get_variable_colors(self, variables: list[str]) -> dict[str, str]: + """Get colors for multiple variables (main API for plotting functions).""" + return {var: self.get_variable_color(var) for var in variables} + + def to_dict(self) -> dict[str, str]: + """Get complete component→color mapping.""" + return self._component_colors.copy() + + # ==================== INTERNAL METHODS ==================== + + def _assign_default_colors(self) -> None: + colors = self._sample_colors_from_colorscale(self.default_colorscale, len(self.components)) + + for component, color in zip(self.components, colors, strict=False): + self._component_colors[component] = color + + @staticmethod + def _load_config_from_file(file_path: str | pathlib.Path) -> dict[str, str | list[str]]: + """Load color configuration from YAML file.""" + file_path = pathlib.Path(file_path) + + if not file_path.exists(): + raise FileNotFoundError(f'Color configuration file not found: {file_path}') + + # Only support YAML + suffix = file_path.suffix.lower() + if suffix not in ['.yaml', '.yml']: + raise ValueError(f'Unsupported file format: {suffix}. Only YAML (.yaml, .yml) is supported.') + + try: + import yaml + except ImportError as e: + raise ImportError( + 'PyYAML is required to load YAML config files. Install it with: pip install pyyaml' + ) from e + + with open(file_path, encoding='utf-8') as f: + config = yaml.safe_load(f) + + # Validate config structure + if not isinstance(config, dict): + raise ValueError(f'Invalid config file structure. Expected dict, got {type(config).__name__}') + + return config + + def _sample_colors_from_colorscale(self, colorscale_name: str, num_colors: int) -> list[str]: + # Check custom families first (ComponentColorManager-specific feature) + if colorscale_name in self.color_families: + color_list = self.color_families[colorscale_name] + # Cycle through colors if needed + if len(color_list) >= num_colors: + return color_list[:num_colors] + else: + return [color_list[i % len(color_list)] for i in range(num_colors)] + + # Delegate everything else to ColorProcessor (handles qualitative, sequential, fallbacks, cycling) + processor = ColorProcessor(engine='plotly', default_colorscale=self.default_colorscale) + return processor._generate_colors_from_colormap(colorscale_name, num_colors) + + def _ensure_dataset(data: xr.Dataset | pd.DataFrame | pd.Series) -> xr.Dataset: """Convert DataFrame or Series to Dataset if needed.""" if isinstance(data, xr.Dataset): @@ -374,14 +610,11 @@ def _validate_plotting_data(data: xr.Dataset, allow_empty: bool = False) -> None def resolve_colors( - data: xr.Dataset, - colors: ColorType, + labels: list[str], + colors: ColorType | ComponentColorManager, engine: PlottingEngine = 'plotly', ) -> dict[str, str]: """Resolve colors parameter to a dict mapping variable names to colors.""" - # Get variable names from Dataset (always strings and unique) - labels = list(data.data_vars.keys()) - # If explicit dict provided, use it directly if isinstance(colors, dict): return colors @@ -391,13 +624,17 @@ def resolve_colors( processor = ColorProcessor(engine=engine) return processor.process_colors(colors, labels, return_mapping=True) + if isinstance(colors, ComponentColorManager): + # Use color manager to resolve colors for variables + return colors.get_variable_colors(labels) + raise TypeError(f'Wrong type passed to resolve_colors(): {type(colors)}') def with_plotly( data: xr.Dataset | pd.DataFrame | pd.Series, mode: Literal['stacked_bar', 'line', 'area', 'grouped_bar'] = 'stacked_bar', - colors: ColorType = 'turbo', + colors: ColorType | None = None, title: str = '', ylabel: str = '', xlabel: str = '', @@ -417,7 +654,7 @@ def with_plotly( data: An xarray Dataset, pandas DataFrame, or pandas Series to plot. mode: The plotting mode. Use 'stacked_bar' for stacked bar charts, 'line' for lines, 'area' for stacked area charts, or 'grouped_bar' for grouped bar charts. - colors: Color specification (colormap, list, or dict mapping labels to colors). + colors: Color specification (colorscale, list, or dict mapping labels to colors). title: The main title of the plot. ylabel: The label for the y-axis. xlabel: The label for the x-axis. @@ -476,9 +713,16 @@ def with_plotly( fig.update_layout(template='plotly_dark', width=1200, height=600) ``` """ + if colors is None: + colors = CONFIG.Plotting.default_qualitative_colorscale + if mode not in ('stacked_bar', 'line', 'area', 'grouped_bar'): raise ValueError(f"'mode' must be one of {{'stacked_bar','line','area', 'grouped_bar'}}, got {mode!r}") + # Apply CONFIG defaults if not explicitly set + if facet_cols is None: + facet_cols = CONFIG.Plotting.default_facet_cols + # Ensure data is a Dataset and validate it data = _ensure_dataset(data) _validate_plotting_data(data, allow_empty=True) @@ -496,7 +740,7 @@ def with_plotly( values = [float(data[var].values) for var in data.data_vars] # Resolve colors - color_discrete_map = resolve_colors(data, colors, engine='plotly') + color_discrete_map = resolve_colors(variables, colors, engine='plotly') marker_colors = [color_discrete_map.get(var, '#636EFA') for var in variables] # Create simple plot based on mode using go (not px) for better color control @@ -587,8 +831,7 @@ def with_plotly( # Process colors all_vars = df_long['variable'].unique().tolist() - processed_colors = ColorProcessor(engine='plotly').process_colors(colors, all_vars) - color_discrete_map = {var: color for var, color in zip(all_vars, processed_colors, strict=True)} + color_discrete_map = resolve_colors(list(data.data_vars), colors, engine='plotly') # Determine which dimension to use for x-axis # Collect dimensions used for faceting and animation @@ -705,7 +948,7 @@ def with_plotly( def with_matplotlib( data: xr.Dataset | pd.DataFrame | pd.Series, mode: Literal['stacked_bar', 'line'] = 'stacked_bar', - colors: ColorType = 'turbo', + colors: ColorType | None = None, title: str = '', ylabel: str = '', xlabel: str = 'Time in h', @@ -720,7 +963,7 @@ def with_matplotlib( the index represents time and each column represents a separate data series (variables). mode: Plotting mode. Use 'stacked_bar' for stacked bar charts or 'line' for stepped lines. colors: Color specification. Can be: - - A colormap name (e.g., 'turbo', 'plasma') + - A colorscale name (e.g., 'turbo', 'plasma') - A list of color strings (e.g., ['#ff0000', '#00ff00']) - A dict mapping column names to colors (e.g., {'Column1': '#ff0000'}) title: The title of the plot. @@ -738,6 +981,9 @@ def with_matplotlib( Negative values are stacked separately without extra labels in the legend. - If `mode` is 'line', stepped lines are drawn for each data series. """ + if colors is None: + colors = CONFIG.Plotting.default_qualitative_colorscale + if mode not in ('stacked_bar', 'line'): raise ValueError(f"'mode' must be one of {{'stacked_bar','line'}} for matplotlib, got {mode!r}") @@ -760,7 +1006,7 @@ def with_matplotlib( values = [float(data[var].values) for var in data.data_vars] # Resolve colors - color_discrete_map = resolve_colors(data, colors, engine='matplotlib') + color_discrete_map = resolve_colors(variables, colors, engine='matplotlib') colors_list = [color_discrete_map.get(var, '#808080') for var in variables] # Create plot based on mode @@ -791,7 +1037,7 @@ def with_matplotlib( return fig, ax # Resolve colors first (includes validation) - color_discrete_map = resolve_colors(data, colors, engine='matplotlib') + color_discrete_map = resolve_colors(list(data.data_vars), colors, engine='matplotlib') # Convert Dataset to DataFrame for matplotlib plotting (naturally wide-form) df = data.to_dataframe() @@ -1199,7 +1445,7 @@ def preprocess_data_for_pie( def dual_pie_with_plotly( data_left: xr.Dataset | pd.DataFrame | pd.Series, data_right: xr.Dataset | pd.DataFrame | pd.Series, - colors: ColorType = 'turbo', + colors: ColorType | None = None, title: str = '', subtitles: tuple[str, str] = ('Left Chart', 'Right Chart'), legend_title: str = '', @@ -1229,6 +1475,9 @@ def dual_pie_with_plotly( Returns: Plotly Figure object """ + if colors is None: + colors = CONFIG.Plotting.default_qualitative_colorscale + # Preprocess data to Series left_series = preprocess_data_for_pie(data_left, lower_percentage_group) right_series = preprocess_data_for_pie(data_right, lower_percentage_group) @@ -1244,7 +1493,7 @@ def dual_pie_with_plotly( all_labels = sorted(set(left_labels) | set(right_labels)) # Create color map - color_map = ColorProcessor(engine='plotly').process_colors(colors, all_labels, return_mapping=True) + color_map = resolve_colors(all_labels, colors, engine='plotly') # Create figure fig = go.Figure() @@ -1294,7 +1543,7 @@ def dual_pie_with_plotly( def dual_pie_with_matplotlib( data_left: xr.Dataset | pd.DataFrame | pd.Series, data_right: xr.Dataset | pd.DataFrame | pd.Series, - colors: ColorType = 'turbo', + colors: ColorType | None = None, title: str = '', subtitles: tuple[str, str] = ('Left Chart', 'Right Chart'), legend_title: str = '', @@ -1308,7 +1557,7 @@ def dual_pie_with_matplotlib( Args: data_left: Data for the left pie chart. data_right: Data for the right pie chart. - colors: Color specification (colormap name, list of colors, or dict mapping) + colors: Color specification (colorscale name, list of colors, or dict mapping) title: The main title of the plot. subtitles: Tuple containing the subtitles for (left, right) charts. legend_title: The title for the legend. @@ -1319,6 +1568,9 @@ def dual_pie_with_matplotlib( Returns: Tuple of (Figure, list of Axes) """ + if colors is None: + colors = CONFIG.Plotting.default_qualitative_colorscale + # Preprocess data to Series left_series = preprocess_data_for_pie(data_left, lower_percentage_group) right_series = preprocess_data_for_pie(data_right, lower_percentage_group) @@ -1400,11 +1652,11 @@ def draw_pie(ax, labels, values, subtitle): def heatmap_with_plotly( data: xr.DataArray, - colors: ColorType = 'turbo', + colors: ColorType | None = None, title: str = '', facet_by: str | list[str] | None = None, animate_by: str | None = None, - facet_cols: int = 3, + facet_cols: int | None = None, reshape_time: tuple[Literal['YS', 'MS', 'W', 'D', 'h', '15min', 'min'], Literal['W', 'D', 'h', '15min', 'min']] | Literal['auto'] | None = 'auto', @@ -1427,7 +1679,7 @@ def heatmap_with_plotly( Args: data: An xarray DataArray containing the data to visualize. Should have at least 2 dimensions, or a 'time' dimension that can be reshaped into 2D. - colors: Color specification (colormap name, list, or dict). Common options: + colors: Color specification (colorscale name, list, or dict). Common options: 'turbo', 'plasma', 'RdBu', 'portland'. title: The main title of the heatmap. facet_by: Dimension to create facets for. Creates a subplot grid. @@ -1484,6 +1736,13 @@ def heatmap_with_plotly( fig = heatmap_with_plotly(data_array, facet_by='scenario', animate_by='period', reshape_time=('W', 'D')) ``` """ + if colors is None: + colors = CONFIG.Plotting.default_sequential_colorscale + + # Apply CONFIG defaults if not explicitly set + if facet_cols is None: + facet_cols = CONFIG.Plotting.default_facet_cols + # Handle empty data if data.size == 0: return go.Figure() @@ -1577,7 +1836,7 @@ def heatmap_with_plotly( # Create the imshow plot - px.imshow can work directly with xarray DataArrays common_args = { 'img': data, - 'color_continuous_scale': colors if isinstance(colors, str) else 'turbo', + 'color_continuous_scale': colors, 'title': title, } @@ -1601,7 +1860,7 @@ def heatmap_with_plotly( # Fallback: create a simple heatmap without faceting fallback_args = { 'img': data.values, - 'color_continuous_scale': colors if isinstance(colors, str) else 'turbo', + 'color_continuous_scale': colors, 'title': title, } fallback_args.update(imshow_kwargs) @@ -1612,7 +1871,7 @@ def heatmap_with_plotly( def heatmap_with_matplotlib( data: xr.DataArray, - colors: ColorType = 'turbo', + colors: ColorType | None = None, title: str = '', figsize: tuple[float, float] = (12, 6), reshape_time: tuple[Literal['YS', 'MS', 'W', 'D', 'h', '15min', 'min'], Literal['W', 'D', 'h', '15min', 'min']] @@ -1635,7 +1894,7 @@ def heatmap_with_matplotlib( data: An xarray DataArray containing the data to visualize. Should have at least 2 dimensions. If more than 2 dimensions exist, additional dimensions will be reduced by taking the first slice. - colors: Color specification. Should be a colormap name (e.g., 'turbo', 'RdBu'). + colors: Color specification. Should be a colorscale name (e.g., 'turbo', 'RdBu'). title: The title of the heatmap. figsize: The size of the figure (width, height) in inches. reshape_time: Time reshaping configuration: @@ -1675,6 +1934,9 @@ def heatmap_with_matplotlib( fig, ax = heatmap_with_matplotlib(data_array, reshape_time=('D', 'h')) ``` """ + if colors is None: + colors = CONFIG.Plotting.default_sequential_colorscale + # Initialize kwargs if not provided if imshow_kwargs is None: imshow_kwargs = {} @@ -1726,11 +1988,8 @@ def heatmap_with_matplotlib( x_labels = 'x' y_labels = 'y' - # Process colormap - cmap = colors if isinstance(colors, str) else 'turbo' - # Create the heatmap using imshow with user customizations - imshow_defaults = {'cmap': cmap, 'aspect': 'auto', 'origin': 'upper', 'vmin': vmin, 'vmax': vmax} + imshow_defaults = {'cmap': colors, 'aspect': 'auto', 'origin': 'upper', 'vmin': vmin, 'vmax': vmax} imshow_defaults.update(imshow_kwargs) # User kwargs override defaults im = ax.imshow(values, **imshow_defaults) @@ -1759,9 +2018,9 @@ def export_figure( default_path: pathlib.Path, default_filetype: str | None = None, user_path: pathlib.Path | None = None, - show: bool = True, + show: bool | None = None, save: bool = False, - dpi: int = 300, + dpi: int | None = None, ) -> go.Figure | tuple[plt.Figure, plt.Axes]: """ Export a figure to a file and or show it. @@ -1771,14 +2030,21 @@ def export_figure( default_path: The default file path if no user filename is provided. default_filetype: The default filetype if the path doesnt end with a filetype. user_path: An optional user-specified file path. - show: Whether to display the figure (default: True). + show: Whether to display the figure. If None, uses CONFIG.Plotting.default_show (default: None). save: Whether to save the figure (default: False). - dpi: DPI (dots per inch) for saving Matplotlib figures. If None, Matplotlib rcParams are used. + dpi: DPI (dots per inch) for saving Matplotlib figures. If None, uses CONFIG.Plotting.default_dpi. Raises: ValueError: If no default filetype is provided and the path doesn't specify a filetype. TypeError: If the figure type is not supported. """ + # Apply CONFIG defaults if not explicitly set + if show is None: + show = CONFIG.Plotting.default_show + + if dpi is None: + dpi = CONFIG.Plotting.default_dpi + filename = user_path or default_path filename = filename.with_name(filename.name.replace('|', '__')) if filename.suffix == '': @@ -1793,25 +2059,17 @@ def export_figure( filename = filename.with_suffix('.html') try: - is_test_env = 'PYTEST_CURRENT_TEST' in os.environ - - if is_test_env: - # Test environment: never open browser, only save if requested - if save: - fig.write_html(str(filename)) - # Ignore show flag in tests - else: - # Production environment: respect show and save flags - if save and show: - # Save and auto-open in browser - plotly.offline.plot(fig, filename=str(filename)) - elif save and not show: - # Save without opening - fig.write_html(str(filename)) - elif show and not save: - # Show interactively without saving - fig.show() - # If neither save nor show: do nothing + # Respect show and save flags (tests should set CONFIG.Plotting.default_show=False) + if save and show: + # Save and auto-open in browser + plotly.offline.plot(fig, filename=str(filename)) + elif save and not show: + # Save without opening + fig.write_html(str(filename)) + elif show and not save: + # Show interactively without saving + fig.show() + # If neither save nor show: do nothing finally: # Cleanup to prevent socket warnings if hasattr(fig, '_renderer'): @@ -1822,12 +2080,11 @@ def export_figure( elif isinstance(figure_like, tuple): fig, ax = figure_like if show: - # Only show if using interactive backend and not in test environment + # Only show if using interactive backend (tests should set CONFIG.Plotting.default_show=False) backend = matplotlib.get_backend().lower() is_interactive = backend not in {'agg', 'pdf', 'ps', 'svg', 'template'} - is_test_env = 'PYTEST_CURRENT_TEST' in os.environ - if is_interactive and not is_test_env: + if is_interactive: plt.show() if save: From dfa385f112d5ff8f7c6690553be120dc5a5e5066 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:14:06 +0200 Subject: [PATCH 04/48] update color management --- flixopt/color_processing.py | 248 ++++++++++++++++++++++++++++++++++++ flixopt/plotting.py | 234 +++------------------------------- 2 files changed, 269 insertions(+), 213 deletions(-) create mode 100644 flixopt/color_processing.py diff --git a/flixopt/color_processing.py b/flixopt/color_processing.py new file mode 100644 index 000000000..f4e2843f3 --- /dev/null +++ b/flixopt/color_processing.py @@ -0,0 +1,248 @@ +"""Simplified color handling for visualization. + +This module provides clean color processing that transforms various input formats +into a label-to-color mapping dictionary, without needing to know about the plotting engine. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +import matplotlib.colors as mcolors +import matplotlib.pyplot as plt +import plotly.express as px +from plotly.exceptions import PlotlyError + +if TYPE_CHECKING: + from .plotting import ComponentColorManager + +logger = logging.getLogger('flixopt') + + +def process_colors( + colors: None | str | list[str] | dict[str, str], + labels: list[str], + default_colorscale: str = 'turbo', +) -> dict[str, str]: + """Process color input and return a label-to-color mapping. + + This function takes flexible color input and always returns a dictionary + mapping each label to a specific color string. The plotting engine can then + use this mapping as needed. + + Args: + colors: Color specification in one of four formats: + - None: Use the default colorscale + - str: Name of a colorscale (e.g., 'turbo', 'plasma', 'Set1', 'portland') + - list[str]: List of color strings (hex, named colors, etc.) + - dict[str, str]: Direct label-to-color mapping + labels: List of labels that need colors assigned + default_colorscale: Fallback colorscale name if requested scale not found + + Returns: + Dictionary mapping each label to a color string + + Examples: + >>> # Using None - applies default colorscale + >>> process_colors(None, ['A', 'B', 'C']) + {'A': '#0d0887', 'B': '#7e03a8', 'C': '#cc4778'} + + >>> # Using a colorscale name + >>> process_colors('plasma', ['A', 'B', 'C']) + {'A': '#0d0887', 'B': '#7e03a8', 'C': '#cc4778'} + + >>> # Using a list of colors + >>> process_colors(['red', 'blue', 'green'], ['A', 'B', 'C']) + {'A': 'red', 'B': 'blue', 'C': 'green'} + + >>> # Using a pre-made mapping + >>> process_colors({'A': 'red', 'B': 'blue'}, ['A', 'B', 'C']) + {'A': 'red', 'B': 'blue', 'C': '#0d0887'} # C gets color from default scale + """ + if not labels: + return {} + + # Case 1: Already a mapping dictionary + if isinstance(colors, dict): + return _fill_missing_colors(colors, labels, default_colorscale) + + # Case 2: None or colorscale name (string) + if colors is None or isinstance(colors, str): + colorscale_name = colors if colors is not None else default_colorscale + color_list = _get_colors_from_scale(colorscale_name, len(labels), default_colorscale) + return dict(zip(labels, color_list, strict=False)) + + # Case 3: List of colors + if isinstance(colors, list): + if len(colors) == 0: + logger.warning(f'Empty color list provided. Using {default_colorscale} instead.') + color_list = _get_colors_from_scale(default_colorscale, len(labels), default_colorscale) + return dict(zip(labels, color_list, strict=False)) + + if len(colors) < len(labels): + logger.debug( + f'Not enough colors provided ({len(colors)}) for all labels ({len(labels)}). Colors will cycle.' + ) + + # Cycle through colors if we don't have enough + return {label: colors[i % len(colors)] for i, label in enumerate(labels)} + + raise TypeError(f'colors must be None, str, list, or dict, got {type(colors)}') + + +def resolve_colors( + labels: list[str], + colors: None | str | list[str] | dict[str, str] | ComponentColorManager, + default_colorscale: str = 'turbo', +) -> dict[str, str]: + """Resolve colors parameter to a dict mapping variable names to colors. + + This is the unified interface that supports both simple color specifications + and ComponentColorManager instances. + + Args: + labels: List of labels/variables that need colors + colors: Color specification (None, str, list, dict) or ComponentColorManager + default_colorscale: Fallback colorscale name + + Returns: + Dictionary mapping each label to a color string + """ + # Import here to avoid circular dependency + from .plotting import ComponentColorManager + + if isinstance(colors, ComponentColorManager): + return colors.get_variable_colors(labels) + + # Use the simplified process_colors for everything else + return process_colors(colors, labels, default_colorscale) + + +def _fill_missing_colors( + color_mapping: dict[str, str], + labels: list[str], + default_colorscale: str, +) -> dict[str, str]: + """Fill in missing labels in a color mapping using a colorscale. + + Args: + color_mapping: Partial label-to-color mapping + labels: All labels that need colors + default_colorscale: Colorscale to use for missing labels + + Returns: + Complete label-to-color mapping + """ + missing_labels = [label for label in labels if label not in color_mapping] + + if not missing_labels: + return color_mapping.copy() + + # Log warning about missing labels + logger.debug(f'Labels missing colors: {missing_labels}. Using {default_colorscale} for these.') + + # Get colors for missing labels + missing_colors = _get_colors_from_scale(default_colorscale, len(missing_labels), default_colorscale) + + # Combine existing and new colors + result = color_mapping.copy() + result.update(dict(zip(missing_labels, missing_colors, strict=False))) + return result + + +def _get_colors_from_scale( + colorscale_name: str, + num_colors: int, + fallback_scale: str, +) -> list[str]: + """Extract a list of colors from a named colorscale. + + Tries to get colors from the named scale (Plotly first, then Matplotlib), + falls back to the fallback scale if not found. + + Args: + colorscale_name: Name of the colorscale to try + num_colors: Number of colors needed + fallback_scale: Fallback colorscale name if first fails + + Returns: + List of color strings (hex format) + """ + # Try to get the requested colorscale + colors = _try_get_colorscale(colorscale_name, num_colors) + + if colors is not None: + return colors + + # Fallback to default + logger.warning(f"Colorscale '{colorscale_name}' not found. Using '{fallback_scale}' instead.") + + colors = _try_get_colorscale(fallback_scale, num_colors) + + if colors is not None: + return colors + + # Ultimate fallback: just use basic colors + logger.warning(f"Fallback colorscale '{fallback_scale}' also not found. Using basic colors.") + basic_colors = [ + '#1f77b4', + '#ff7f0e', + '#2ca02c', + '#d62728', + '#9467bd', + '#8c564b', + '#e377c2', + '#7f7f7f', + '#bcbd22', + '#17becf', + ] + return [basic_colors[i % len(basic_colors)] for i in range(num_colors)] + + +def _try_get_colorscale(colorscale_name: str, num_colors: int) -> list[str] | None: + """Try to get colors from Plotly or Matplotlib colorscales. + + Tries Plotly colorscales first (both qualitative and sequential), + then falls back to Matplotlib colorscales. + + Args: + colorscale_name: Name of the colorscale + num_colors: Number of colors needed + + Returns: + List of color strings (hex format) if successful, None if colorscale not found + """ + # First try Plotly qualitative (discrete) color sequences + colorscale_title = colorscale_name.title() + if hasattr(px.colors.qualitative, colorscale_title): + color_list = getattr(px.colors.qualitative, colorscale_title) + return [color_list[i % len(color_list)] for i in range(num_colors)] + + # Then try Plotly sequential/continuous colorscales + try: + colorscale = px.colors.get_colorscale(colorscale_name) + # Sample evenly from the colorscale + if num_colors == 1: + sample_points = [0.5] + else: + sample_points = [i / (num_colors - 1) for i in range(num_colors)] + return px.colors.sample_colorscale(colorscale, sample_points) + except PlotlyError: + pass + + # Finally try Matplotlib colorscales + try: + cmap = plt.get_cmap(colorscale_name) + + # Sample evenly from the colorscale + if num_colors == 1: + colors = [cmap(0.5)] + else: + colors = [cmap(i / (num_colors - 1)) for i in range(num_colors)] + + # Convert RGBA tuples to hex strings + return [mcolors.rgb2hex(color[:3]) for color in colors] + + except (ValueError, KeyError): + return None diff --git a/flixopt/plotting.py b/flixopt/plotting.py index 3a11b24f6..1e75cc38c 100644 --- a/flixopt/plotting.py +++ b/flixopt/plotting.py @@ -42,6 +42,7 @@ import xarray as xr from plotly.exceptions import PlotlyError +from .color_processing import process_colors, resolve_colors from .config import CONFIG if TYPE_CHECKING: @@ -122,187 +123,6 @@ """Identifier for the plotting engine to use.""" -class ColorProcessor: - """Intelligent color management system for consistent multi-backend visualization. - - This class provides unified color processing across Plotly and Matplotlib backends, - ensuring consistent visual appearance regardless of the plotting engine used. - It handles color palette generation, named colorscale translation, and intelligent - color cycling for complex datasets with many categories. - - Key Features: - **Backend Agnostic**: Automatic color format conversion between engines - **Palette Management**: Support for named colorscales, custom palettes, and color lists - **Intelligent Cycling**: Smart color assignment for datasets with many categories - **Fallback Handling**: Graceful degradation when requested colorscales are unavailable - **Energy System Colors**: Built-in palettes optimized for energy system visualization - - Color Input Types: - - **Named colorscales**: 'turbo', 'plasma', 'portland', etc. - - **Color Lists**: ['red', 'blue', 'green'] or ['#FF0000', '#0000FF', '#00FF00'] - - **Label Dictionaries**: {'Generator': 'red', 'Storage': 'blue', 'Load': 'green'} - - Example: - ```python - processor = ColorProcessor(engine='plotly', default_colorscale='turbo') - colors = processor.process_colors('plasma', ['Gen1', 'Gen2', 'Storage']) - ``` - - Args: - engine: Plotting backend ('plotly' or 'matplotlib'). Determines output color format. - default_colorscale: Fallback colorscale when requested palettes are unavailable. - Common options: 'turbo', 'plasma', 'portland'. - - """ - - def __init__(self, engine: PlottingEngine = 'plotly', default_colorscale: str | None = None): - """Initialize the color processor with specified backend and defaults.""" - if engine not in ['plotly', 'matplotlib']: - raise TypeError(f'engine must be "plotly" or "matplotlib", but is {engine}') - self.engine = engine - self.default_colorscale = ( - default_colorscale if default_colorscale is not None else CONFIG.Plotting.default_qualitative_colorscale - ) - - def _generate_colors_from_colorscale(self, colorscale_name: str, num_colors: int) -> list[Any]: - """ - Generate colors from a named colorscale. - - Args: - colorscale_name: Name of the colorscale - num_colors: Number of colors to generate - - Returns: - list of colors in the format appropriate for the engine - """ - if self.engine == 'plotly': - return self._get_plotly_colormap_robust(colorscale_name, num_colors) - - else: # matplotlib - try: - cmap = plt.get_cmap(colorscale_name, num_colors) - except ValueError as e: - logger.warning( - f"Colormap '{colorscale_name}' not found in Matplotlib. Trying default '{self.default_colorscale}': {e}" - ) - try: - cmap = plt.get_cmap(self.default_colorscale, num_colors) - except ValueError: - logger.warning( - f"Default colormap '{self.default_colorscale}' also not found in Matplotlib. " - f"Using hardcoded fallback 'tab10'" - ) - cmap = plt.get_cmap('tab10', num_colors) - - return [cmap(i) for i in range(num_colors)] - - def _handle_color_list(self, colors: list[str], num_labels: int) -> list[str]: - """ - Handle a list of colors, cycling if necessary. - - Args: - colors: list of color strings - num_labels: Number of labels that need colors - - Returns: - list of colors matching the number of labels - """ - if len(colors) == 0: - logger.error(f'Empty color list provided. Using {self.default_colorscale} instead.') - return self._generate_colors_from_colorscale(self.default_colorscale, num_labels) - - if len(colors) < num_labels: - logger.warning( - f'Not enough colors provided ({len(colors)}) for all labels ({num_labels}). Colors will cycle.' - ) - # Cycle through the colors - color_iter = itertools.cycle(colors) - return [next(color_iter) for _ in range(num_labels)] - else: - # Trim if necessary - if len(colors) > num_labels: - logger.warning( - f'More colors provided ({len(colors)}) than labels ({num_labels}). Extra colors will be ignored.' - ) - return colors[:num_labels] - - def _handle_color_dict(self, colors: dict[str, str], labels: list[str]) -> list[str]: - """ - Handle a dictionary mapping labels to colors. - - Args: - colors: Dictionary mapping labels to colors - labels: list of labels that need colors - - Returns: - list of colors in the same order as labels - """ - if len(colors) == 0: - logger.warning(f'Empty color dictionary provided. Using {self.default_colorscale} instead.') - return self._generate_colors_from_colorscale(self.default_colorscale, len(labels)) - - # Find missing labels - missing_labels = sorted(set(labels) - set(colors.keys())) - if missing_labels: - logger.warning( - f'Some labels have no color specified: {missing_labels}. Using {self.default_colorscale} for these.' - ) - - # Generate colors for missing labels - missing_colors = self._generate_colors_from_colorscale(self.default_colorscale, len(missing_labels)) - - # Create a copy to avoid modifying the original - colors_copy = colors.copy() - for i, label in enumerate(missing_labels): - colors_copy[label] = missing_colors[i] - else: - colors_copy = colors - - # Create color list in the same order as labels - return [colors_copy[label] for label in labels] - - def process_colors( - self, - colors: ColorType, - labels: list[str], - return_mapping: bool = False, - ) -> list[Any] | dict[str, Any]: - """ - Process colors for the specified labels. - - Args: - colors: Color specification (colorscale name, list of colors, or label-to-color mapping) - labels: list of data labels that need colors assigned - return_mapping: If True, returns a dictionary mapping labels to colors; - if False, returns a list of colors in the same order as labels - - Returns: - Either a list of colors or a dictionary mapping labels to colors - """ - if len(labels) == 0: - logger.error('No labels provided for color assignment.') - return {} if return_mapping else [] - - # Process based on type of colors input - if isinstance(colors, str): - color_list = self._generate_colors_from_colorscale(colors, len(labels)) - elif isinstance(colors, list): - color_list = self._handle_color_list(colors, len(labels)) - elif isinstance(colors, dict): - color_list = self._handle_color_dict(colors, labels) - else: - logger.error( - f'Unsupported color specification type: {type(colors)}. Using {self.default_colorscale} instead.' - ) - color_list = self._generate_colors_from_colorscale(self.default_colorscale, len(labels)) - - # Return either a list or a mapping - if return_mapping: - return {label: color_list[i] for i, label in enumerate(labels)} - else: - return color_list - - class ComponentColorManager: """Manage consistent colors for flow system components. @@ -559,9 +379,11 @@ def _sample_colors_from_colorscale(self, colorscale_name: str, num_colors: int) else: return [color_list[i % len(color_list)] for i in range(num_colors)] - # Delegate everything else to ColorProcessor (handles qualitative, sequential, fallbacks, cycling) - processor = ColorProcessor(engine='plotly', default_colorscale=self.default_colorscale) - return processor._generate_colors_from_colormap(colorscale_name, num_colors) + # Delegate to the simplified color processing (handles qualitative, sequential, fallbacks, cycling) + # Use dummy labels since we just need the color values + dummy_labels = [f'_{i}' for i in range(num_colors)] + color_map = process_colors(colorscale_name, dummy_labels, default_colorscale=self.default_colorscale) + return list(color_map.values()) def _ensure_dataset(data: xr.Dataset | pd.DataFrame | pd.Series) -> xr.Dataset: @@ -609,28 +431,6 @@ def _validate_plotting_data(data: xr.Dataset, allow_empty: bool = False) -> None logger.debug(f"Variable '{var}' contains Inf values which may affect visualization.") -def resolve_colors( - labels: list[str], - colors: ColorType | ComponentColorManager, - engine: PlottingEngine = 'plotly', -) -> dict[str, str]: - """Resolve colors parameter to a dict mapping variable names to colors.""" - # If explicit dict provided, use it directly - if isinstance(colors, dict): - return colors - - # If string or list, use ColorProcessor (traditional behavior) - if isinstance(colors, (str, list)): - processor = ColorProcessor(engine=engine) - return processor.process_colors(colors, labels, return_mapping=True) - - if isinstance(colors, ComponentColorManager): - # Use color manager to resolve colors for variables - return colors.get_variable_colors(labels) - - raise TypeError(f'Wrong type passed to resolve_colors(): {type(colors)}') - - def with_plotly( data: xr.Dataset | pd.DataFrame | pd.Series, mode: Literal['stacked_bar', 'line', 'area', 'grouped_bar'] = 'stacked_bar', @@ -740,7 +540,9 @@ def with_plotly( values = [float(data[var].values) for var in data.data_vars] # Resolve colors - color_discrete_map = resolve_colors(variables, colors, engine='plotly') + color_discrete_map = resolve_colors( + variables, colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale + ) marker_colors = [color_discrete_map.get(var, '#636EFA') for var in variables] # Create simple plot based on mode using go (not px) for better color control @@ -831,7 +633,9 @@ def with_plotly( # Process colors all_vars = df_long['variable'].unique().tolist() - color_discrete_map = resolve_colors(list(data.data_vars), colors, engine='plotly') + color_discrete_map = resolve_colors( + list(data.data_vars), colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale + ) # Determine which dimension to use for x-axis # Collect dimensions used for faceting and animation @@ -1006,7 +810,9 @@ def with_matplotlib( values = [float(data[var].values) for var in data.data_vars] # Resolve colors - color_discrete_map = resolve_colors(variables, colors, engine='matplotlib') + color_discrete_map = resolve_colors( + variables, colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale + ) colors_list = [color_discrete_map.get(var, '#808080') for var in variables] # Create plot based on mode @@ -1037,7 +843,9 @@ def with_matplotlib( return fig, ax # Resolve colors first (includes validation) - color_discrete_map = resolve_colors(list(data.data_vars), colors, engine='matplotlib') + color_discrete_map = resolve_colors( + list(data.data_vars), colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale + ) # Convert Dataset to DataFrame for matplotlib plotting (naturally wide-form) df = data.to_dataframe() @@ -1493,7 +1301,7 @@ def dual_pie_with_plotly( all_labels = sorted(set(left_labels) | set(right_labels)) # Create color map - color_map = resolve_colors(all_labels, colors, engine='plotly') + color_map = resolve_colors(all_labels, colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale) # Create figure fig = go.Figure() @@ -1585,8 +1393,8 @@ def dual_pie_with_matplotlib( # Get all unique labels for consistent coloring all_labels = sorted(set(left_labels) | set(right_labels)) - # Create color map - color_map = ColorProcessor(engine='matplotlib').process_colors(colors, all_labels, return_mapping=True) + # Create color map (process_colors always returns a dict) + color_map = process_colors(colors, all_labels, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale) # Create figure fig, axes = plt.subplots(1, 2, figsize=figsize) From 2346759c07c8b9c0a67df5a51fe60e75208d35a2 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:17:59 +0200 Subject: [PATCH 05/48] Add rgb to hex for matplotlib --- flixopt/color_processing.py | 44 ++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/flixopt/color_processing.py b/flixopt/color_processing.py index f4e2843f3..adf0d6d82 100644 --- a/flixopt/color_processing.py +++ b/flixopt/color_processing.py @@ -20,6 +20,23 @@ logger = logging.getLogger('flixopt') +def _rgb_string_to_hex(color: str) -> str: + """Convert Plotly RGB string format to hex. + + Args: + color: Color in format 'rgb(R, G, B)' or already in hex + + Returns: + Color in hex format '#RRGGBB' + """ + if color.startswith('rgb('): + # Extract RGB values from 'rgb(R, G, B)' format + rgb_str = color[4:-1] # Remove 'rgb(' and ')' + r, g, b = map(int, rgb_str.split(',')) + return f'#{r:02x}{g:02x}{b:02x}' + return color + + def process_colors( colors: None | str | list[str] | dict[str, str], labels: list[str], @@ -96,26 +113,6 @@ def resolve_colors( colors: None | str | list[str] | dict[str, str] | ComponentColorManager, default_colorscale: str = 'turbo', ) -> dict[str, str]: - """Resolve colors parameter to a dict mapping variable names to colors. - - This is the unified interface that supports both simple color specifications - and ComponentColorManager instances. - - Args: - labels: List of labels/variables that need colors - colors: Color specification (None, str, list, dict) or ComponentColorManager - default_colorscale: Fallback colorscale name - - Returns: - Dictionary mapping each label to a color string - """ - # Import here to avoid circular dependency - from .plotting import ComponentColorManager - - if isinstance(colors, ComponentColorManager): - return colors.get_variable_colors(labels) - - # Use the simplified process_colors for everything else return process_colors(colors, labels, default_colorscale) @@ -217,7 +214,8 @@ def _try_get_colorscale(colorscale_name: str, num_colors: int) -> list[str] | No colorscale_title = colorscale_name.title() if hasattr(px.colors.qualitative, colorscale_title): color_list = getattr(px.colors.qualitative, colorscale_title) - return [color_list[i % len(color_list)] for i in range(num_colors)] + # Convert to hex format for matplotlib compatibility + return [_rgb_string_to_hex(color_list[i % len(color_list)]) for i in range(num_colors)] # Then try Plotly sequential/continuous colorscales try: @@ -227,7 +225,9 @@ def _try_get_colorscale(colorscale_name: str, num_colors: int) -> list[str] | No sample_points = [0.5] else: sample_points = [i / (num_colors - 1) for i in range(num_colors)] - return px.colors.sample_colorscale(colorscale, sample_points) + colors = px.colors.sample_colorscale(colorscale, sample_points) + # Convert to hex format for matplotlib compatibility + return [_rgb_string_to_hex(c) for c in colors] except PlotlyError: pass From acdf93d3e5463896f7af1a2e0724a25bd74db8c7 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:18:23 +0200 Subject: [PATCH 06/48] Add rgb to hex for matplotlib --- flixopt/color_processing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flixopt/color_processing.py b/flixopt/color_processing.py index adf0d6d82..17b54c463 100644 --- a/flixopt/color_processing.py +++ b/flixopt/color_processing.py @@ -113,6 +113,7 @@ def resolve_colors( colors: None | str | list[str] | dict[str, str] | ComponentColorManager, default_colorscale: str = 'turbo', ) -> dict[str, str]: + """Temporary wrapper""" return process_colors(colors, labels, default_colorscale) From 5c24d251bc8d27bce7c307ace082819adaa3074a Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:19:17 +0200 Subject: [PATCH 07/48] Remove colormanager class --- flixopt/color_processing.py | 5 +- flixopt/plotting.py | 263 ------------------------------------ 2 files changed, 1 insertion(+), 267 deletions(-) diff --git a/flixopt/color_processing.py b/flixopt/color_processing.py index 17b54c463..366a5e71c 100644 --- a/flixopt/color_processing.py +++ b/flixopt/color_processing.py @@ -14,9 +14,6 @@ import plotly.express as px from plotly.exceptions import PlotlyError -if TYPE_CHECKING: - from .plotting import ComponentColorManager - logger = logging.getLogger('flixopt') @@ -110,7 +107,7 @@ def process_colors( def resolve_colors( labels: list[str], - colors: None | str | list[str] | dict[str, str] | ComponentColorManager, + colors: None | str | list[str] | dict[str, str], default_colorscale: str = 'turbo', ) -> dict[str, str]: """Temporary wrapper""" diff --git a/flixopt/plotting.py b/flixopt/plotting.py index 1e75cc38c..be1a521c4 100644 --- a/flixopt/plotting.py +++ b/flixopt/plotting.py @@ -123,269 +123,6 @@ """Identifier for the plotting engine to use.""" -class ComponentColorManager: - """Manage consistent colors for flow system components. - - Assign direct colors or group components to get shades from colorscales. - Colorscale families: blues, greens, oranges, reds, purples, teals, greys, etc. - - Example: - ```python - manager = ComponentColorManager() - manager.configure( - { - 'Boiler1': '#FF0000', # Direct color - 'oranges': ['Solar1', 'Solar2'], # Group gets orange shades - } - ) - colors = manager.get_variable_colors(['Boiler1(Bus_A)|flow']) - ``` - """ - - # Class-level colorscale family defaults (Plotly sequential palettes, reversed) - # Reversed so darker colors come first when assigning to components - DEFAULT_FAMILIES = { - 'blues': px.colors.sequential.Blues[7:0:-1], - 'greens': px.colors.sequential.Greens[7:0:-1], - 'reds': px.colors.sequential.Reds[7:0:-1], - 'purples': px.colors.sequential.Purples[7:0:-1], - 'oranges': px.colors.sequential.Oranges[7:0:-1], - 'teals': px.colors.sequential.Teal[7:0:-1], - 'greys': px.colors.sequential.Greys[7:0:-1], - 'pinks': px.colors.sequential.Pinkyl[7:0:-1], - 'peach': px.colors.sequential.Peach[7:0:-1], - 'burg': px.colors.sequential.Burg[7:0:-1], - 'sunsetdark': px.colors.sequential.Sunsetdark[7:0:-1], - 'mint': px.colors.sequential.Mint[7:0:-1], - 'emrld': px.colors.sequential.Emrld[7:0:-1], - 'darkmint': px.colors.sequential.Darkmint[7:0:-1], - } - - def __init__( - self, - components: list[str] | None = None, - default_colorscale: str | None = None, - ) -> None: - """Initialize component color manager. - - Args: - components: Optional list of all component names. If not provided, - components will be discovered from configure() calls. - default_colorscale: Default colormap for ungrouped components. - If None, uses CONFIG.Plotting.default_qualitative_colorscale. - """ - self.components = sorted(set(components)) if components else [] - self.default_colorscale = default_colorscale or CONFIG.Plotting.default_qualitative_colorscale - self.color_families = self.DEFAULT_FAMILIES.copy() - - # Computed colors: {component_name: color} - self._component_colors: dict[str, str] = {} - - # Variable color cache for performance: {variable_name: color} - self._variable_cache: dict[str, str] = {} - - # Auto-assign default colors if components provided - if self.components: - self._assign_default_colors() - - def __repr__(self) -> str: - return ( - f'ComponentColorManager(components={len(self.components)}, ' - f'colors_configured={len(self._component_colors)}, ' - f"default_colorscale='{self.default_colorscale}')" - ) - - def __str__(self) -> str: - lines = [ - 'ComponentColorManager', - f' Components: {len(self.components)}', - ] - - # Show first few components as examples - if self.components: - sample = self.components[:5] - if len(self.components) > 5: - sample_str = ', '.join(sample) + f', ... ({len(self.components) - 5} more)' - else: - sample_str = ', '.join(sample) - lines.append(f' [{sample_str}]') - - lines.append(f' Colors configured: {len(self._component_colors)}') - if self._component_colors: - for comp, color in list(self._component_colors.items())[:3]: - lines.append(f' - {comp}: {color}') - if len(self._component_colors) > 3: - lines.append(f' ... and {len(self._component_colors) - 3} more') - - lines.append(f' Default colormap: {self.default_colorscale}') - - return '\n'.join(lines) - - @classmethod - def from_flow_system(cls, flow_system, **kwargs): - """Create ComponentColorManager from a FlowSystem.""" - from .flow_system import FlowSystem - - if not isinstance(flow_system, FlowSystem): - raise TypeError(f'Expected FlowSystem, got {type(flow_system).__name__}') - - # Extract component names - components = list(flow_system.components.keys()) - - return cls(components=components, **kwargs) - - def configure(self, config: dict[str, str | list[str]] | str | pathlib.Path) -> ComponentColorManager: - """Configure component colors from dict or YAML file. - - Args: - config: Dict with 'component': 'color' or 'colorscale': ['comp1', 'comp2'], - or path to YAML file with same format. - """ - # Load from file if path provided - if isinstance(config, (str, pathlib.Path)): - config = self._load_config_from_file(config) - - if not isinstance(config, dict): - raise TypeError(f'Config must be dict or file path, got {type(config).__name__}') - - # Process config: distinguish between direct colors and grouped colors - for key, value in config.items(): - if isinstance(value, str): - # Direct assignment: component → color - self._component_colors[key] = value - # Add to components list if not already there - if key not in self.components: - self.components.append(key) - self.components.sort() - - elif isinstance(value, list): - # Group assignment: colorscale → [components] - colorscale_name = key - components = value - - # Sample N colors from the colorscale - colors = self._sample_colors_from_colorscale(colorscale_name, len(components)) - - # Assign each component a color - for component, color in zip(components, colors, strict=False): - self._component_colors[component] = color - # Add to components list if not already there - if component not in self.components: - self.components.append(component) - self.components.sort() - - else: - raise TypeError( - f'Invalid config value type for key "{key}". ' - f'Expected str (color) or list[str] (components), got {type(value).__name__}' - ) - - # Clear cache since colors changed - self._variable_cache.clear() - - return self - - def get_color(self, component: str) -> str: - """Get color for a component (defaults to grey if unknown).""" - return self._component_colors.get(component, '#808080') - - def extract_component(self, variable: str) -> str: - """Extract component name from variable name (e.g., 'Boiler1(Bus_A)|flow' → 'Boiler1').""" - component, _ = self._extract_component_and_flow(variable) - return component - - def _extract_component_and_flow(self, variable: str) -> tuple[str, str | None]: - # Try "Component(Flow)|attribute" format - if '(' in variable and ')' in variable: - component = variable.split('(')[0] - flow = variable.split('(')[1].split(')')[0] - return component, flow - - # Try "Component|attribute" format (no flow) - if '|' in variable: - return variable.split('|')[0], None - - # Just the component name itself - return variable, None - - def get_variable_color(self, variable: str) -> str: - """Get color for a variable (extracts component automatically).""" - # Check cache first - if variable in self._variable_cache: - return self._variable_cache[variable] - - # Extract component name from variable - component = self.extract_component(variable) - - # Get color for component - color = self.get_color(component) - - # Cache and return - self._variable_cache[variable] = color - return color - - def get_variable_colors(self, variables: list[str]) -> dict[str, str]: - """Get colors for multiple variables (main API for plotting functions).""" - return {var: self.get_variable_color(var) for var in variables} - - def to_dict(self) -> dict[str, str]: - """Get complete component→color mapping.""" - return self._component_colors.copy() - - # ==================== INTERNAL METHODS ==================== - - def _assign_default_colors(self) -> None: - colors = self._sample_colors_from_colorscale(self.default_colorscale, len(self.components)) - - for component, color in zip(self.components, colors, strict=False): - self._component_colors[component] = color - - @staticmethod - def _load_config_from_file(file_path: str | pathlib.Path) -> dict[str, str | list[str]]: - """Load color configuration from YAML file.""" - file_path = pathlib.Path(file_path) - - if not file_path.exists(): - raise FileNotFoundError(f'Color configuration file not found: {file_path}') - - # Only support YAML - suffix = file_path.suffix.lower() - if suffix not in ['.yaml', '.yml']: - raise ValueError(f'Unsupported file format: {suffix}. Only YAML (.yaml, .yml) is supported.') - - try: - import yaml - except ImportError as e: - raise ImportError( - 'PyYAML is required to load YAML config files. Install it with: pip install pyyaml' - ) from e - - with open(file_path, encoding='utf-8') as f: - config = yaml.safe_load(f) - - # Validate config structure - if not isinstance(config, dict): - raise ValueError(f'Invalid config file structure. Expected dict, got {type(config).__name__}') - - return config - - def _sample_colors_from_colorscale(self, colorscale_name: str, num_colors: int) -> list[str]: - # Check custom families first (ComponentColorManager-specific feature) - if colorscale_name in self.color_families: - color_list = self.color_families[colorscale_name] - # Cycle through colors if needed - if len(color_list) >= num_colors: - return color_list[:num_colors] - else: - return [color_list[i % len(color_list)] for i in range(num_colors)] - - # Delegate to the simplified color processing (handles qualitative, sequential, fallbacks, cycling) - # Use dummy labels since we just need the color values - dummy_labels = [f'_{i}' for i in range(num_colors)] - color_map = process_colors(colorscale_name, dummy_labels, default_colorscale=self.default_colorscale) - return list(color_map.values()) - - def _ensure_dataset(data: xr.Dataset | pd.DataFrame | pd.Series) -> xr.Dataset: """Convert DataFrame or Series to Dataset if needed.""" if isinstance(data, xr.Dataset): From e7b0a1ea25b6eae02236f9f246b0d61f41a66dea Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:21:06 +0200 Subject: [PATCH 08/48] Update type hints --- flixopt/results.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/flixopt/results.py b/flixopt/results.py index 81bc812f2..39f60ff25 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -720,7 +720,7 @@ def plot_heatmap( variable_name: str | list[str], save: bool | pathlib.Path = False, show: bool = True, - colors: plotting.ColorType = 'turbo', + colors: plotting.ColorType | None = None, engine: plotting.PlottingEngine = 'plotly', select: dict[FlowSystemDimensions, Any] | None = None, facet_by: str | list[str] | None = 'scenario', @@ -1025,7 +1025,7 @@ def plot_node_balance( self, save: bool | pathlib.Path = False, show: bool = True, - colors: plotting.ColorType = 'turbo', + colors: plotting.ColorType | None = None, engine: plotting.PlottingEngine = 'plotly', select: dict[FlowSystemDimensions, Any] | None = None, unit_type: Literal['flow_rate', 'flow_hours'] = 'flow_rate', @@ -1214,7 +1214,7 @@ def plot_node_balance( def plot_node_balance_pie( self, lower_percentage_group: float = 5, - colors: plotting.ColorType = 'turbo', + colors: plotting.ColorType | None = None, text_info: str = 'percent+label+value', save: bool | pathlib.Path = False, show: bool = True, @@ -1481,7 +1481,7 @@ def plot_charge_state( self, save: bool | pathlib.Path = False, show: bool = True, - colors: plotting.ColorType = 'turbo', + colors: plotting.ColorType | None = None, engine: plotting.PlottingEngine = 'plotly', mode: Literal['area', 'stacked_bar', 'line'] = 'area', select: dict[FlowSystemDimensions, Any] | None = None, @@ -2099,7 +2099,7 @@ def plot_heatmap( data: xr.DataArray | xr.Dataset, name: str | None = None, folder: pathlib.Path | None = None, - colors: plotting.ColorType = 'turbo', + colors: plotting.ColorType | None = None, save: bool | pathlib.Path = False, show: bool = True, engine: plotting.PlottingEngine = 'plotly', @@ -2186,12 +2186,10 @@ def plot_heatmap( reshape_time = (heatmap_timeframes, heatmap_timesteps_per_frame) # Handle deprecated color_map parameter - if color_map is not None: - # Check for conflict with new parameter - if colors != 'turbo': # User explicitly set colors - raise ValueError( - "Cannot use both deprecated parameter 'color_map' and new parameter 'colors'. Use only 'colors'." - ) + if color_map is not None and colors is not None: # User explicitly set colors + raise ValueError( + "Cannot use both deprecated parameter 'color_map' and new parameter 'colors'. Use only 'colors'." + ) import warnings From cabe8be01760ff8df74eea6f87988df0d5da6569 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:26:28 +0200 Subject: [PATCH 09/48] Update type hints and use Config defaults --- flixopt/results.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/flixopt/results.py b/flixopt/results.py index 39f60ff25..50d1afabb 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -719,13 +719,13 @@ def plot_heatmap( self, variable_name: str | list[str], save: bool | pathlib.Path = False, - show: bool = True, + show: bool | None = None, colors: plotting.ColorType | None = None, engine: plotting.PlottingEngine = 'plotly', select: dict[FlowSystemDimensions, Any] | None = None, facet_by: str | list[str] | None = 'scenario', animate_by: str | None = 'period', - facet_cols: int = 3, + facet_cols: int | None = None, reshape_time: tuple[Literal['YS', 'MS', 'W', 'D', 'h', '15min', 'min'], Literal['W', 'D', 'h', '15min', 'min']] | Literal['auto'] | None = 'auto', @@ -1024,7 +1024,7 @@ def __init__( def plot_node_balance( self, save: bool | pathlib.Path = False, - show: bool = True, + show: bool | None = None, colors: plotting.ColorType | None = None, engine: plotting.PlottingEngine = 'plotly', select: dict[FlowSystemDimensions, Any] | None = None, @@ -1033,7 +1033,7 @@ def plot_node_balance( drop_suffix: bool = True, facet_by: str | list[str] | None = 'scenario', animate_by: str | None = 'period', - facet_cols: int = 3, + facet_cols: int | None = None, # Deprecated parameter (kept for backwards compatibility) indexer: dict[FlowSystemDimensions, Any] | None = None, **plot_kwargs: Any, @@ -1217,7 +1217,7 @@ def plot_node_balance_pie( colors: plotting.ColorType | None = None, text_info: str = 'percent+label+value', save: bool | pathlib.Path = False, - show: bool = True, + show: bool | None = None, engine: plotting.PlottingEngine = 'plotly', select: dict[FlowSystemDimensions, Any] | None = None, # Deprecated parameter (kept for backwards compatibility) @@ -1480,14 +1480,14 @@ def charge_state(self) -> xr.DataArray: def plot_charge_state( self, save: bool | pathlib.Path = False, - show: bool = True, + show: bool | None = None, colors: plotting.ColorType | None = None, engine: plotting.PlottingEngine = 'plotly', mode: Literal['area', 'stacked_bar', 'line'] = 'area', select: dict[FlowSystemDimensions, Any] | None = None, facet_by: str | list[str] | None = 'scenario', animate_by: str | None = 'period', - facet_cols: int = 3, + facet_cols: int | None = None, # Deprecated parameter (kept for backwards compatibility) indexer: dict[FlowSystemDimensions, Any] | None = None, **plot_kwargs: Any, @@ -1968,13 +1968,13 @@ def plot_heatmap( reshape_time: tuple[Literal['YS', 'MS', 'W', 'D', 'h', '15min', 'min'], Literal['W', 'D', 'h', '15min', 'min']] | Literal['auto'] | None = 'auto', - colors: str = 'portland', + colors: plotting.ColorType | None = None, save: bool | pathlib.Path = False, - show: bool = True, + show: bool | None = None, engine: plotting.PlottingEngine = 'plotly', facet_by: str | list[str] | None = None, animate_by: str | None = None, - facet_cols: int = 3, + facet_cols: int | None = None, fill: Literal['ffill', 'bfill'] | None = 'ffill', # Deprecated parameters (kept for backwards compatibility) heatmap_timeframes: Literal['YS', 'MS', 'W', 'D', 'h', '15min', 'min'] | None = None, @@ -2039,7 +2039,7 @@ def plot_heatmap( if color_map is not None: # Check for conflict with new parameter - if colors != 'portland': # Check if user explicitly set colors + if colors is not None: # Check if user explicitly set colors raise ValueError( "Cannot use both deprecated parameter 'color_map' and new parameter 'colors'. Use only 'colors'." ) @@ -2101,12 +2101,12 @@ def plot_heatmap( folder: pathlib.Path | None = None, colors: plotting.ColorType | None = None, save: bool | pathlib.Path = False, - show: bool = True, + show: bool | None = None, engine: plotting.PlottingEngine = 'plotly', select: dict[str, Any] | None = None, facet_by: str | list[str] | None = None, animate_by: str | None = None, - facet_cols: int = 3, + facet_cols: int | None = None, reshape_time: tuple[Literal['YS', 'MS', 'W', 'D', 'h', '15min', 'min'], Literal['W', 'D', 'h', '15min', 'min']] | Literal['auto'] | None = 'auto', From 94c16ba86cbda02681b54bade05cc53ab8e469a1 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:26:43 +0200 Subject: [PATCH 10/48] Add stable colors --- flixopt/results.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/flixopt/results.py b/flixopt/results.py index 50d1afabb..9b5f87218 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -254,6 +254,8 @@ def __init__( self._sizes = None self._effects_per_component = None + self.colors: dict[str, str] | None = None + def __getitem__(self, key: str) -> ComponentResults | BusResults | EffectResults: if key in self.components: return self.components[key] @@ -320,6 +322,72 @@ def flow_system(self) -> FlowSystem: logger.level = old_level return self._flow_system + def setup_colors( + self, + config: dict[str, str | list[str]] | str | pathlib.Path | None = None, + default_colorscale: str | None = None, + ) -> plotting.ComponentColorManager: + """Initialize and return a ColorManager for configuring plot colors. + + Convenience method that creates a ComponentColorManager with all components + registered and assigns it to `self.color_manager`. Optionally load configuration + from a dict or file. + + Args: + config: Optional color configuration: + - dict: Mixed {component: color} or {colorscale: [components]} mapping + - str/Path: Path to YAML file + - None: Create empty manager for manual config (default) + default_colorscale: Optional default colorscale to use. Defaults to CONFIG.Plotting.default_default_qualitative_colorscale + + Returns: + ComponentColorManager instance ready for configuration. + + Examples: + Dict-based configuration (mixed direct + grouped): + + ```python + results.setup_colors( + { + # Direct colors + 'Boiler1': '#FF0000', + 'CHP': 'darkred', + # Grouped colors + 'oranges': ['Solar1', 'Solar2'], + 'blues': ['Wind1', 'Wind2'], + 'greens': ['Battery1', 'Battery2', 'Battery3'], + } + ) + results['ElectricityBus'].plot_node_balance() + ``` + + Load from YAML file: + + ```python + # colors.yaml contains: + # Boiler1: '#FF0000' + # oranges: + # - Solar1 + # - Solar2 + results.setup_colors('colors.yaml') + ``` + + Disable automatic coloring: + + ```python + results.color_manager = None # Plots use default colorscales + ``` + """ + self.color_manager = plotting.ComponentColorManager.from_flow_system( + self.flow_system, default_colorscale=default_colorscale + ) + + # Apply configuration if provided + if config is not None: + self.color_manager.configure(config) + + return self.color_manager + def filter_solution( self, variable_dims: Literal['scalar', 'time', 'scenario', 'timeonly', 'scenarioonly'] | None = None, From 34650054721a2629042cb0d78c7647f14cc54f46 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:56:13 +0200 Subject: [PATCH 11/48] V1 --- flixopt/results.py | 104 ++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/flixopt/results.py b/flixopt/results.py index 9b5f87218..2dcae93ab 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -15,6 +15,7 @@ from . import io as fx_io from . import plotting +from .color_processing import process_colors from .flow_system import FlowSystem if TYPE_CHECKING: @@ -326,67 +327,72 @@ def setup_colors( self, config: dict[str, str | list[str]] | str | pathlib.Path | None = None, default_colorscale: str | None = None, - ) -> plotting.ComponentColorManager: - """Initialize and return a ColorManager for configuring plot colors. - - Convenience method that creates a ComponentColorManager with all components - registered and assigns it to `self.color_manager`. Optionally load configuration - from a dict or file. + ) -> dict[str, str]: + """ + Setup colors for all variables across all elements. Args: - config: Optional color configuration: - - dict: Mixed {component: color} or {colorscale: [components]} mapping - - str/Path: Path to YAML file - - None: Create empty manager for manual config (default) - default_colorscale: Optional default colorscale to use. Defaults to CONFIG.Plotting.default_default_qualitative_colorscale + config: Optional configuration dictionary mapping colors/colorscales to Components: + - 'color_name': 'component1' # Single color to single object + - 'color_name': ['component1', 'component1'] # Single color to multiple objects + - 'colorscale_name': ['component1', 'component1'] # Colorscale across objects + default_colorscale: Default colorscale for unconfigured objects + + Examples: + setup_colors({ + # Direct colors + 'Boiler1': '#FF0000', + 'CHP': 'darkred', + # Grouped by colorscale + 'oranges': ['Solar1', 'Solar2'], + 'blues': ['Wind1', 'Wind2'], + 'greens': ['Battery1', 'Battery2', 'Battery3'], + }) Returns: - ComponentColorManager instance ready for configuration. + Complete variable-to-color mapping dictionary + """ - Examples: - Dict-based configuration (mixed direct + grouped): + def get_all_variable_names(comp: str) -> list[str]: + # Collect all variables from the component, including its own name and the name of its flows, and flow_hours + comp_object = self.components[comp] + var_names = [comp] + list(comp_object._variable_names) + for flow in comp_object.flows: + var_names.extend([flow, f'{flow}|flow_hours']) - ```python - results.setup_colors( - { - # Direct colors - 'Boiler1': '#FF0000', - 'CHP': 'darkred', - # Grouped colors - 'oranges': ['Solar1', 'Solar2'], - 'blues': ['Wind1', 'Wind2'], - 'greens': ['Battery1', 'Battery2', 'Battery3'], - } - ) - results['ElectricityBus'].plot_node_balance() - ``` + return var_names - Load from YAML file: + config = config or {} - ```python - # colors.yaml contains: - # Boiler1: '#FF0000' - # oranges: - # - Solar1 - # - Solar2 - results.setup_colors('colors.yaml') - ``` + # Track which objects have been configured + configured_components = set() - Disable automatic coloring: + # Process each configuration entry + for color_spec, component_spec in config.items(): + components: list[str] = [component_spec] if isinstance(component_spec, str) else list(component_spec) - ```python - results.color_manager = None # Plots use default colorscales - ``` - """ - self.color_manager = plotting.ComponentColorManager.from_flow_system( - self.flow_system, default_colorscale=default_colorscale - ) + configured_components.update(components) + + # Collect all variables from these objects + all_variables = [] + for component in components: + if component not in self.components: + raise ValueError(f"Component '{component}' not found") + all_variables.extend(get_all_variable_names(component)) + + # Use process_colors to assign colors to these variables + color_mapping = process_colors(color_spec, all_variables) + self.colors.update(color_mapping) - # Apply configuration if provided - if config is not None: - self.color_manager.configure(config) + # Assign defaults to remaining objects + remaining_components = set(self.components.keys()) - configured_components + if remaining_components: + all_remaining_variables = [] + for remaining_component in remaining_components: + all_remaining_variables.extend(get_all_variable_names(remaining_component)) + self.colors.update(process_colors(default_colorscale, all_remaining_variables)) - return self.color_manager + return self.colors def filter_solution( self, From f2848fcaa7ab306021ae2fd3a06762506a91a2d7 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 14:06:56 +0200 Subject: [PATCH 12/48] V2 --- flixopt/results.py | 104 ++++++++++++++++++++++++++++++++------------- 1 file changed, 75 insertions(+), 29 deletions(-) diff --git a/flixopt/results.py b/flixopt/results.py index 2dcae93ab..7a45a7b56 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -16,6 +16,7 @@ from . import io as fx_io from . import plotting from .color_processing import process_colors +from .config import CONFIG from .flow_system import FlowSystem if TYPE_CHECKING: @@ -332,21 +333,25 @@ def setup_colors( Setup colors for all variables across all elements. Args: - config: Optional configuration dictionary mapping colors/colorscales to Components: - - 'color_name': 'component1' # Single color to single object - - 'color_name': ['component1', 'component1'] # Single color to multiple objects - - 'colorscale_name': ['component1', 'component1'] # Colorscale across objects - default_colorscale: Default colorscale for unconfigured objects + config: Configuration for color assignment. Can be: + - dict: Maps colors/colorscales to component(s): + * 'color_name': 'component1' # Single color to single component + * 'color_name': ['component1', 'component2'] # Single color to multiple components + * 'colorscale_name': ['component1', 'component2'] # Colorscale across components + - str: Path to a JSON/YAML config file or a colorscale name to apply to all + - Path: Path to a JSON/YAML config file + - None: Use default_colorscale for all components + default_colorscale: Default colorscale for unconfigured components (default: 'turbo') Examples: setup_colors({ # Direct colors - 'Boiler1': '#FF0000', - 'CHP': 'darkred', + '#FF0000': 'Boiler1', + 'darkred': 'CHP', # Grouped by colorscale - 'oranges': ['Solar1', 'Solar2'], - 'blues': ['Wind1', 'Wind2'], - 'greens': ['Battery1', 'Battery2', 'Battery3'], + 'Oranges': ['Solar1', 'Solar2'], + 'Blues': ['Wind1', 'Wind2'], + 'Greens': ['Battery1', 'Battery2', 'Battery3'], }) Returns: @@ -354,43 +359,84 @@ def setup_colors( """ def get_all_variable_names(comp: str) -> list[str]: - # Collect all variables from the component, including its own name and the name of its flows, and flow_hours + """Collect all variables from the component, including flows and flow_hours.""" comp_object = self.components[comp] var_names = [comp] + list(comp_object._variable_names) for flow in comp_object.flows: var_names.extend([flow, f'{flow}|flow_hours']) - return var_names - config = config or {} + # Set default colorscale if not provided + if default_colorscale is None: + default_colorscale = CONFIG.Plotting.default_qualitative_colorscale + + # Handle different config input types + if config is None: + # Apply default colorscale to all components + config_dict = {} + elif isinstance(config, (str, pathlib.Path)): + # Try to load from file first + config_path = pathlib.Path(config) + if config_path.exists(): + # Load config from file + import json + + try: + with open(config_path) as f: + config_dict = json.load(f) + except json.JSONDecodeError: + # Try YAML if available + try: + import yaml + + with open(config_path) as f: + config_dict = yaml.safe_load(f) + except (ImportError, Exception): + raise ValueError(f'Could not load config from {config_path}') from None + else: + # Treat as colorscale name to apply to all components + all_components = list(self.components.keys()) + config_dict = {config: all_components} + elif isinstance(config, dict): + config_dict = config + else: + raise TypeError(f'config must be dict, str, Path, or None, got {type(config)}') + + # Step 1: Build component-to-color mapping + component_colors: dict[str, str] = {} - # Track which objects have been configured + # Track which components are configured configured_components = set() # Process each configuration entry - for color_spec, component_spec in config.items(): + for color_spec, component_spec in config_dict.items(): + # Normalize to list of component names components: list[str] = [component_spec] if isinstance(component_spec, str) else list(component_spec) - configured_components.update(components) - - # Collect all variables from these objects - all_variables = [] + # Validate components exist for component in components: if component not in self.components: raise ValueError(f"Component '{component}' not found") - all_variables.extend(get_all_variable_names(component)) - # Use process_colors to assign colors to these variables - color_mapping = process_colors(color_spec, all_variables) - self.colors.update(color_mapping) + configured_components.update(components) + + # Get colors for these components using process_colors + colors_for_components = process_colors(color_spec, components) + component_colors.update(colors_for_components) - # Assign defaults to remaining objects - remaining_components = set(self.components.keys()) - configured_components + # Step 2: Assign colors to remaining unconfigured components + remaining_components = list(set(self.components.keys()) - configured_components) if remaining_components: - all_remaining_variables = [] - for remaining_component in remaining_components: - all_remaining_variables.extend(get_all_variable_names(remaining_component)) - self.colors.update(process_colors(default_colorscale, all_remaining_variables)) + # Use default colorscale to assign one color per remaining component + default_colors = process_colors(default_colorscale, remaining_components) + component_colors.update(default_colors) + + # Step 3: Build variable-to-color mapping + # Each component's variables all get the same color as the component + for component, color in component_colors.items(): + variable_names = get_all_variable_names(component) + for var_name in variable_names: + self.colors[var_name] = color return self.colors From 2bc06246e0d22dcd7742ba222a7bf920f7467df5 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 14:10:44 +0200 Subject: [PATCH 13/48] Use calculation.colors if direct colors is None --- flixopt/results.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/flixopt/results.py b/flixopt/results.py index 7a45a7b56..94eb5f52f 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -256,7 +256,7 @@ def __init__( self._sizes = None self._effects_per_component = None - self.colors: dict[str, str] | None = None + self.colors: dict[str, str] = {} def __getitem__(self, key: str) -> ComponentResults | BusResults | EffectResults: if key in self.components: @@ -961,7 +961,7 @@ def plot_heatmap( data=self.solution[variable_name], name=variable_name if isinstance(variable_name, str) else None, folder=self.folder, - colors=colors, + colors=self.colors or colors, save=save, show=show, engine=engine, @@ -1303,7 +1303,7 @@ def plot_node_balance( ds, facet_by=facet_by, animate_by=animate_by, - colors=colors, + colors=self._calculation_results.colors or colors, mode=mode, title=title, facet_cols=facet_cols, @@ -1314,7 +1314,7 @@ def plot_node_balance( else: figure_like = plotting.with_matplotlib( ds, - colors=colors, + colors=self._calculation_results.colors or colors, mode=mode, title=title, **plot_kwargs, @@ -1471,7 +1471,7 @@ def plot_node_balance_pie( figure_like = plotting.dual_pie_with_plotly( data_left=inputs, data_right=outputs, - colors=colors, + colors=self._calculation_results.colors or colors, title=title, text_info=text_info, subtitles=('Inputs', 'Outputs'), @@ -1485,7 +1485,7 @@ def plot_node_balance_pie( figure_like = plotting.dual_pie_with_matplotlib( data_left=inputs.to_pandas(), data_right=outputs.to_pandas(), - colors=colors, + colors=self._calculation_results.colors or colors, title=title, subtitles=('Inputs', 'Outputs'), legend_title='Flows', @@ -1721,7 +1721,7 @@ def plot_charge_state( ds, facet_by=facet_by, animate_by=animate_by, - colors=colors, + colors=self._calculation_results.colors or colors, mode=mode, title=title, facet_cols=facet_cols, @@ -1737,7 +1737,7 @@ def plot_charge_state( charge_state_ds, facet_by=facet_by, animate_by=animate_by, - colors=colors, + colors=self._calculation_results.colors or colors, mode='line', # Always line for charge_state title='', # No title needed for this temp figure facet_cols=facet_cols, @@ -1777,7 +1777,7 @@ def plot_charge_state( # For matplotlib, plot flows (node balance), then add charge_state as line fig, ax = plotting.with_matplotlib( ds, - colors=colors, + colors=self._calculation_results.colors or colors, mode=mode, title=title, **plot_kwargs, @@ -2178,7 +2178,7 @@ def plot_heatmap( name=variable_name, folder=self.folder, reshape_time=reshape_time, - colors=colors, + colors=self._calculation_results.colors or colors, save=save, show=show, engine=engine, From 472cf1ceb1745f4c86704f6f9431e63b1058eac1 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 14:10:57 +0200 Subject: [PATCH 14/48] Bugfix --- flixopt/results.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/flixopt/results.py b/flixopt/results.py index 94eb5f52f..9d10285c2 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -2306,10 +2306,11 @@ def plot_heatmap( reshape_time = (heatmap_timeframes, heatmap_timesteps_per_frame) # Handle deprecated color_map parameter - if color_map is not None and colors is not None: # User explicitly set colors - raise ValueError( - "Cannot use both deprecated parameter 'color_map' and new parameter 'colors'. Use only 'colors'." - ) + if color_map is not None: + if colors is not None: # User explicitly set colors + raise ValueError( + "Cannot use both deprecated parameter 'color_map' and new parameter 'colors'. Use only 'colors'." + ) import warnings From 7f790e46bb3231d2db99d9beea610d3132539ca8 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 14:16:03 +0200 Subject: [PATCH 15/48] Bugfix --- flixopt/results.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flixopt/results.py b/flixopt/results.py index 9d10285c2..c0b292780 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -961,7 +961,7 @@ def plot_heatmap( data=self.solution[variable_name], name=variable_name if isinstance(variable_name, str) else None, folder=self.folder, - colors=self.colors or colors, + colors=colors, save=save, show=show, engine=engine, @@ -2178,7 +2178,7 @@ def plot_heatmap( name=variable_name, folder=self.folder, reshape_time=reshape_time, - colors=self._calculation_results.colors or colors, + colors=colors, save=save, show=show, engine=engine, From 72b2a2cf23ac63e67d8a57fd5652d7188d90eb16 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 14:42:07 +0200 Subject: [PATCH 16/48] Update setup_colors --- flixopt/results.py | 62 ++++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/flixopt/results.py b/flixopt/results.py index c0b292780..06fb55f09 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -334,9 +334,10 @@ def setup_colors( Args: config: Configuration for color assignment. Can be: - - dict: Maps colors/colorscales to component(s): - * 'color_name': 'component1' # Single color to single component - * 'color_name': ['component1', 'component2'] # Single color to multiple components + - dict: Maps components to colors/colorscales: + * 'component1': 'red' # Single component to single color + * 'component1': '#FF0000' # Single component to hex color + - OR maps colorscales to multiple components: * 'colorscale_name': ['component1', 'component2'] # Colorscale across components - str: Path to a JSON/YAML config file or a colorscale name to apply to all - Path: Path to a JSON/YAML config file @@ -345,10 +346,10 @@ def setup_colors( Examples: setup_colors({ - # Direct colors - '#FF0000': 'Boiler1', - 'darkred': 'CHP', - # Grouped by colorscale + # Direct component-to-color mappings + 'Boiler1': '#FF0000', + 'CHP': 'darkred', + # Colorscale for multiple components 'Oranges': ['Solar1', 'Solar2'], 'Blues': ['Wind1', 'Wind2'], 'Greens': ['Battery1', 'Battery2', 'Battery3'], @@ -409,20 +410,39 @@ def get_all_variable_names(comp: str) -> list[str]: configured_components = set() # Process each configuration entry - for color_spec, component_spec in config_dict.items(): - # Normalize to list of component names - components: list[str] = [component_spec] if isinstance(component_spec, str) else list(component_spec) - - # Validate components exist - for component in components: - if component not in self.components: - raise ValueError(f"Component '{component}' not found") - - configured_components.update(components) - - # Get colors for these components using process_colors - colors_for_components = process_colors(color_spec, components) - component_colors.update(colors_for_components) + for key, value in config_dict.items(): + # Check if value is a list (colorscale -> [components]) + # or a string (component -> color OR colorscale -> [components]) + + if isinstance(value, list): + # key is colorscale, value is list of components + # Format: 'Blues': ['Wind1', 'Wind2'] + components = value + colorscale_name = key + + # Validate components exist + for component in components: + if component not in self.components: + raise ValueError(f"Component '{component}' not found") + + configured_components.update(components) + + # Use process_colors to get one color per component from the colorscale + colors_for_components = process_colors(colorscale_name, components) + component_colors.update(colors_for_components) + + elif isinstance(value, str): + # Check if key is an existing component + if key in self.components: + # Format: 'CHP': 'red' (component -> color) + component, color = key, value + + configured_components.add(component) + component_colors[component] = color + else: + raise ValueError(f"Component '{key}' not found") + else: + raise TypeError(f'Config value must be str or list, got {type(value)}') # Step 2: Assign colors to remaining unconfigured components remaining_components = list(set(self.components.keys()) - configured_components) From 3fcdbffeccac2b9a0c30345390f79a8704cabd73 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 14:48:14 +0200 Subject: [PATCH 17/48] Add color setup to examples --- examples/01_Simple/simple_example.py | 3 +++ examples/04_Scenarios/scenario_example.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/examples/01_Simple/simple_example.py b/examples/01_Simple/simple_example.py index 906c24622..6b62d6712 100644 --- a/examples/01_Simple/simple_example.py +++ b/examples/01_Simple/simple_example.py @@ -112,6 +112,9 @@ calculation.solve(fx.solvers.HighsSolver(mip_gap=0, time_limit_seconds=30)) # --- Analyze Results --- + # Colors are automatically assigned using default colormap + # Optional: Configure custom colors with + calculation.results.setup_colors() calculation.results['Fernwärme'].plot_node_balance_pie() calculation.results['Fernwärme'].plot_node_balance() calculation.results['Storage'].plot_charge_state() diff --git a/examples/04_Scenarios/scenario_example.py b/examples/04_Scenarios/scenario_example.py index 834e55782..d258d4142 100644 --- a/examples/04_Scenarios/scenario_example.py +++ b/examples/04_Scenarios/scenario_example.py @@ -196,6 +196,15 @@ # --- Solve the Calculation and Save Results --- calculation.solve(fx.solvers.HighsSolver(mip_gap=0, time_limit_seconds=30)) + calculation.results.setup_colors( + { + 'CHP': 'red', + 'Greys': ['Gastarif', 'Einspeisung', 'Heat Demand'], + 'Storage': 'blue', + 'Boiler': 'orange', + } + ) + calculation.results.plot_heatmap('CHP(Q_th)|flow_rate') # --- Analyze Results --- From 47407631cf119d229883ace6af0ea45608699f94 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 14:54:50 +0200 Subject: [PATCH 18/48] Final touches --- flixopt/aggregation.py | 11 ++++++++--- flixopt/color_processing.py | 9 --------- flixopt/plotting.py | 13 ++++++------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/flixopt/aggregation.py b/flixopt/aggregation.py index 427937596..cd0fdde3c 100644 --- a/flixopt/aggregation.py +++ b/flixopt/aggregation.py @@ -20,7 +20,9 @@ except ImportError: TSAM_AVAILABLE = False +from .color_processing import process_colors from .components import Storage +from .config import CONFIG from .structure import ( FlowSystemModel, Submodel, @@ -141,7 +143,7 @@ def describe_clusters(self) -> str: def use_extreme_periods(self): return self.time_series_for_high_peaks or self.time_series_for_low_peaks - def plot(self, colormap: str = 'turbo', show: bool = True, save: pathlib.Path | None = None) -> go.Figure: + def plot(self, colormap: str | None = None, show: bool = True, save: pathlib.Path | None = None) -> go.Figure: from . import plotting df_org = self.original_data.copy().rename( @@ -150,10 +152,13 @@ def plot(self, colormap: str = 'turbo', show: bool = True, save: pathlib.Path | df_agg = self.aggregated_data.copy().rename( columns={col: f'Aggregated - {col}' for col in self.aggregated_data.columns} ) - fig = plotting.with_plotly(df_org.to_xarray(), 'line', colors=colormap, xlabel='Time in h') + colors = list( + process_colors(colormap or CONFIG.Plotting.default_qualitative_colorscale, list(df_org.columns)).values() + ) + fig = plotting.with_plotly(df_org.to_xarray(), 'line', colors=colors, xlabel='Time in h') for trace in fig.data: trace.update(dict(line=dict(dash='dash'))) - fig2 = plotting.with_plotly(df_agg.to_xarray(), 'line', colors=colormap, xlabel='Time in h') + fig2 = plotting.with_plotly(df_agg.to_xarray(), 'line', colors=colors, xlabel='Time in h') for trace in fig2.data: fig.add_trace(trace) diff --git a/flixopt/color_processing.py b/flixopt/color_processing.py index 366a5e71c..12d5a976b 100644 --- a/flixopt/color_processing.py +++ b/flixopt/color_processing.py @@ -105,15 +105,6 @@ def process_colors( raise TypeError(f'colors must be None, str, list, or dict, got {type(colors)}') -def resolve_colors( - labels: list[str], - colors: None | str | list[str] | dict[str, str], - default_colorscale: str = 'turbo', -) -> dict[str, str]: - """Temporary wrapper""" - return process_colors(colors, labels, default_colorscale) - - def _fill_missing_colors( color_mapping: dict[str, str], labels: list[str], diff --git a/flixopt/plotting.py b/flixopt/plotting.py index be1a521c4..2685e4bfa 100644 --- a/flixopt/plotting.py +++ b/flixopt/plotting.py @@ -40,9 +40,8 @@ import plotly.graph_objects as go import plotly.offline import xarray as xr -from plotly.exceptions import PlotlyError -from .color_processing import process_colors, resolve_colors +from .color_processing import process_colors from .config import CONFIG if TYPE_CHECKING: @@ -277,7 +276,7 @@ def with_plotly( values = [float(data[var].values) for var in data.data_vars] # Resolve colors - color_discrete_map = resolve_colors( + color_discrete_map = process_colors( variables, colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale ) marker_colors = [color_discrete_map.get(var, '#636EFA') for var in variables] @@ -370,7 +369,7 @@ def with_plotly( # Process colors all_vars = df_long['variable'].unique().tolist() - color_discrete_map = resolve_colors( + color_discrete_map = process_colors( list(data.data_vars), colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale ) @@ -547,7 +546,7 @@ def with_matplotlib( values = [float(data[var].values) for var in data.data_vars] # Resolve colors - color_discrete_map = resolve_colors( + color_discrete_map = process_colors( variables, colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale ) colors_list = [color_discrete_map.get(var, '#808080') for var in variables] @@ -580,7 +579,7 @@ def with_matplotlib( return fig, ax # Resolve colors first (includes validation) - color_discrete_map = resolve_colors( + color_discrete_map = process_colors( list(data.data_vars), colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale ) @@ -1038,7 +1037,7 @@ def dual_pie_with_plotly( all_labels = sorted(set(left_labels) | set(right_labels)) # Create color map - color_map = resolve_colors(all_labels, colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale) + color_map = process_colors(all_labels, colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale) # Create figure fig = go.Figure() From 664e8ff3376e3d347a0cb910d7467e819bf3f621 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 14:58:41 +0200 Subject: [PATCH 19/48] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca9600f04..ade13a17d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,8 @@ If upgrading from v2.x, see the [v3.0.0 release notes](https://github.com/flixOp ### ✨ Added - Support for plotting kwargs in `results.py`, passed to plotly express and matplotlib. +- Added method `setup_colors()` to create a colormapping with similar colors for all variables of a Component. THis is used by default to produce plots with `CalculationResults` +- More Config options ### 💥 Breaking Changes From f6c721b5bb941ee6ebdcbdd4b14e07ec64011c1c Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 15:03:07 +0200 Subject: [PATCH 20/48] Update CHANGELOG.md --- CHANGELOG.md | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ade13a17d..94b0f5ad6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,19 +54,36 @@ If upgrading from v2.x, see the [v3.0.0 release notes](https://github.com/flixOp ### ✨ Added - Support for plotting kwargs in `results.py`, passed to plotly express and matplotlib. -- Added method `setup_colors()` to create a colormapping with similar colors for all variables of a Component. THis is used by default to produce plots with `CalculationResults` -- More Config options +- **Color management system**: New `color_processing.py` module with `process_colors()` function for unified color handling across plotting backends + - Supports flexible color inputs: colorscale names (e.g., 'turbo', 'plasma'), color lists, and label-to-color dictionaries + - Automatic fallback handling when requested colorscales are unavailable + - Seamless integration with both Plotly and Matplotlib colorscales +- **Component color grouping**: Added `CalculationResults.setup_colors()` method to create color mappings with similar colors for all variables of a component (and its flows) + - Allows grouping components by custom colorscales: `{'CHP': 'red', 'Greys': ['Gastarif', 'Einspeisung'], 'Storage': 'blue'}` + - Colors are automatically assigned using default colorscale if not specified +- **Plotting configuration**: New `CONFIG.Plotting` section with extensive customization options: + - `default_show`: Control default visibility of plots + - `default_engine`: Choose between 'plotly' or 'matplotlib' + - `default_dpi`: Configure resolution for saved plots (with matplotlib) + - `default_facet_cols`: Set default columns for faceted plots + - `default_sequential_colorscale`: Default for heatmaps and continuous data (default: 'turbo') + - `default_qualitative_colorscale`: Default for categorical plots (default: 'plotly') ### 💥 Breaking Changes ### ♻️ Changed - **Template integration**: Plotly templates now fully control plot styling without hardcoded overrides - **Dataset first plotting**: Underlying plotting methods in `plotting.py` now use `xr.Dataset` as the main datatype. DataFrames are automatically converted via `_ensure_dataset()`. Both DataFrames and Datasets can be passed to plotting functions without code changes. +- **Color terminology**: Standardized terminology from "colormap" to "colorscale" throughout the codebase for consistency with Plotly conventions +- **Default colorscales**: Changed default sequential colorscale from 'viridis' to 'turbo' for better perceptual uniformity; qualitative colorscale now defaults to 'plotly' +- **Aggregation plotting**: `Aggregation.plot()` now respects `CONFIG.Plotting.default_qualitative_colorscale` and uses `process_colors()` for consistent color handling ### 🗑️ Deprecated ### 🔥 Removed -- Removed `plotting.pie_with_plotly()` method as it was not used +- Removed `plotting.pie_with_plotly()` method as it was not used +- Removed `ColorProcessor` class - replaced by simpler `process_colors()` function +- Removed `resolve_colors()` helper function - color resolution now handled directly by `process_colors()` ### 🐛 Fixed - Improved error messages for `engine='matplotlib'` with multidimensional data @@ -78,9 +95,15 @@ If upgrading from v2.x, see the [v3.0.0 release notes](https://github.com/flixOp ### 📝 Docs - Moved `linked_periods` into correct section of the docstring (was in deprecated params) +- Updated terminology in docstrings from "colormap" to "colorscale" for consistency +- Enhanced examples to demonstrate `setup_colors()` usage: + - `simple_example.py`: Shows automatic color assignment and optional custom configuration + - `scenario_example.py`: Demonstrates component grouping with custom colorscales ### 👷 Development - Fixed concurrency issue in CI +- **Code architecture**: Extracted color processing logic into dedicated `color_processing.py` module for better separation of concerns +- Refactored from class-based (`ColorProcessor`) to function-based color handling for simpler API and reduced complexity ### 🚧 Known Issues From 59c399f921affab8c2a63e5a12cd5e6c87068dfc Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 15:03:17 +0200 Subject: [PATCH 21/48] Bugfix --- flixopt/color_processing.py | 1 - flixopt/config.py | 1 - 2 files changed, 2 deletions(-) diff --git a/flixopt/color_processing.py b/flixopt/color_processing.py index 12d5a976b..623ff3a6a 100644 --- a/flixopt/color_processing.py +++ b/flixopt/color_processing.py @@ -7,7 +7,6 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING import matplotlib.colors as mcolors import matplotlib.pyplot as plt diff --git a/flixopt/config.py b/flixopt/config.py index 89b9abd78..b7162e55f 100644 --- a/flixopt/config.py +++ b/flixopt/config.py @@ -57,7 +57,6 @@ 'plotting': MappingProxyType( { 'default_show': True, - 'default_save_path': None, 'default_engine': 'plotly', 'default_dpi': 300, 'default_facet_cols': 3, From 0fd989bc0ff826b0d8919874828bbb9056514f9e Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 15:08:44 +0200 Subject: [PATCH 22/48] Update fro SegmentedCalculationResults --- CHANGELOG.md | 3 ++- flixopt/results.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94b0f5ad6..603a5ead5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,9 +58,10 @@ If upgrading from v2.x, see the [v3.0.0 release notes](https://github.com/flixOp - Supports flexible color inputs: colorscale names (e.g., 'turbo', 'plasma'), color lists, and label-to-color dictionaries - Automatic fallback handling when requested colorscales are unavailable - Seamless integration with both Plotly and Matplotlib colorscales -- **Component color grouping**: Added `CalculationResults.setup_colors()` method to create color mappings with similar colors for all variables of a component (and its flows) +- **Component color grouping**: Added `setup_colors()` method to `CalculationResults` and `SegmentedCalculationResults` to create color mappings with similar colors for all variables of a component - Allows grouping components by custom colorscales: `{'CHP': 'red', 'Greys': ['Gastarif', 'Einspeisung'], 'Storage': 'blue'}` - Colors are automatically assigned using default colorscale if not specified + - For segmented calculations, colors are propagated to all segments for consistent visualization - **Plotting configuration**: New `CONFIG.Plotting` section with extensive customization options: - `default_show`: Control default visibility of plots - `default_engine`: Choose between 'plotly' or 'matplotlib' diff --git a/flixopt/results.py b/flixopt/results.py index 06fb55f09..28a3fcb4d 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -1,5 +1,6 @@ from __future__ import annotations +import copy import datetime import json import logging @@ -2073,6 +2074,7 @@ def __init__( self.name = name self.folder = pathlib.Path(folder) if folder is not None else pathlib.Path.cwd() / 'results' self.hours_per_timestep = FlowSystem.calculate_hours_per_timestep(self.all_timesteps) + self._colors = {} @property def meta_data(self) -> dict[str, int | list[str]]: @@ -2087,6 +2089,64 @@ def meta_data(self) -> dict[str, int | list[str]]: def segment_names(self) -> list[str]: return [segment.name for segment in self.segment_results] + @property + def colors(self) -> dict[str, str]: + return self._colors + + @colors.setter + def colors(self, colors: dict[str, str]): + """Applys colors to all segements""" + self._colors = colors + for segment in self.segment_results: + segment.colors = copy.deepcopy(colors) + + def setup_colors( + self, + config: dict[str, str | list[str]] | str | pathlib.Path | None = None, + default_colorscale: str | None = None, + ) -> dict[str, str]: + """ + Setup colors for all variables across all segment results. + + This method applies the same color configuration to all segments, ensuring + consistent visualization across the entire segmented calculation. The color + mapping is propagated to each segment's CalculationResults instance. + + Args: + config: Configuration for color assignment. Can be: + - dict: Maps components to colors/colorscales: + * 'component1': 'red' # Single component to single color + * 'component1': '#FF0000' # Single component to hex color + - OR maps colorscales to multiple components: + * 'colorscale_name': ['component1', 'component2'] # Colorscale across components + - str: Path to a JSON/YAML config file or a colorscale name to apply to all + - Path: Path to a JSON/YAML config file + - None: Use default_colorscale for all components + default_colorscale: Default colorscale for unconfigured components (default: 'turbo') + + Examples: + ```python + # Apply colors to all segments + segmented_results.setup_colors( + { + 'CHP': 'red', + 'Blues': ['Storage1', 'Storage2'], + 'Oranges': ['Solar1', 'Solar2'], + } + ) + + # Use a single colorscale for all components in all segments + segmented_results.setup_colors('portland') + ``` + + Returns: + Complete variable-to-color mapping dictionary from the first segment + (all segments will have the same mapping) + """ + self.colors = self.segment_results[0].setup_colors(config=config, default_colorscale=default_colorscale) + + return self.colors + def solution_without_overlap(self, variable_name: str) -> xr.DataArray: """Get variable solution removing segment overlaps. From 9a7b8d71ac821cf45d517c51f780172d70f1d598 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 15:14:27 +0200 Subject: [PATCH 23/48] Default show = False in tests --- tests/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index ac5255562..bd940b843 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -848,4 +848,6 @@ def set_test_environment(): pio.renderers.default = 'json' # Use non-interactive renderer + fx.CONFIG.Plotting.default_show = False + yield From c1622ff1b1b29142ee9eb8d36dfe79154422d208 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 15:31:07 +0200 Subject: [PATCH 24/48] Bugfix --- flixopt/plotting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flixopt/plotting.py b/flixopt/plotting.py index 2685e4bfa..1617b6695 100644 --- a/flixopt/plotting.py +++ b/flixopt/plotting.py @@ -370,7 +370,7 @@ def with_plotly( # Process colors all_vars = df_long['variable'].unique().tolist() color_discrete_map = process_colors( - list(data.data_vars), colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale + all_vars, colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale ) # Determine which dimension to use for x-axis From bff1ad6af69e1882f571721e53df534f934af0ba Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 15:59:12 +0200 Subject: [PATCH 25/48] Bugfix --- flixopt/plotting.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flixopt/plotting.py b/flixopt/plotting.py index 1617b6695..045cf7e99 100644 --- a/flixopt/plotting.py +++ b/flixopt/plotting.py @@ -277,7 +277,7 @@ def with_plotly( # Resolve colors color_discrete_map = process_colors( - variables, colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale + colors, variables, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale ) marker_colors = [color_discrete_map.get(var, '#636EFA') for var in variables] @@ -370,7 +370,7 @@ def with_plotly( # Process colors all_vars = df_long['variable'].unique().tolist() color_discrete_map = process_colors( - all_vars, colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale + colors, all_vars, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale ) # Determine which dimension to use for x-axis @@ -547,7 +547,7 @@ def with_matplotlib( # Resolve colors color_discrete_map = process_colors( - variables, colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale + colors, variables, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale ) colors_list = [color_discrete_map.get(var, '#808080') for var in variables] @@ -580,7 +580,7 @@ def with_matplotlib( # Resolve colors first (includes validation) color_discrete_map = process_colors( - list(data.data_vars), colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale + colors, list(data.data_vars), default_colorscale=CONFIG.Plotting.default_qualitative_colorscale ) # Convert Dataset to DataFrame for matplotlib plotting (naturally wide-form) @@ -1037,7 +1037,7 @@ def dual_pie_with_plotly( all_labels = sorted(set(left_labels) | set(right_labels)) # Create color map - color_map = process_colors(all_labels, colors, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale) + color_map = process_colors(colors, all_labels, default_colorscale=CONFIG.Plotting.default_qualitative_colorscale) # Create figure fig = go.Figure() From 4e64f52eebea857d13a95c2224715b4e45ad4c25 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 16:05:03 +0200 Subject: [PATCH 26/48] Add show default to plot_network --- flixopt/flow_system.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/flixopt/flow_system.py b/flixopt/flow_system.py index ad43c183b..fd0f6a98d 100644 --- a/flixopt/flow_system.py +++ b/flixopt/flow_system.py @@ -4,7 +4,6 @@ from __future__ import annotations -import json import logging import warnings from typing import TYPE_CHECKING, Any, Literal, Optional @@ -13,6 +12,7 @@ import pandas as pd import xarray as xr +from .config import CONFIG from .core import ( ConversionError, DataConverter, @@ -484,7 +484,7 @@ def plot_network( | list[ Literal['nodes', 'edges', 'layout', 'interaction', 'manipulation', 'physics', 'selection', 'renderer'] ] = True, - show: bool = False, + show: bool | None = None, ) -> pyvis.network.Network | None: """ Visualizes the network structure of a FlowSystem using PyVis, saving it as an interactive HTML file. @@ -514,7 +514,9 @@ def plot_network( from . import plotting node_infos, edge_infos = self.network_infos() - return plotting.plot_network(node_infos, edge_infos, path, controls, show) + return plotting.plot_network( + node_infos, edge_infos, path, controls, show if show is not None else CONFIG.Plotting.default_show + ) def start_network_app(self): """Visualizes the network structure of a FlowSystem using Dash, Cytoscape, and networkx. From 8d458b74a6da63104227727ae9b9b9c88ca87a0f Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 16:09:10 +0200 Subject: [PATCH 27/48] Make _rgb_string_to_hex more robust --- flixopt/color_processing.py | 39 ++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/flixopt/color_processing.py b/flixopt/color_processing.py index 623ff3a6a..9a370d6a2 100644 --- a/flixopt/color_processing.py +++ b/flixopt/color_processing.py @@ -17,20 +17,45 @@ def _rgb_string_to_hex(color: str) -> str: - """Convert Plotly RGB string format to hex. + """Convert Plotly RGB/RGBA string format to hex. Args: - color: Color in format 'rgb(R, G, B)' or already in hex + color: Color in format 'rgb(R, G, B)', 'rgba(R, G, B, A)' or already in hex Returns: Color in hex format '#RRGGBB' """ - if color.startswith('rgb('): - # Extract RGB values from 'rgb(R, G, B)' format - rgb_str = color[4:-1] # Remove 'rgb(' and ')' - r, g, b = map(int, rgb_str.split(',')) + color = color.strip() + + # If already hex, return as-is + if color.startswith('#'): + return color + + # Try to parse rgb() or rgba() + try: + if color.startswith('rgb('): + # Extract RGB values from 'rgb(R, G, B)' format + rgb_str = color[4:-1] # Remove 'rgb(' and ')' + elif color.startswith('rgba('): + # Extract RGBA values from 'rgba(R, G, B, A)' format + rgb_str = color[5:-1] # Remove 'rgba(' and ')' + else: + return color + + # Split on commas and parse first three components + components = rgb_str.split(',') + if len(components) < 3: + return color + + # Parse and clamp the first three components + r = max(0, min(255, int(round(float(components[0].strip()))))) + g = max(0, min(255, int(round(float(components[1].strip()))))) + b = max(0, min(255, int(round(float(components[2].strip()))))) + return f'#{r:02x}{g:02x}{b:02x}' - return color + except (ValueError, IndexError): + # If parsing fails, return original + return color def process_colors( From 9145cce744b4546db54968e3153361cd2eb82959 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 16:09:22 +0200 Subject: [PATCH 28/48] Improve Error Handling --- flixopt/color_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flixopt/color_processing.py b/flixopt/color_processing.py index 9a370d6a2..2959acc82 100644 --- a/flixopt/color_processing.py +++ b/flixopt/color_processing.py @@ -241,7 +241,7 @@ def _try_get_colorscale(colorscale_name: str, num_colors: int) -> list[str] | No colors = px.colors.sample_colorscale(colorscale, sample_points) # Convert to hex format for matplotlib compatibility return [_rgb_string_to_hex(c) for c in colors] - except PlotlyError: + except (PlotlyError, ValueError): pass # Finally try Matplotlib colorscales From 8822cd6afebfb915a4c20b4519408a513a215633 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 16:11:12 +0200 Subject: [PATCH 29/48] Overwrite colors explicitly in setup_colors --- flixopt/results.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flixopt/results.py b/flixopt/results.py index 28a3fcb4d..5355877ae 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -331,7 +331,7 @@ def setup_colors( default_colorscale: str | None = None, ) -> dict[str, str]: """ - Setup colors for all variables across all elements. + Setup colors for all variables across all elements. Overwrites existing ones. Args: config: Configuration for color assignment. Can be: @@ -453,6 +453,8 @@ def get_all_variable_names(comp: str) -> list[str]: component_colors.update(default_colors) # Step 3: Build variable-to-color mapping + # Clear existing colors to avoid stale keys + self.colors = {} # Each component's variables all get the same color as the component for component, color in component_colors.items(): variable_names = get_all_variable_names(component) @@ -2095,7 +2097,7 @@ def colors(self) -> dict[str, str]: @colors.setter def colors(self, colors: dict[str, str]): - """Applys colors to all segements""" + """Applies colors to all segments""" self._colors = colors for segment in self.segment_results: segment.colors = copy.deepcopy(colors) From e94a61c083d25bda46bde90c6b0c099eace1ae81 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 16:11:27 +0200 Subject: [PATCH 30/48] Improve config loader --- flixopt/results.py | 68 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/flixopt/results.py b/flixopt/results.py index 5355877ae..cf58d4793 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -32,6 +32,57 @@ logger = logging.getLogger('flixopt') +def load_mapping_from_file(path: pathlib.Path) -> dict[str, str | list[str]]: + """Load color mapping from JSON or YAML file. + + Tries loader based on file suffix first, with fallback to the other format. + + Args: + path: Path to config file (.json or .yaml/.yml) + + Returns: + Dictionary mapping components to colors or colorscales to component lists + + Raises: + ValueError: If file cannot be loaded as JSON or YAML + """ + suffix = path.suffix.lower() + + if suffix == '.json': + # Try JSON first, fallback to YAML + try: + with open(path) as f: + return json.load(f) + except json.JSONDecodeError: + try: + with open(path) as f: + return yaml.safe_load(f) + except Exception: + raise ValueError(f'Could not load config from {path}') from None + elif suffix in {'.yaml', '.yml'}: + # Try YAML first, fallback to JSON + try: + with open(path) as f: + return yaml.safe_load(f) + except yaml.YAMLError: + try: + with open(path) as f: + return json.load(f) + except Exception: + raise ValueError(f'Could not load config from {path}') from None + else: + # Unknown extension, try both starting with JSON + try: + with open(path) as f: + return json.load(f) + except json.JSONDecodeError: + try: + with open(path) as f: + return yaml.safe_load(f) + except Exception: + raise ValueError(f'Could not load config from {path}') from None + + class _FlowSystemRestorationError(Exception): """Exception raised when a FlowSystem cannot be restored from dataset.""" @@ -380,21 +431,8 @@ def get_all_variable_names(comp: str) -> list[str]: # Try to load from file first config_path = pathlib.Path(config) if config_path.exists(): - # Load config from file - import json - - try: - with open(config_path) as f: - config_dict = json.load(f) - except json.JSONDecodeError: - # Try YAML if available - try: - import yaml - - with open(config_path) as f: - config_dict = yaml.safe_load(f) - except (ImportError, Exception): - raise ValueError(f'Could not load config from {config_path}') from None + # Load config from file using helper + config_dict = load_mapping_from_file(config_path) else: # Treat as colorscale name to apply to all components all_components = list(self.components.keys()) From e697ac06e7ce872810fac4c26de5756cdf4fb0d2 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 16:11:48 +0200 Subject: [PATCH 31/48] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 603a5ead5..8eb16a4c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,10 +58,12 @@ If upgrading from v2.x, see the [v3.0.0 release notes](https://github.com/flixOp - Supports flexible color inputs: colorscale names (e.g., 'turbo', 'plasma'), color lists, and label-to-color dictionaries - Automatic fallback handling when requested colorscales are unavailable - Seamless integration with both Plotly and Matplotlib colorscales + - Automatic rgba→hex color conversion for Matplotlib compatibility - **Component color grouping**: Added `setup_colors()` method to `CalculationResults` and `SegmentedCalculationResults` to create color mappings with similar colors for all variables of a component - Allows grouping components by custom colorscales: `{'CHP': 'red', 'Greys': ['Gastarif', 'Einspeisung'], 'Storage': 'blue'}` - Colors are automatically assigned using default colorscale if not specified - For segmented calculations, colors are propagated to all segments for consistent visualization + - Explicit `colors` arguments in plot methods override configured colors (when provided) - **Plotting configuration**: New `CONFIG.Plotting` section with extensive customization options: - `default_show`: Control default visibility of plots - `default_engine`: Choose between 'plotly' or 'matplotlib' From a36ce890cad5cf0f85cb99c7322fc92a6a19132c Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 16:14:01 +0200 Subject: [PATCH 32/48] Make colors arg always overwrite the default behaviour --- flixopt/results.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flixopt/results.py b/flixopt/results.py index cf58d4793..847ee5a7f 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -1364,7 +1364,7 @@ def plot_node_balance( ds, facet_by=facet_by, animate_by=animate_by, - colors=self._calculation_results.colors or colors, + colors=colors if colors is not None else self._calculation_results.colors, mode=mode, title=title, facet_cols=facet_cols, @@ -1375,7 +1375,7 @@ def plot_node_balance( else: figure_like = plotting.with_matplotlib( ds, - colors=self._calculation_results.colors or colors, + colors=colors if colors is not None else self._calculation_results.colors, mode=mode, title=title, **plot_kwargs, @@ -1532,7 +1532,7 @@ def plot_node_balance_pie( figure_like = plotting.dual_pie_with_plotly( data_left=inputs, data_right=outputs, - colors=self._calculation_results.colors or colors, + colors=colors if colors is not None else self._calculation_results.colors, title=title, text_info=text_info, subtitles=('Inputs', 'Outputs'), @@ -1546,7 +1546,7 @@ def plot_node_balance_pie( figure_like = plotting.dual_pie_with_matplotlib( data_left=inputs.to_pandas(), data_right=outputs.to_pandas(), - colors=self._calculation_results.colors or colors, + colors=colors if colors is not None else self._calculation_results.colors, title=title, subtitles=('Inputs', 'Outputs'), legend_title='Flows', @@ -1782,7 +1782,7 @@ def plot_charge_state( ds, facet_by=facet_by, animate_by=animate_by, - colors=self._calculation_results.colors or colors, + colors=colors if colors is not None else self._calculation_results.colors, mode=mode, title=title, facet_cols=facet_cols, @@ -1798,7 +1798,7 @@ def plot_charge_state( charge_state_ds, facet_by=facet_by, animate_by=animate_by, - colors=self._calculation_results.colors or colors, + colors=colors if colors is not None else self._calculation_results.colors, mode='line', # Always line for charge_state title='', # No title needed for this temp figure facet_cols=facet_cols, @@ -1838,7 +1838,7 @@ def plot_charge_state( # For matplotlib, plot flows (node balance), then add charge_state as line fig, ax = plotting.with_matplotlib( ds, - colors=self._calculation_results.colors or colors, + colors=colors if colors is not None else self._calculation_results.colors, mode=mode, title=title, **plot_kwargs, From c45343b16acc947e77ee3b9c0703f60f4de98bc9 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 24 Oct 2025 16:20:25 +0200 Subject: [PATCH 33/48] centralize yaml and json io --- flixopt/config.py | 9 +-- flixopt/io.py | 164 +++++++++++++++++++++++++++++++++++++++++++ flixopt/results.py | 50 ++----------- flixopt/structure.py | 4 +- 4 files changed, 175 insertions(+), 52 deletions(-) diff --git a/flixopt/config.py b/flixopt/config.py index b7162e55f..e51c0ac8c 100644 --- a/flixopt/config.py +++ b/flixopt/config.py @@ -8,7 +8,6 @@ from types import MappingProxyType from typing import Literal -import yaml from rich.console import Console from rich.logging import RichHandler from rich.style import Style @@ -299,13 +298,15 @@ def load_from_file(cls, config_file: str | Path): Raises: FileNotFoundError: If the config file does not exist. """ + # Import here to avoid circular import + from . import io as fx_io + config_path = Path(config_file) if not config_path.exists(): raise FileNotFoundError(f'Config file not found: {config_file}') - with config_path.open() as file: - config_dict = yaml.safe_load(file) or {} - cls._apply_config_dict(config_dict) + config_dict = fx_io.load_yaml(config_path, safe=True) + cls._apply_config_dict(config_dict) cls.apply() diff --git a/flixopt/io.py b/flixopt/io.py index 53d3d8e8a..aef1fb968 100644 --- a/flixopt/io.py +++ b/flixopt/io.py @@ -34,6 +34,170 @@ def remove_none_and_empty(obj): return obj +# ============================================================================ +# Centralized JSON and YAML I/O Functions +# ============================================================================ + + +def load_json(path: str | pathlib.Path) -> dict | list: + """ + Load data from a JSON file. + + Args: + path: Path to the JSON file. + + Returns: + Loaded data (typically dict or list). + + Raises: + FileNotFoundError: If the file does not exist. + json.JSONDecodeError: If the file is not valid JSON. + """ + path = pathlib.Path(path) + with open(path, encoding='utf-8') as f: + return json.load(f) + + +def save_json( + data: dict | list, + path: str | pathlib.Path, + indent: int = 4, + ensure_ascii: bool = False, + **kwargs, +) -> None: + """ + Save data to a JSON file with consistent formatting. + + Args: + data: Data to save (dict or list). + path: Path to save the JSON file. + indent: Number of spaces for indentation (default: 4). + ensure_ascii: If False, allow Unicode characters (default: False). + **kwargs: Additional arguments to pass to json.dump(). + """ + path = pathlib.Path(path) + with open(path, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=indent, ensure_ascii=ensure_ascii, **kwargs) + + +def load_yaml(path: str | pathlib.Path, safe: bool = True) -> dict | list: + """ + Load data from a YAML file. + + Args: + path: Path to the YAML file. + safe: If True, use safe_load for security (default: True). + If False, use FullLoader (allows arbitrary Python objects). + + Returns: + Loaded data (typically dict or list), or empty dict if file is empty. + + Raises: + FileNotFoundError: If the file does not exist. + yaml.YAMLError: If the file is not valid YAML. + """ + path = pathlib.Path(path) + with open(path, encoding='utf-8') as f: + if safe: + return yaml.safe_load(f) or {} + else: + return yaml.load(f, Loader=yaml.FullLoader) or {} + + +def save_yaml( + data: dict | list, + path: str | pathlib.Path, + indent: int = 4, + width: int = 1000, + allow_unicode: bool = True, + sort_keys: bool = False, + **kwargs, +) -> None: + """ + Save data to a YAML file with consistent formatting. + + Args: + data: Data to save (dict or list). + path: Path to save the YAML file. + indent: Number of spaces for indentation (default: 4). + width: Maximum line width (default: 1000). + allow_unicode: If True, allow Unicode characters (default: True). + sort_keys: If True, sort dictionary keys (default: False). + **kwargs: Additional arguments to pass to yaml.dump(). + """ + path = pathlib.Path(path) + with open(path, 'w', encoding='utf-8') as f: + yaml.dump( + data, + f, + indent=indent, + width=width, + allow_unicode=allow_unicode, + sort_keys=sort_keys, + default_flow_style=False, + **kwargs, + ) + + +def load_config_file(path: str | pathlib.Path) -> dict: + """ + Load a configuration file, automatically detecting JSON or YAML format. + + This function intelligently tries to load the file based on its extension, + with fallback support if the primary format fails. + + Supported extensions: + - .json: Tries JSON first, falls back to YAML + - .yaml, .yml: Tries YAML first, falls back to JSON + - Others: Tries YAML, then JSON + + Args: + path: Path to the configuration file. + + Returns: + Loaded configuration as a dictionary. + + Raises: + FileNotFoundError: If the file does not exist. + ValueError: If neither JSON nor YAML parsing succeeds. + """ + path = pathlib.Path(path) + + if not path.exists(): + raise FileNotFoundError(f'Configuration file not found: {path}') + + # Try based on file extension + if path.suffix == '.json': + try: + return load_json(path) + except json.JSONDecodeError: + logger.warning(f'Failed to parse {path} as JSON, trying YAML') + try: + return load_yaml(path, safe=True) + except yaml.YAMLError as e: + raise ValueError(f'Failed to parse {path} as JSON or YAML') from e + + elif path.suffix in ['.yaml', '.yml']: + try: + return load_yaml(path, safe=True) + except yaml.YAMLError: + logger.warning(f'Failed to parse {path} as YAML, trying JSON') + try: + return load_json(path) + except json.JSONDecodeError as e: + raise ValueError(f'Failed to parse {path} as YAML or JSON') from e + + else: + # Unknown extension, try YAML first (more common for config) + try: + return load_yaml(path, safe=True) + except yaml.YAMLError: + try: + return load_json(path) + except json.JSONDecodeError as e: + raise ValueError(f'Failed to parse {path} as YAML or JSON') from e + + def _save_to_yaml(data, output_file='formatted_output.yaml'): """ Save dictionary data to YAML with proper multi-line string formatting. diff --git a/flixopt/results.py b/flixopt/results.py index 847ee5a7f..e6268f2b2 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -2,7 +2,6 @@ import copy import datetime -import json import logging import pathlib import warnings @@ -12,7 +11,6 @@ import numpy as np import pandas as pd import xarray as xr -import yaml from . import io as fx_io from . import plotting @@ -46,41 +44,7 @@ def load_mapping_from_file(path: pathlib.Path) -> dict[str, str | list[str]]: Raises: ValueError: If file cannot be loaded as JSON or YAML """ - suffix = path.suffix.lower() - - if suffix == '.json': - # Try JSON first, fallback to YAML - try: - with open(path) as f: - return json.load(f) - except json.JSONDecodeError: - try: - with open(path) as f: - return yaml.safe_load(f) - except Exception: - raise ValueError(f'Could not load config from {path}') from None - elif suffix in {'.yaml', '.yml'}: - # Try YAML first, fallback to JSON - try: - with open(path) as f: - return yaml.safe_load(f) - except yaml.YAMLError: - try: - with open(path) as f: - return json.load(f) - except Exception: - raise ValueError(f'Could not load config from {path}') from None - else: - # Unknown extension, try both starting with JSON - try: - with open(path) as f: - return json.load(f) - except json.JSONDecodeError: - try: - with open(path) as f: - return yaml.safe_load(f) - except Exception: - raise ValueError(f'Could not load config from {path}') from None + return fx_io.load_config_file(path) class _FlowSystemRestorationError(Exception): @@ -205,8 +169,7 @@ def from_file(cls, folder: str | pathlib.Path, name: str) -> CalculationResults: except Exception as e: logger.critical(f'Could not load the linopy model "{name}" from file ("{paths.linopy_model}"): {e}') - with open(paths.summary, encoding='utf-8') as f: - summary = yaml.load(f, Loader=yaml.FullLoader) + summary = fx_io.load_yaml(paths.summary, safe=False) return cls( solution=fx_io.load_dataset_from_netcdf(paths.solution), @@ -1093,8 +1056,7 @@ def to_file( fx_io.save_dataset_to_netcdf(self.solution, paths.solution, compression=compression) fx_io.save_dataset_to_netcdf(self.flow_system_data, paths.flow_system, compression=compression) - with open(paths.summary, 'w', encoding='utf-8') as f: - yaml.dump(self.summary, f, allow_unicode=True, sort_keys=False, indent=4, width=1000) + fx_io.save_yaml(self.summary, paths.summary) if save_linopy_model: if self.model is None: @@ -2085,8 +2047,7 @@ def from_file(cls, folder: str | pathlib.Path, name: str) -> SegmentedCalculatio folder = pathlib.Path(folder) path = folder / name logger.info(f'loading calculation "{name}" from file ("{path.with_suffix(".nc4")}")') - with open(path.with_suffix('.json'), encoding='utf-8') as f: - meta_data = json.load(f) + meta_data = fx_io.load_json(path.with_suffix('.json')) return cls( [CalculationResults.from_file(folder, sub_name) for sub_name in meta_data['sub_calculations']], all_timesteps=pd.DatetimeIndex( @@ -2330,8 +2291,7 @@ def to_file(self, folder: str | pathlib.Path | None = None, name: str | None = N for segment in self.segment_results: segment.to_file(folder=folder, name=segment.name, compression=compression) - with open(path.with_suffix('.json'), 'w', encoding='utf-8') as f: - json.dump(self.meta_data, f, indent=4, ensure_ascii=False) + fx_io.save_json(self.meta_data, path.with_suffix('.json')) logger.info(f'Saved calculation "{name}" to {path}') diff --git a/flixopt/structure.py b/flixopt/structure.py index 07c558eee..6ea618454 100644 --- a/flixopt/structure.py +++ b/flixopt/structure.py @@ -6,7 +6,6 @@ from __future__ import annotations import inspect -import json import logging from dataclasses import dataclass from io import StringIO @@ -788,8 +787,7 @@ def to_json(self, path: str | pathlib.Path): try: # Use the stats mode for JSON export (cleaner output) data = self.get_structure(clean=True, stats=True) - with open(path, 'w', encoding='utf-8') as f: - json.dump(data, f, indent=4, ensure_ascii=False) + fx_io.save_json(data, path) except Exception as e: raise OSError(f'Failed to save {self.__class__.__name__} to JSON file {path}: {e}') from e From 56d4139a9c64750c449ea1691bcb92bf52aa5a39 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sat, 25 Oct 2025 15:41:03 +0200 Subject: [PATCH 34/48] Improve docstring an use safe=True --- flixopt/io.py | 4 +++- flixopt/results.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/flixopt/io.py b/flixopt/io.py index aef1fb968..ba6155b5c 100644 --- a/flixopt/io.py +++ b/flixopt/io.py @@ -87,7 +87,8 @@ def load_yaml(path: str | pathlib.Path, safe: bool = True) -> dict | list: Args: path: Path to the YAML file. safe: If True, use safe_load for security (default: True). - If False, use FullLoader (allows arbitrary Python objects). + If False, use FullLoader (allows arbitrary Python objects - SECURITY RISK). + Only use safe=False for trusted, internally-generated files. Returns: Loaded data (typically dict or list), or empty dict if file is empty. @@ -95,6 +96,7 @@ def load_yaml(path: str | pathlib.Path, safe: bool = True) -> dict | list: Raises: FileNotFoundError: If the file does not exist. yaml.YAMLError: If the file is not valid YAML. + Note: Returns {} for empty YAML files instead of None. """ path = pathlib.Path(path) with open(path, encoding='utf-8') as f: diff --git a/flixopt/results.py b/flixopt/results.py index e6268f2b2..03f6ba174 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -169,7 +169,7 @@ def from_file(cls, folder: str | pathlib.Path, name: str) -> CalculationResults: except Exception as e: logger.critical(f'Could not load the linopy model "{name}" from file ("{paths.linopy_model}"): {e}') - summary = fx_io.load_yaml(paths.summary, safe=False) + summary = fx_io.load_yaml(paths.summary, safe=True) return cls( solution=fx_io.load_dataset_from_netcdf(paths.solution), From 745dac5dcc14f4e1a4fe19505eab888a31c1d7ad Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sat, 25 Oct 2025 17:02:45 +0200 Subject: [PATCH 35/48] Move round_nested_floats to io.py and remove utils.py module --- flixopt/calculation.py | 4 +- flixopt/io.py | 44 ++++++++++++++++++++- flixopt/utils.py | 86 ------------------------------------------ 3 files changed, 45 insertions(+), 89 deletions(-) delete mode 100644 flixopt/utils.py diff --git a/flixopt/calculation.py b/flixopt/calculation.py index 9d2164e1e..2c50b97a3 100644 --- a/flixopt/calculation.py +++ b/flixopt/calculation.py @@ -144,7 +144,7 @@ def main_results(self) -> dict[str, Scalar | dict]: ], } - return utils.round_nested_floats(main_results) + return fx_io.round_nested_floats(main_results) @property def summary(self): @@ -253,7 +253,7 @@ def solve( logger.info( f'{" Main Results ":#^80}\n' + yaml.dump( - utils.round_nested_floats(self.main_results), + fx_io.round_nested_floats(self.main_results), default_flow_style=False, sort_keys=False, allow_unicode=True, diff --git a/flixopt/io.py b/flixopt/io.py index ba6155b5c..ea06936f7 100644 --- a/flixopt/io.py +++ b/flixopt/io.py @@ -6,8 +6,9 @@ import pathlib import re from dataclasses import dataclass -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal +import numpy as np import xarray as xr import yaml @@ -34,6 +35,47 @@ def remove_none_and_empty(obj): return obj +def round_nested_floats(obj: dict | list | float | int | Any, decimals: int = 2) -> dict | list | float | int | Any: + """Recursively round floating point numbers in nested data structures and convert it to python native types. + + This function traverses nested data structures (dictionaries, lists) and rounds + any floating point numbers to the specified number of decimal places. It handles + various data types including NumPy arrays and xarray DataArrays by converting + them to lists with rounded values. + + Args: + obj: The object to process. Can be a dict, list, float, int, numpy.ndarray, + xarray.DataArray, or any other type. + decimals (int, optional): Number of decimal places to round to. Defaults to 2. + + Returns: + The processed object with the same structure as the input, but with all floating point numbers rounded to the specified precision. NumPy arrays and xarray DataArrays are converted to lists. + + Examples: + >>> data = {'a': 3.14159, 'b': [1.234, 2.678]} + >>> round_nested_floats(data, decimals=2) + {'a': 3.14, 'b': [1.23, 2.68]} + + >>> import numpy as np + >>> arr = np.array([1.234, 5.678]) + >>> round_nested_floats(arr, decimals=1) + [1.2, 5.7] + """ + if isinstance(obj, dict): + return {k: round_nested_floats(v, decimals) for k, v in obj.items()} + elif isinstance(obj, list): + return [round_nested_floats(v, decimals) for v in obj] + elif isinstance(obj, float): + return round(obj, decimals) + elif isinstance(obj, int): + return obj + elif isinstance(obj, np.ndarray): + return np.round(obj, decimals).tolist() + elif isinstance(obj, xr.DataArray): + return obj.round(decimals).values.tolist() + return obj + + # ============================================================================ # Centralized JSON and YAML I/O Functions # ============================================================================ diff --git a/flixopt/utils.py b/flixopt/utils.py deleted file mode 100644 index dd1f93d64..000000000 --- a/flixopt/utils.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -This module contains several utility functions used throughout the flixopt framework. -""" - -from __future__ import annotations - -import logging -from typing import Any, Literal - -import numpy as np -import xarray as xr - -logger = logging.getLogger('flixopt') - - -def round_nested_floats(obj: dict | list | float | int | Any, decimals: int = 2) -> dict | list | float | int | Any: - """Recursively round floating point numbers in nested data structures. - - This function traverses nested data structures (dictionaries, lists) and rounds - any floating point numbers to the specified number of decimal places. It handles - various data types including NumPy arrays and xarray DataArrays by converting - them to lists with rounded values. - - Args: - obj: The object to process. Can be a dict, list, float, int, numpy.ndarray, - xarray.DataArray, or any other type. - decimals (int, optional): Number of decimal places to round to. Defaults to 2. - - Returns: - The processed object with the same structure as the input, but with all floating point numbers rounded to the specified precision. NumPy arrays and xarray DataArrays are converted to lists. - - Examples: - >>> data = {'a': 3.14159, 'b': [1.234, 2.678]} - >>> round_nested_floats(data, decimals=2) - {'a': 3.14, 'b': [1.23, 2.68]} - - >>> import numpy as np - >>> arr = np.array([1.234, 5.678]) - >>> round_nested_floats(arr, decimals=1) - [1.2, 5.7] - """ - if isinstance(obj, dict): - return {k: round_nested_floats(v, decimals) for k, v in obj.items()} - elif isinstance(obj, list): - return [round_nested_floats(v, decimals) for v in obj] - elif isinstance(obj, float): - return round(obj, decimals) - elif isinstance(obj, int): - return obj - elif isinstance(obj, np.ndarray): - return np.round(obj, decimals).tolist() - elif isinstance(obj, xr.DataArray): - return obj.round(decimals).values.tolist() - return obj - - -def convert_dataarray( - data: xr.DataArray, mode: Literal['py', 'numpy', 'xarray', 'structure'] -) -> list | np.ndarray | xr.DataArray | str: - """ - Convert a DataArray to a different format. - - Args: - data: The DataArray to convert. - mode: The mode to convert to. - - 'py': Convert to python native types (for json) - - 'numpy': Convert to numpy array - - 'xarray': Convert to xarray.DataArray - - 'structure': Convert to strings (for structure, storing variable names) - - Returns: - The converted data. - - Raises: - ValueError: If the mode is unknown. - """ - if mode == 'numpy': - return data.values - elif mode == 'py': - return data.values.tolist() - elif mode == 'xarray': - return data - elif mode == 'structure': - return f':::{data.name}' - else: - raise ValueError(f'Unknown mode {mode}') From f15076329272df11fd74f644abd481f394fd58b9 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sat, 25 Oct 2025 17:04:24 +0200 Subject: [PATCH 36/48] Rename special yaml safe method --- flixopt/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flixopt/io.py b/flixopt/io.py index ea06936f7..bd7e0c8da 100644 --- a/flixopt/io.py +++ b/flixopt/io.py @@ -242,7 +242,7 @@ def load_config_file(path: str | pathlib.Path) -> dict: raise ValueError(f'Failed to parse {path} as YAML or JSON') from e -def _save_to_yaml(data, output_file='formatted_output.yaml'): +def _save_yaml_multiline(data, output_file='formatted_output.yaml'): """ Save dictionary data to YAML with proper multi-line string formatting. Handles complex string patterns including backticks, special characters, @@ -398,7 +398,7 @@ def document_linopy_model(model: linopy.Model, path: pathlib.Path | None = None) if path is not None: if path.suffix not in ['.yaml', '.yml']: raise ValueError(f'Invalid file extension for path {path}. Only .yaml and .yml are supported') - _save_to_yaml(documentation, str(path)) + _save_yaml_multiline(documentation, str(path)) return documentation From 382ff84a40d74d62524d64ba58adeccfe1ada504 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sun, 26 Oct 2025 10:33:14 +0100 Subject: [PATCH 37/48] Remove import utils --- flixopt/calculation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flixopt/calculation.py b/flixopt/calculation.py index 2c50b97a3..f89cde86d 100644 --- a/flixopt/calculation.py +++ b/flixopt/calculation.py @@ -22,7 +22,6 @@ import yaml from . import io as fx_io -from . import utils as utils from .aggregation import Aggregation, AggregationModel, AggregationParameters from .components import Storage from .config import CONFIG From ca6cf070ff7cc6e712a4581b0dc40848b7813dc0 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sun, 26 Oct 2025 10:35:37 +0100 Subject: [PATCH 38/48] Ensure native types --- flixopt/io.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flixopt/io.py b/flixopt/io.py index bd7e0c8da..57c4e538b 100644 --- a/flixopt/io.py +++ b/flixopt/io.py @@ -65,6 +65,12 @@ def round_nested_floats(obj: dict | list | float | int | Any, decimals: int = 2) return {k: round_nested_floats(v, decimals) for k, v in obj.items()} elif isinstance(obj, list): return [round_nested_floats(v, decimals) for v in obj] + elif isinstance(obj, np.floating): + return round(float(obj), decimals) + elif isinstance(obj, np.integer): + return int(obj) + elif isinstance(obj, np.bool_): + return bool(obj) elif isinstance(obj, float): return round(obj, decimals) elif isinstance(obj, int): From 34e2d398bac4379ddf89b0cbfd06ef1e97e5ada6 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sun, 26 Oct 2025 10:37:13 +0100 Subject: [PATCH 39/48] Use safe dump everywhere and normalize file suffixes --- flixopt/config.py | 2 +- flixopt/io.py | 51 ++++++++++++++++++++++++++++++---------------- flixopt/results.py | 2 +- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/flixopt/config.py b/flixopt/config.py index e51c0ac8c..670f86da2 100644 --- a/flixopt/config.py +++ b/flixopt/config.py @@ -305,7 +305,7 @@ def load_from_file(cls, config_file: str | Path): if not config_path.exists(): raise FileNotFoundError(f'Config file not found: {config_file}') - config_dict = fx_io.load_yaml(config_path, safe=True) + config_dict = fx_io.load_yaml(config_path) cls._apply_config_dict(config_dict) cls.apply() diff --git a/flixopt/io.py b/flixopt/io.py index 57c4e538b..1ebc8fc67 100644 --- a/flixopt/io.py +++ b/flixopt/io.py @@ -128,15 +128,12 @@ def save_json( json.dump(data, f, indent=indent, ensure_ascii=ensure_ascii, **kwargs) -def load_yaml(path: str | pathlib.Path, safe: bool = True) -> dict | list: +def load_yaml(path: str | pathlib.Path) -> dict | list: """ Load data from a YAML file. Args: path: Path to the YAML file. - safe: If True, use safe_load for security (default: True). - If False, use FullLoader (allows arbitrary Python objects - SECURITY RISK). - Only use safe=False for trusted, internally-generated files. Returns: Loaded data (typically dict or list), or empty dict if file is empty. @@ -148,10 +145,25 @@ def load_yaml(path: str | pathlib.Path, safe: bool = True) -> dict | list: """ path = pathlib.Path(path) with open(path, encoding='utf-8') as f: - if safe: - return yaml.safe_load(f) or {} - else: - return yaml.load(f, Loader=yaml.FullLoader) or {} + return yaml.safe_load(f) or {} + + +def _load_yaml_unsafe(path: str | pathlib.Path) -> dict | list: + """ + INTERNAL: Load YAML allowing arbitrary tags. Do not use on untrusted input. + + This function exists only for loading internally-generated files that may + contain custom YAML tags. Never use this on user-provided files. + + Args: + path: Path to the YAML file. + + Returns: + Loaded data (typically dict or list), or empty dict if file is empty. + """ + path = pathlib.Path(path) + with open(path, encoding='utf-8') as f: + return yaml.unsafe_load(f) or {} def save_yaml( @@ -173,11 +185,11 @@ def save_yaml( width: Maximum line width (default: 1000). allow_unicode: If True, allow Unicode characters (default: True). sort_keys: If True, sort dictionary keys (default: False). - **kwargs: Additional arguments to pass to yaml.dump(). + **kwargs: Additional arguments to pass to yaml.safe_dump(). """ path = pathlib.Path(path) with open(path, 'w', encoding='utf-8') as f: - yaml.dump( + yaml.safe_dump( data, f, indent=indent, @@ -217,19 +229,22 @@ def load_config_file(path: str | pathlib.Path) -> dict: raise FileNotFoundError(f'Configuration file not found: {path}') # Try based on file extension - if path.suffix == '.json': + # Normalize extension to lowercase for case-insensitive matching + suffix = path.suffix.lower() + + if suffix == '.json': try: return load_json(path) except json.JSONDecodeError: logger.warning(f'Failed to parse {path} as JSON, trying YAML') try: - return load_yaml(path, safe=True) + return load_yaml(path) except yaml.YAMLError as e: raise ValueError(f'Failed to parse {path} as JSON or YAML') from e - elif path.suffix in ['.yaml', '.yml']: + elif suffix in ['.yaml', '.yml']: try: - return load_yaml(path, safe=True) + return load_yaml(path) except yaml.YAMLError: logger.warning(f'Failed to parse {path} as YAML, trying JSON') try: @@ -240,7 +255,7 @@ def load_config_file(path: str | pathlib.Path) -> dict: else: # Unknown extension, try YAML first (more common for config) try: - return load_yaml(path, safe=True) + return load_yaml(path) except yaml.YAMLError: try: return load_json(path) @@ -276,14 +291,14 @@ def represent_str(dumper, data): # Use plain style for simple strings return dumper.represent_scalar('tag:yaml.org,2002:str', data) - # Add the string representer to SafeDumper - yaml.add_representer(str, represent_str, Dumper=yaml.SafeDumper) - # Configure dumper options for better formatting class CustomDumper(yaml.SafeDumper): def increase_indent(self, flow=False, indentless=False): return super().increase_indent(flow, False) + # Bind representer locally to CustomDumper to avoid global side effects + CustomDumper.add_representer(str, represent_str) + # Write to file with settings that ensure proper formatting with open(output_file, 'w', encoding='utf-8') as file: yaml.dump( diff --git a/flixopt/results.py b/flixopt/results.py index 03f6ba174..575c7811b 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -169,7 +169,7 @@ def from_file(cls, folder: str | pathlib.Path, name: str) -> CalculationResults: except Exception as e: logger.critical(f'Could not load the linopy model "{name}" from file ("{paths.linopy_model}"): {e}') - summary = fx_io.load_yaml(paths.summary, safe=True) + summary = fx_io.load_yaml(paths.summary) return cls( solution=fx_io.load_dataset_from_netcdf(paths.solution), From 45e6a054c796aefa1c86290751ce620c7eaca338 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sun, 26 Oct 2025 10:39:10 +0100 Subject: [PATCH 40/48] Avoid double rounding --- flixopt/calculation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flixopt/calculation.py b/flixopt/calculation.py index f89cde86d..f744c5247 100644 --- a/flixopt/calculation.py +++ b/flixopt/calculation.py @@ -252,7 +252,7 @@ def solve( logger.info( f'{" Main Results ":#^80}\n' + yaml.dump( - fx_io.round_nested_floats(self.main_results), + self.main_results, default_flow_style=False, sort_keys=False, allow_unicode=True, From 9dbc5f069078af4e11698fd76eac3d4a65c356a6 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sun, 26 Oct 2025 10:39:23 +0100 Subject: [PATCH 41/48] Set indent to 4 consistently --- flixopt/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flixopt/io.py b/flixopt/io.py index 1ebc8fc67..45299e9f9 100644 --- a/flixopt/io.py +++ b/flixopt/io.py @@ -309,7 +309,7 @@ def increase_indent(self, flow=False, indentless=False): default_flow_style=False, # Use block style for mappings width=1000, # Set a reasonable line width allow_unicode=True, # Support Unicode characters - indent=2, # Set consistent indentation + indent=4, # Set consistent indentation ) From 14e871f44a53c2c3bb0adc0a5f7d4b6a980b0d3a Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sun, 26 Oct 2025 10:39:50 +0100 Subject: [PATCH 42/48] Simplify netcdf file io --- flixopt/io.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/flixopt/io.py b/flixopt/io.py index 45299e9f9..9e8c9ae31 100644 --- a/flixopt/io.py +++ b/flixopt/io.py @@ -1,12 +1,11 @@ from __future__ import annotations -import importlib.util import json import logging import pathlib import re from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any import numpy as np import xarray as xr @@ -428,7 +427,6 @@ def save_dataset_to_netcdf( ds: xr.Dataset, path: str | pathlib.Path, compression: int = 0, - engine: Literal['netcdf4', 'scipy', 'h5netcdf'] = 'h5netcdf', ) -> None: """ Save a dataset to a netcdf file. Store all attrs as JSON strings in 'attrs' attributes. @@ -445,16 +443,6 @@ def save_dataset_to_netcdf( if path.suffix not in ['.nc', '.nc4']: raise ValueError(f'Invalid file extension for path {path}. Only .nc and .nc4 are supported') - apply_encoding = False - if compression != 0: - if importlib.util.find_spec(engine) is not None: - apply_encoding = True - else: - logger.warning( - f'Dataset was exported without compression due to missing dependency "{engine}".' - f'Install {engine} via `pip install {engine}`.' - ) - ds = ds.copy(deep=True) ds.attrs = {'attrs': json.dumps(ds.attrs)} @@ -471,9 +459,9 @@ def save_dataset_to_netcdf( ds.to_netcdf( path, encoding=None - if not apply_encoding + if compression == 0 else {data_var: {'zlib': True, 'complevel': compression} for data_var in ds.data_vars}, - engine=engine, + engine='h5netcdf', ) From b21b41a96c817165de3069961769205a72871c96 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sun, 26 Oct 2025 11:18:27 +0100 Subject: [PATCH 43/48] Improve benchmark_file_io.py --- tests/benchmark_file_io.py | 159 +++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 tests/benchmark_file_io.py diff --git a/tests/benchmark_file_io.py b/tests/benchmark_file_io.py new file mode 100644 index 000000000..a7fe06afe --- /dev/null +++ b/tests/benchmark_file_io.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +""" +Benchmark comparing h5netcdf vs netcdf4 for file I/O with compression. +Tests with large xarray datasets (300 variables, 80,000 timesteps). +""" + +import os +import tempfile +import time +from pathlib import Path + +import numpy as np +import xarray as xr +import yaml + + +def create_dataset(time_steps, n_vars): + """Create a dataset with specified dimensions. Each variable is 1D (time only).""" + # Create coordinate array + time_coords = np.arange(time_steps) + + # Create dataset with n_vars variables, each 1D + np.random.seed(42) + data_vars = {} + + for i in range(n_vars): + # Generate random 1D data (time series) + data = np.random.randn(time_steps).astype(np.float32) + var_name = f'var_{i:03d}' + data_vars[var_name] = (['time'], data) + + ds = xr.Dataset( + data_vars, + coords={ + 'time': time_coords, + }, + ) + + ds.attrs['description'] = f'Test dataset: {n_vars} vars, {time_steps} timesteps' + + return ds + + +def benchmark_write(ds, filepath, engine, compression_level=4): + """Benchmark write performance.""" + encoding = {var: {'zlib': True, 'complevel': compression_level} for var in ds.data_vars} + + start = time.perf_counter() + ds.to_netcdf(filepath, engine=engine, encoding=encoding) + elapsed = time.perf_counter() - start + + file_size = os.path.getsize(filepath) + + return elapsed, file_size + + +def benchmark_read(filepath, engine): + """Benchmark read performance.""" + start = time.perf_counter() + ds = xr.open_dataset(filepath, engine=engine) + ds.load() + ds.close() + elapsed = time.perf_counter() - start + + return elapsed + + +def run_benchmark(): + """Run the complete benchmark for multiple dataset sizes.""" + + # Define different dataset configurations to test + # (time_steps, n_vars, name) + dataset_configs = [ + (100, 20, 'tiny'), + (1000, 50, 'small'), + (5000, 100, 'medium'), + (10000, 200, 'large'), + # (20000, 300, 'xlarge'), + # (80000, 300, 'xxlarge'), + ] + + # Test compression levels + compression_levels = [0, 4, 9] + engines = ['h5netcdf', 'netcdf4'] + + results = {'configurations': [], 'benchmarks': {}} + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = Path(tmpdir) + + for time_steps, n_vars, config_name in dataset_configs: + ds = create_dataset(time_steps, n_vars) + + # Calculate uncompressed size + uncompressed_size = sum(ds[var].nbytes for var in ds.data_vars) + + config_info = { + 'name': config_name, + 'time_steps': time_steps, + 'num_variables': n_vars, + 'dimensions': dict(ds.dims), + 'uncompressed_size_mb': round(uncompressed_size / (1024 * 1024), 2), + } + results['configurations'].append(config_info) + + results['benchmarks'][config_name] = {} + + for comp_level in compression_levels: + comp_key = f'compression_{comp_level}' + results['benchmarks'][config_name][comp_key] = {} + + for engine in engines: + filepath = tmpdir / f'test_{config_name}_{engine}_comp{comp_level}.nc' + + try: + # Write benchmark + write_time, file_size = benchmark_write(ds, filepath, engine, comp_level) + + # Read benchmark + read_time = benchmark_read(filepath, engine) + + results['benchmarks'][config_name][comp_key][engine] = { + 'write_time_seconds': round(write_time, 3), + 'read_time_seconds': round(read_time, 3), + 'file_size_mb': round(file_size / (1024 * 1024), 2), + 'compression_ratio': round(uncompressed_size / file_size, 2), + 'write_throughput_mbps': round((uncompressed_size / (1024 * 1024)) / write_time, 2) + if write_time > 0 + else 0, + 'read_throughput_mbps': round((uncompressed_size / (1024 * 1024)) / read_time, 2) + if read_time > 0 + else 0, + } + + except Exception as e: + results['benchmarks'][config_name][comp_key][engine] = {'error': str(e)} + + # Clean up + if filepath.exists(): + filepath.unlink() + + # Clean up dataset + del ds + + return results + + +def main(): + results = run_benchmark() + + # Write results to YAML + output_file = 'netcdf_benchmark_results.yaml' + + with open(output_file, 'w') as f: + yaml.dump(results, f, default_flow_style=False, sort_keys=False) + + +if __name__ == '__main__': + main() From 62b1f66e5be01146df9256a8fd121adc739186f0 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sun, 26 Oct 2025 11:21:16 +0100 Subject: [PATCH 44/48] Improve benchmark_file_io.py --- tests/benchmark_file_io.py | 150 ++++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 2 deletions(-) diff --git a/tests/benchmark_file_io.py b/tests/benchmark_file_io.py index a7fe06afe..2a3733723 100644 --- a/tests/benchmark_file_io.py +++ b/tests/benchmark_file_io.py @@ -10,8 +10,10 @@ from pathlib import Path import numpy as np +import plotly.graph_objects as go import xarray as xr import yaml +from plotly.subplots import make_subplots def create_dataset(time_steps, n_vars): @@ -65,6 +67,146 @@ def benchmark_read(filepath, engine): return elapsed +def create_plots(results): + """Create interactive Plotly visualizations of benchmark results.""" + + # Extract data for plotting + config_names = [c['name'] for c in results['configurations']] + + # Create subplots + fig = make_subplots( + rows=2, + cols=2, + subplot_titles=( + 'Write Time Comparison', + 'Read Time Comparison', + 'File Size with Compression', + 'Throughput Comparison', + ), + specs=[[{'secondary_y': False}, {'secondary_y': False}], [{'secondary_y': False}, {'secondary_y': False}]], + ) + + colors = {'h5netcdf': '#1f77b4', 'netcdf4': '#ff7f0e'} + + compression_levels = [0, 4, 9] + engines = ['h5netcdf', 'netcdf4'] + + # Plot 1: Write times (compression level 4) + for engine in engines: + write_times = [] + for config_name in config_names: + try: + time_val = results['benchmarks'][config_name]['compression_4'][engine]['write_time_seconds'] + write_times.append(time_val) + except KeyError: + write_times.append(None) + + fig.add_trace( + go.Scatter( + x=config_names, + y=write_times, + name=engine, + mode='lines+markers', + line=dict(color=colors[engine]), + legendgroup=engine, + showlegend=True, + ), + row=1, + col=1, + ) + + # Plot 2: Read times (compression level 4) + for engine in engines: + read_times = [] + for config_name in config_names: + try: + time_val = results['benchmarks'][config_name]['compression_4'][engine]['read_time_seconds'] + read_times.append(time_val) + except KeyError: + read_times.append(None) + + fig.add_trace( + go.Scatter( + x=config_names, + y=read_times, + name=engine, + mode='lines+markers', + line=dict(color=colors[engine]), + legendgroup=engine, + showlegend=False, + ), + row=1, + col=2, + ) + + # Plot 3: File sizes for different compression levels + for comp_level in compression_levels: + for engine in engines: + file_sizes = [] + for config_name in config_names: + try: + size_val = results['benchmarks'][config_name][f'compression_{comp_level}'][engine]['file_size_mb'] + file_sizes.append(size_val) + except KeyError: + file_sizes.append(None) + + fig.add_trace( + go.Scatter( + x=config_names, + y=file_sizes, + name=f'{engine} (comp={comp_level})', + mode='lines+markers', + line=dict(color=colors[engine], dash=['solid', 'dash', 'dot'][comp_level // 4]), + legendgroup=f'{engine}_comp', + showlegend=True, + ), + row=2, + col=1, + ) + + # Plot 4: Write throughput (compression level 4) + for engine in engines: + throughputs = [] + for config_name in config_names: + try: + tp_val = results['benchmarks'][config_name]['compression_4'][engine]['write_throughput_mbps'] + throughputs.append(tp_val) + except KeyError: + throughputs.append(None) + + fig.add_trace( + go.Scatter( + x=config_names, + y=throughputs, + name=engine, + mode='lines+markers', + line=dict(color=colors[engine]), + legendgroup=engine, + showlegend=False, + ), + row=2, + col=2, + ) + + # Update axes labels + fig.update_xaxes(title_text='Dataset Size', row=1, col=1) + fig.update_xaxes(title_text='Dataset Size', row=1, col=2) + fig.update_xaxes(title_text='Dataset Size', row=2, col=1) + fig.update_xaxes(title_text='Dataset Size', row=2, col=2) + + fig.update_yaxes(title_text='Time (seconds)', row=1, col=1) + fig.update_yaxes(title_text='Time (seconds)', row=1, col=2) + fig.update_yaxes(title_text='File Size (MB)', row=2, col=1) + fig.update_yaxes(title_text='Throughput (MB/s)', row=2, col=2) + + # Update layout + fig.update_layout( + height=800, title_text='NetCDF Engine Benchmark: h5netcdf vs netcdf4', showlegend=True, hovermode='x unified' + ) + + return fig + + def run_benchmark(): """Run the complete benchmark for multiple dataset sizes.""" @@ -75,8 +217,8 @@ def run_benchmark(): (1000, 50, 'small'), (5000, 100, 'medium'), (10000, 200, 'large'), - # (20000, 300, 'xlarge'), - # (80000, 300, 'xxlarge'), + (20000, 300, 'xlarge'), + (80000, 300, 'xxlarge'), ] # Test compression levels @@ -154,6 +296,10 @@ def main(): with open(output_file, 'w') as f: yaml.dump(results, f, default_flow_style=False, sort_keys=False) + # Create and save plots + fig = create_plots(results) + fig.write_html('netcdf_benchmark_plots.html') + if __name__ == '__main__': main() From e84457b874e430f3fc4780cc9fbbede5aa4857c3 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sun, 26 Oct 2025 11:25:29 +0100 Subject: [PATCH 45/48] Revert to using netcdf4 for file io --- CHANGELOG.md | 1 + flixopt/io.py | 4 ++-- flixopt/results.py | 2 +- pyproject.toml | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eb16a4c6..9ce2af42a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,7 @@ If upgrading from v2.x, see the [v3.0.0 release notes](https://github.com/flixOp - **Color terminology**: Standardized terminology from "colormap" to "colorscale" throughout the codebase for consistency with Plotly conventions - **Default colorscales**: Changed default sequential colorscale from 'viridis' to 'turbo' for better perceptual uniformity; qualitative colorscale now defaults to 'plotly' - **Aggregation plotting**: `Aggregation.plot()` now respects `CONFIG.Plotting.default_qualitative_colorscale` and uses `process_colors()` for consistent color handling +- **netcdf engine**: FOllowing the xarray revert in `xarray==2025.09.02` and after running some benchmarks, we go back to using the netcdf4 engine ### 🗑️ Deprecated diff --git a/flixopt/io.py b/flixopt/io.py index 9e8c9ae31..059670ddd 100644 --- a/flixopt/io.py +++ b/flixopt/io.py @@ -461,7 +461,7 @@ def save_dataset_to_netcdf( encoding=None if compression == 0 else {data_var: {'zlib': True, 'complevel': compression} for data_var in ds.data_vars}, - engine='h5netcdf', + engine='netcdf4', ) @@ -475,7 +475,7 @@ def load_dataset_from_netcdf(path: str | pathlib.Path) -> xr.Dataset: Returns: Dataset: Loaded dataset with restored attrs. """ - ds = xr.load_dataset(str(path), engine='h5netcdf') + ds = xr.load_dataset(str(path), engine='netcdf4') # Restore Dataset attrs if 'attrs' in ds.attrs: diff --git a/flixopt/results.py b/flixopt/results.py index 575c7811b..950570df3 100644 --- a/flixopt/results.py +++ b/flixopt/results.py @@ -1062,7 +1062,7 @@ def to_file( if self.model is None: logger.critical('No model in the CalculationResults. Saving the model is not possible.') else: - self.model.to_netcdf(paths.linopy_model, engine='h5netcdf') + self.model.to_netcdf(paths.linopy_model, engine='netcdf4') if document_model: if self.model is None: diff --git a/pyproject.toml b/pyproject.toml index 227eca49e..8c44d025f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "xarray >= 2024.2.0, < 2026.0", # CalVer: allow through next calendar year # Optimization and data handling "linopy >= 0.5.1, < 0.6", # Widened from patch pin to minor range - "h5netcdf>=1.0.0, < 2", + "netcdf4 >= 1.6.1, < 2", # Utilities "pyyaml >= 6.0.0, < 7", "rich >= 13.0.0, < 15", From 8625d4b7ea806c1423153ff29e6e3bca09584fb8 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sun, 26 Oct 2025 11:25:50 +0100 Subject: [PATCH 46/48] Remove temporary benchmark file --- tests/benchmark_file_io.py | 305 ------------------------------------- 1 file changed, 305 deletions(-) delete mode 100644 tests/benchmark_file_io.py diff --git a/tests/benchmark_file_io.py b/tests/benchmark_file_io.py deleted file mode 100644 index 2a3733723..000000000 --- a/tests/benchmark_file_io.py +++ /dev/null @@ -1,305 +0,0 @@ -#!/usr/bin/env python3 -""" -Benchmark comparing h5netcdf vs netcdf4 for file I/O with compression. -Tests with large xarray datasets (300 variables, 80,000 timesteps). -""" - -import os -import tempfile -import time -from pathlib import Path - -import numpy as np -import plotly.graph_objects as go -import xarray as xr -import yaml -from plotly.subplots import make_subplots - - -def create_dataset(time_steps, n_vars): - """Create a dataset with specified dimensions. Each variable is 1D (time only).""" - # Create coordinate array - time_coords = np.arange(time_steps) - - # Create dataset with n_vars variables, each 1D - np.random.seed(42) - data_vars = {} - - for i in range(n_vars): - # Generate random 1D data (time series) - data = np.random.randn(time_steps).astype(np.float32) - var_name = f'var_{i:03d}' - data_vars[var_name] = (['time'], data) - - ds = xr.Dataset( - data_vars, - coords={ - 'time': time_coords, - }, - ) - - ds.attrs['description'] = f'Test dataset: {n_vars} vars, {time_steps} timesteps' - - return ds - - -def benchmark_write(ds, filepath, engine, compression_level=4): - """Benchmark write performance.""" - encoding = {var: {'zlib': True, 'complevel': compression_level} for var in ds.data_vars} - - start = time.perf_counter() - ds.to_netcdf(filepath, engine=engine, encoding=encoding) - elapsed = time.perf_counter() - start - - file_size = os.path.getsize(filepath) - - return elapsed, file_size - - -def benchmark_read(filepath, engine): - """Benchmark read performance.""" - start = time.perf_counter() - ds = xr.open_dataset(filepath, engine=engine) - ds.load() - ds.close() - elapsed = time.perf_counter() - start - - return elapsed - - -def create_plots(results): - """Create interactive Plotly visualizations of benchmark results.""" - - # Extract data for plotting - config_names = [c['name'] for c in results['configurations']] - - # Create subplots - fig = make_subplots( - rows=2, - cols=2, - subplot_titles=( - 'Write Time Comparison', - 'Read Time Comparison', - 'File Size with Compression', - 'Throughput Comparison', - ), - specs=[[{'secondary_y': False}, {'secondary_y': False}], [{'secondary_y': False}, {'secondary_y': False}]], - ) - - colors = {'h5netcdf': '#1f77b4', 'netcdf4': '#ff7f0e'} - - compression_levels = [0, 4, 9] - engines = ['h5netcdf', 'netcdf4'] - - # Plot 1: Write times (compression level 4) - for engine in engines: - write_times = [] - for config_name in config_names: - try: - time_val = results['benchmarks'][config_name]['compression_4'][engine]['write_time_seconds'] - write_times.append(time_val) - except KeyError: - write_times.append(None) - - fig.add_trace( - go.Scatter( - x=config_names, - y=write_times, - name=engine, - mode='lines+markers', - line=dict(color=colors[engine]), - legendgroup=engine, - showlegend=True, - ), - row=1, - col=1, - ) - - # Plot 2: Read times (compression level 4) - for engine in engines: - read_times = [] - for config_name in config_names: - try: - time_val = results['benchmarks'][config_name]['compression_4'][engine]['read_time_seconds'] - read_times.append(time_val) - except KeyError: - read_times.append(None) - - fig.add_trace( - go.Scatter( - x=config_names, - y=read_times, - name=engine, - mode='lines+markers', - line=dict(color=colors[engine]), - legendgroup=engine, - showlegend=False, - ), - row=1, - col=2, - ) - - # Plot 3: File sizes for different compression levels - for comp_level in compression_levels: - for engine in engines: - file_sizes = [] - for config_name in config_names: - try: - size_val = results['benchmarks'][config_name][f'compression_{comp_level}'][engine]['file_size_mb'] - file_sizes.append(size_val) - except KeyError: - file_sizes.append(None) - - fig.add_trace( - go.Scatter( - x=config_names, - y=file_sizes, - name=f'{engine} (comp={comp_level})', - mode='lines+markers', - line=dict(color=colors[engine], dash=['solid', 'dash', 'dot'][comp_level // 4]), - legendgroup=f'{engine}_comp', - showlegend=True, - ), - row=2, - col=1, - ) - - # Plot 4: Write throughput (compression level 4) - for engine in engines: - throughputs = [] - for config_name in config_names: - try: - tp_val = results['benchmarks'][config_name]['compression_4'][engine]['write_throughput_mbps'] - throughputs.append(tp_val) - except KeyError: - throughputs.append(None) - - fig.add_trace( - go.Scatter( - x=config_names, - y=throughputs, - name=engine, - mode='lines+markers', - line=dict(color=colors[engine]), - legendgroup=engine, - showlegend=False, - ), - row=2, - col=2, - ) - - # Update axes labels - fig.update_xaxes(title_text='Dataset Size', row=1, col=1) - fig.update_xaxes(title_text='Dataset Size', row=1, col=2) - fig.update_xaxes(title_text='Dataset Size', row=2, col=1) - fig.update_xaxes(title_text='Dataset Size', row=2, col=2) - - fig.update_yaxes(title_text='Time (seconds)', row=1, col=1) - fig.update_yaxes(title_text='Time (seconds)', row=1, col=2) - fig.update_yaxes(title_text='File Size (MB)', row=2, col=1) - fig.update_yaxes(title_text='Throughput (MB/s)', row=2, col=2) - - # Update layout - fig.update_layout( - height=800, title_text='NetCDF Engine Benchmark: h5netcdf vs netcdf4', showlegend=True, hovermode='x unified' - ) - - return fig - - -def run_benchmark(): - """Run the complete benchmark for multiple dataset sizes.""" - - # Define different dataset configurations to test - # (time_steps, n_vars, name) - dataset_configs = [ - (100, 20, 'tiny'), - (1000, 50, 'small'), - (5000, 100, 'medium'), - (10000, 200, 'large'), - (20000, 300, 'xlarge'), - (80000, 300, 'xxlarge'), - ] - - # Test compression levels - compression_levels = [0, 4, 9] - engines = ['h5netcdf', 'netcdf4'] - - results = {'configurations': [], 'benchmarks': {}} - - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = Path(tmpdir) - - for time_steps, n_vars, config_name in dataset_configs: - ds = create_dataset(time_steps, n_vars) - - # Calculate uncompressed size - uncompressed_size = sum(ds[var].nbytes for var in ds.data_vars) - - config_info = { - 'name': config_name, - 'time_steps': time_steps, - 'num_variables': n_vars, - 'dimensions': dict(ds.dims), - 'uncompressed_size_mb': round(uncompressed_size / (1024 * 1024), 2), - } - results['configurations'].append(config_info) - - results['benchmarks'][config_name] = {} - - for comp_level in compression_levels: - comp_key = f'compression_{comp_level}' - results['benchmarks'][config_name][comp_key] = {} - - for engine in engines: - filepath = tmpdir / f'test_{config_name}_{engine}_comp{comp_level}.nc' - - try: - # Write benchmark - write_time, file_size = benchmark_write(ds, filepath, engine, comp_level) - - # Read benchmark - read_time = benchmark_read(filepath, engine) - - results['benchmarks'][config_name][comp_key][engine] = { - 'write_time_seconds': round(write_time, 3), - 'read_time_seconds': round(read_time, 3), - 'file_size_mb': round(file_size / (1024 * 1024), 2), - 'compression_ratio': round(uncompressed_size / file_size, 2), - 'write_throughput_mbps': round((uncompressed_size / (1024 * 1024)) / write_time, 2) - if write_time > 0 - else 0, - 'read_throughput_mbps': round((uncompressed_size / (1024 * 1024)) / read_time, 2) - if read_time > 0 - else 0, - } - - except Exception as e: - results['benchmarks'][config_name][comp_key][engine] = {'error': str(e)} - - # Clean up - if filepath.exists(): - filepath.unlink() - - # Clean up dataset - del ds - - return results - - -def main(): - results = run_benchmark() - - # Write results to YAML - output_file = 'netcdf_benchmark_results.yaml' - - with open(output_file, 'w') as f: - yaml.dump(results, f, default_flow_style=False, sort_keys=False) - - # Create and save plots - fig = create_plots(results) - fig.write_html('netcdf_benchmark_plots.html') - - -if __name__ == '__main__': - main() From c5a6b1b8a0df951aaa827e6883f2082d0daefe31 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sun, 26 Oct 2025 11:26:43 +0100 Subject: [PATCH 47/48] Typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ce2af42a..9b52ccd96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,7 +80,7 @@ If upgrading from v2.x, see the [v3.0.0 release notes](https://github.com/flixOp - **Color terminology**: Standardized terminology from "colormap" to "colorscale" throughout the codebase for consistency with Plotly conventions - **Default colorscales**: Changed default sequential colorscale from 'viridis' to 'turbo' for better perceptual uniformity; qualitative colorscale now defaults to 'plotly' - **Aggregation plotting**: `Aggregation.plot()` now respects `CONFIG.Plotting.default_qualitative_colorscale` and uses `process_colors()` for consistent color handling -- **netcdf engine**: FOllowing the xarray revert in `xarray==2025.09.02` and after running some benchmarks, we go back to using the netcdf4 engine +- **netcdf engine**: Following the xarray revert in `xarray==2025.09.02` and after running some benchmarks, we go back to using the netcdf4 engine ### 🗑️ Deprecated From a8b1f52041df78d9c90c089e7df0500a8063d55c Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sun, 26 Oct 2025 12:10:23 +0100 Subject: [PATCH 48/48] Typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b52ccd96..663f087a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,7 +80,7 @@ If upgrading from v2.x, see the [v3.0.0 release notes](https://github.com/flixOp - **Color terminology**: Standardized terminology from "colormap" to "colorscale" throughout the codebase for consistency with Plotly conventions - **Default colorscales**: Changed default sequential colorscale from 'viridis' to 'turbo' for better perceptual uniformity; qualitative colorscale now defaults to 'plotly' - **Aggregation plotting**: `Aggregation.plot()` now respects `CONFIG.Plotting.default_qualitative_colorscale` and uses `process_colors()` for consistent color handling -- **netcdf engine**: Following the xarray revert in `xarray==2025.09.02` and after running some benchmarks, we go back to using the netcdf4 engine +- **netcdf engine**: Following the xarray revert in `xarray==2025.09.2` and after running some benchmarks, we go back to using the netcdf4 engine ### 🗑️ Deprecated