Adds pay as bid option to the complex clearing (#520)

-add pricing mechanism as param_dict argument
-move log_flows also to the param_dict
-update the docstrings for better explanation
-rename the market clearing to complex clearing
-rename it in config files and add param_dict for demonstration
- run pre-commit (apparently this was not done before the release)

assume/common/forecasts.py

@@ -557,8 +557,6 @@ def __init__(
         for key, value in kwargs.items():
             self.data_dict[key] = FastSeries(index=self.index, value=value, name=key)
 
-
-
     def __getitem__(self, column: str) -> FastSeries:
         """
         Retrieves forecasted values.

assume/common/grid_utils.py

@@ -49,7 +49,12 @@ def add_generators(
         )
     else:
         # add generators
-        generators.drop(["p_min_pu", "p_max_pu", "marginal_cost"], axis=1, inplace=True, errors="ignore")
+        generators.drop(
+            ["p_min_pu", "p_max_pu", "marginal_cost"],
+            axis=1,
+            inplace=True,
+            errors="ignore",
+        )
         network.add(
             "Generator",
             name=generators.index,

assume/common/outputs.py

@@ -687,7 +687,7 @@ def get_sum_reward(self):
         )
         if self.db is None:
             return []
-        
+
         with self.db.begin() as db:
             rewards_by_unit = db.execute(query).fetchall()
 

assume/markets/clearing_algorithms/__init__.py

@@ -14,7 +14,7 @@
     "pay_as_clear": PayAsClearRole,
     "pay_as_bid": PayAsBidRole,
     "pay_as_bid_contract": PayAsBidContractRole,
-    "pay_as_clear_complex": ComplexClearingRole,
+    "complex_clearing": ComplexClearingRole,
     "pay_as_clear_complex_dmas": ComplexDmasClearingRole,
 }
 

assume/markets/clearing_algorithms/complex_clearing.py

@@ -258,33 +258,51 @@ def energy_balance_rule(model, node, t):
 
 class ComplexClearingRole(MarketRole):
     """
-    Defines the clearing algorithm for the complex market.
+    This class defines an optimization-based market clearing algorithm with support for complex bid types,
+    including block bids, linked bids, minimum acceptance ratios, and profiled volumes. It supports network
+    representations with either zonal or nodal configurations, enabling the modeling of complex markets with
+    multiple zones and power flow constraints.
 
-    The complex market is a pay-as-clear market with more complex bid structures, including minimum acceptance ratios, bid types, and profiled volumes.
+    The market clearing algorithm accepts additional arguments via the `param_dict` in the market configuration.
 
-    The class supports two types of network representations:
-
-    1. **Zonal Representation**: The network is divided into zones, and the incidence matrix represents the connections between these zones.
-
-    2. **Nodal Representation**: Each bus in the network is treated as a node, and the incidence matrix represents the connections between these nodes.
-
-    Zonal Representation:
-        If a `zones_identifier` is provided in the market configuration param_dict, the buses are grouped into zones based on this identifier.
-        The incidence matrix is then constructed to represent the power connections between these zones. The total transfer
-        capacity between zones is determined by the sum of the capacities of the lines connecting the zones.
-        - `zones_identifier` (str): The key in the bus data that identifies the zone to which each bus belongs.
-
-    Nodal Representation:
-        If no `zones_identifier` is provided, each bus is treated as a separate node.
-        The incidence matrix is constructed to represent the power connections between these nodes.
+    Args:
+        marketconfig (MarketConfig): The market configuration object containing all parameters for the market
+            clearing process.
 
     Attributes:
         marketconfig (MarketConfig): The market configuration.
-        incidence_matrix (pd.DataFrame): The incidence matrix representing the network connections.
-        nodes (list): List of nodes or zones in the network.
-
-    Args:
-        marketconfig (MarketConfig): The market configuration.
+        incidence_matrix (pd.DataFrame): The incidence matrix representing the power network connections.
+        nodes (list): List of nodes or zones in the network, depending on the selected representation.
+
+    Supported Parameters in `param_dict`:
+        solver (str): Specifies the solver to be used for the optimization problem. Default is 'appsi_highs'.
+        log_flows (bool): Indicates whether to log the power flows on the lines. Default is False.
+        pricing_mechanism (str): Defines the pricing mechanism to be used. Default is 'pay_as_clear', with an
+            alternative option of 'pay_as_bid'.
+        zones_identifier (str): The key in the bus data that identifies the zone each bus belongs to. Used
+            for zonal representation.
+
+    Example:
+        Example market configuration:
+        ```
+        market_mechanism: complex_clearing
+        param_dict:
+            solver: apps_highs
+            log_flows: true
+            pricing_mechanism: pay_as_clear
+            zones_identifier: zone_id
+        ```
+
+    Network Representations:
+        - Zonal Representation: The network is divided into zones, and the incidence matrix represents
+        the connections between these zones.
+            - If a `zones_identifier` is provided, buses are grouped into zones based on this identifier.
+            The incidence matrix is constructed to represent the power connections between these zones.
+            The total transfer capacity between zones is determined by the sum of the capacities of the
+            lines connecting the zones.
+
+        - Nodal Representation: If no `zones_identifier` is provided, each bus is treated as a separate
+        node, and the incidence matrix represents the connections between these nodes.
     """
 
     required_fields = ["bid_type"]
@@ -320,6 +338,11 @@ def __init__(self, marketconfig: MarketConfig):
                 self.incidence_matrix = create_incidence_matrix(self.lines, buses)
                 self.nodes = buses.index.values
 
+        self.log_flows = self.marketconfig.param_dict.get("log_flows", False)
+        self.pricing_mechanism = self.marketconfig.param_dict.get(
+            "pricing_mechanism", "pay_as_clear"
+        )
+
     def define_solver(self, solver: str):
         # Get the solver from the market configuration
         if solver == "highs":
@@ -423,6 +446,7 @@ def clear(
             accepted_orders (Orderbook): The accepted orders.
             rejected_orders (Orderbook): The rejected orders.
             meta (list[dict]): The market clearing results.
+            flows (dict): The power flows on the lines.
 
         Notes:
             First the market clearing is solved using the cost minimization with the pyomo model market_clearing_opt.
@@ -523,15 +547,14 @@ def clear(
             if all(order_surplus >= 0 for order_surplus in orders_surplus):
                 break
 
-        log_flows = True
-
         accepted_orders, rejected_orders, meta, flows = extract_results(
             model=instance,
             orders=orderbook,
             rejected_orders=rejected_orders,
             market_products=market_products,
             market_clearing_prices=market_clearing_prices,
-            log_flows=log_flows,
+            pricing_mechanism=self.pricing_mechanism,
+            log_flows=self.log_flows,
         )
 
         self.all_orders = []
@@ -622,6 +645,7 @@ def extract_results(
     rejected_orders: Orderbook,
     market_products: list[MarketProduct],
     market_clearing_prices: dict,
+    pricing_mechanism: str = "pay_as_clear",
     log_flows: bool = False,
 ):
     """
@@ -638,6 +662,9 @@ def extract_results(
         tuple[Orderbook, Orderbook, list[dict]]: The accepted orders, rejected orders, and meta information
 
     """
+    if pricing_mechanism not in ["pay_as_clear", "pay_as_bid"]:
+        raise ValueError(f"Invalid pricing mechanism {pricing_mechanism}")
+
     accepted_orders: Orderbook = []
     meta = []
 
@@ -651,9 +678,12 @@ def extract_results(
 
             # set the accepted volume and price for each simple bid
             order["accepted_volume"] = acceptance * order["volume"]
-            order["accepted_price"] = market_clearing_prices[order["node"]][
-                order["start_time"]
-            ]
+            if pricing_mechanism == "pay_as_clear":
+                order["accepted_price"] = market_clearing_prices[order["node"]][
+                    order["start_time"]
+                ]
+            elif pricing_mechanism == "pay_as_bid":
+                order["accepted_price"] = order["price"]
 
             # calculate the total cleared supply and demand volume
             if order["accepted_volume"] > 0:
@@ -672,9 +702,12 @@ def extract_results(
             # set the accepted volume and price for each block bid
             for start_time, volume in order["volume"].items():
                 order["accepted_volume"][start_time] = acceptance * volume
-                order["accepted_price"][start_time] = market_clearing_prices[
-                    order["node"]
-                ][start_time]
+                if pricing_mechanism == "pay_as_clear":
+                    order["accepted_price"][start_time] = market_clearing_prices[
+                        order["node"]
+                    ][start_time]
+                elif pricing_mechanism == "pay_as_bid":
+                    order["accepted_price"][start_time] = order["price"]
 
                 # calculate the total cleared supply and demand volume
                 if order["accepted_volume"][start_time] > 0:

assume/markets/clearing_algorithms/redispatch.py

@@ -214,8 +214,12 @@ def clear(
         )
 
         # return orderbook_df back to orderbook format as list of dicts
-        accepted_orders = orderbook_df[orderbook_df["accepted_volume"] != 0].to_dict("records")
-        rejected_orders = orderbook_df[orderbook_df["accepted_volume"] == 0].to_dict("records")
+        accepted_orders = orderbook_df[orderbook_df["accepted_volume"] != 0].to_dict(
+            "records"
+        )
+        rejected_orders = orderbook_df[orderbook_df["accepted_volume"] == 0].to_dict(
+            "records"
+        )
         meta = []
 
         # calculate meta data such as total upwared and downward redispatch, total backup dispatch

assume/scenario/loader_amiris.py

@@ -255,7 +255,7 @@ def add_agent_to_world(
                         "price": load["ValueOfLostLoad"],
                     },
                     # demand_series might contain more values than index
-                    NaiveForecast(index, demand=demand_series[:len(index)]),
+                    NaiveForecast(index, demand=demand_series[: len(index)]),
                 )
 
         case "StorageTrader":

assume/scenario/loader_oeds.py

@@ -215,12 +215,12 @@ def load_oeds(
         "postgresql://readonly:[email protected]:5432/opendata",
     )
 
-    default_nuts_config = 'DE1, DEA, DEB, DEC, DED, DEE, DEF'
+    default_nuts_config = "DE1, DEA, DEB, DEC, DED, DEE, DEF"
     nuts_config = os.getenv("NUTS_CONFIG", default_nuts_config).split(",")
     nuts_config = [n.strip() for n in nuts_config]
     year = 2019
     start = datetime(year, 1, 1)
-    end = datetime(year, 1+1, 1) - timedelta(hours=1)
+    end = datetime(year, 1 + 1, 1) - timedelta(hours=1)
     marketdesign = [
         MarketConfig(
             "EOM",

assume/scenario/loader_pypsa.py

@@ -95,7 +95,7 @@ def load_pypsa(
                 "bidding_strategies": bidding_strategies[unit_type][generator.name],
                 "technology": "conventional",
                 "node": generator.node,
-                "efficiency": 1, # do not use generator.efficiency as it is respected in marginal_cost,
+                "efficiency": 1,  # do not use generator.efficiency as it is respected in marginal_cost,
                 "fuel_type": generator.carrier,
                 "ramp_up": ramp_up,
                 "ramp_down": ramp_down,
@@ -173,7 +173,7 @@ def load_pypsa(
     scenario = "world_pypsa"
     study_case = "ac_dc_meshed"
     # "pay_as_clear", "redispatch" or "nodal"
-    market_mechanism = "pay_as_clear_complex"
+    market_mechanism = "complex_clearing"
 
     match study_case:
         case "ac_dc_meshed":
@@ -186,7 +186,7 @@ def load_pypsa(
             logger.info(f"invalid studycase: {study_case}")
             network = pd.DataFrame()
 
-    study_case = f"{study_case}_{market_mechanism}" 
+    study_case = f"{study_case}_{market_mechanism}"
 
     start = network.snapshots[0]
     end = network.snapshots[-1]
@@ -206,7 +206,12 @@ def load_pypsa(
         marketdesign.append(
             MarketConfig(
                 "EOM",
-                rr.rrule(rr.HOURLY, interval=1, dtstart=start-timedelta(hours=0.5), until=end),
+                rr.rrule(
+                    rr.HOURLY,
+                    interval=1,
+                    dtstart=start - timedelta(hours=0.5),
+                    until=end,
+                ),
                 timedelta(hours=0.25),
                 "pay_as_clear",
                 [MarketProduct(timedelta(hours=1), 1, timedelta(hours=1.5))],
@@ -225,7 +230,9 @@ def load_pypsa(
 
     bidding_strategies = {
         "power_plant": defaultdict(lambda: default_strategies),
-        "demand": defaultdict(lambda: {mc.market_id: "naive_eom" for mc in marketdesign}),
+        "demand": defaultdict(
+            lambda: {mc.market_id: "naive_eom" for mc in marketdesign}
+        ),
         "storage": defaultdict(lambda: default_strategies),
     }
 

docker_configs/dashboard-definitions/ASSUME Comparison.json

@@ -2396,4 +2396,4 @@
   "uid": "vP8U8-q4k",
   "version": 5,
   "weekStart": ""
-}
+}

docker_configs/dashboard-definitions/ASSUME.json

@@ -4683,4 +4683,4 @@
   "uid": "mQ3Lvkr4k",
   "version": 3,
   "weekStart": ""
-}
+}

docker_configs/dashboard-definitions/ASSUME_nodal.json

@@ -1048,4 +1048,4 @@
   "uid": "nodalview",
   "version": 21,
   "weekStart": ""
-}
+}

examples/inputs/example_01a/config.yaml

@@ -92,7 +92,10 @@ dam_with_complex_clearing:
       maximum_bid_price: 3000
       minimum_bid_price: -500
       price_unit: EUR/MWh
-      market_mechanism: pay_as_clear_complex
+      market_mechanism: complex_clearing
+      param_dict:
+        solver: highs
+        pricing_mechanism: pay_as_clear
       additional_fields:
         - bid_type
         - min_acceptance_ratio

examples/inputs/example_01c/config.yaml

@@ -25,9 +25,10 @@ eom_only:
       price_unit: EUR/MWh
       additional_fields:
         - bid_type
-      market_mechanism: pay_as_clear_complex
+      market_mechanism: complex_clearing
       param_dict:
         solver: highs
+        pricing_mechanism: pay_as_clear
 
 eom_and_crm:
   start_date: 2019-01-01 00:00
@@ -55,9 +56,10 @@ eom_and_crm:
       price_unit: EUR/MWh
       additional_fields:
         - bid_type
-      market_mechanism: pay_as_clear_complex
+      market_mechanism: complex_clearing
       param_dict:
         solver: highs
+        pricing_mechanism: pay_as_clear
 
     CRM_pos:
       operator: CRM_operator
@@ -114,10 +116,11 @@ dam_with_complex_opt_clearing:
       maximum_bid_price: 3000
       minimum_bid_price: -500
       price_unit: EUR/MWh
-      market_mechanism: pay_as_clear_complex
+      market_mechanism: complex_clearing
+      param_dict:
+        solver: highs
+        pricing_mechanism: pay_as_clear
       additional_fields:
         - bid_type
         - min_acceptance_ratio
         - parent_bid_id
-      param_dict:
-        solver: highs

examples/inputs/example_01d/config.yaml

@@ -73,11 +73,12 @@ zonal_case:
       maximum_bid_price: 3000
       minimum_bid_price: -500
       price_unit: EUR/MWh
-      market_mechanism: pay_as_clear_complex
-      additional_fields:
-        - bid_type
-        - node
+      market_mechanism: complex_clearing
       param_dict:
         network_path: .
         solver: highs
         zones_identifier: zone_id
+        pricing_mechanism: pay_as_clear
+      additional_fields:
+        - bid_type
+        - node