Adds pay as bid option to the complex clearing

assume/common/forecasts.py

@@ -557,8 +557,6 @@ def __init__(
         for key, value in kwargs.items():
             self.data_dict[key] = FastSeries(index=self.index, value=value, name=key)
 
-
-
     def __getitem__(self, column: str) -> FastSeries:
         """
         Retrieves forecasted values.

assume/common/grid_utils.py

@@ -49,7 +49,12 @@
         )
     else:
         # add generators
-        generators.drop(["p_min_pu", "p_max_pu", "marginal_cost"], axis=1, inplace=True, errors="ignore")
+        generators.drop(
+            ["p_min_pu", "p_max_pu", "marginal_cost"],
+            axis=1,
+            inplace=True,
+            errors="ignore",
+        )
         network.add(
             "Generator",
             name=generators.index,

assume/common/outputs.py

@@ -687,7 +687,7 @@ def get_sum_reward(self):
         )
         if self.db is None:
             return []
-        
+
         with self.db.begin() as db:
             rewards_by_unit = db.execute(query).fetchall()
 

assume/markets/clearing_algorithms/__init__.py

@@ -14,7 +14,7 @@
     "pay_as_clear": PayAsClearRole,
     "pay_as_bid": PayAsBidRole,
     "pay_as_bid_contract": PayAsBidContractRole,
-    "pay_as_clear_complex": ComplexClearingRole,
+    "complex_clearing": ComplexClearingRole,
     "pay_as_clear_complex_dmas": ComplexDmasClearingRole,
 }
 

assume/markets/clearing_algorithms/complex_clearing.py

@@ -258,33 +258,43 @@
 
 class ComplexClearingRole(MarketRole):
     """
-    Defines the clearing algorithm for the complex market.
+    Defines an optimization-based market clearing algorithm with support for complex bid types, including block bids, linked bids,
+    minimum acceptance ratios, and profiled volumes. This class also supports network representations with either zonal or nodal
+    configurations, allowing the modeling of complex markets with multiple zones and power flow constraints.
 
-    The complex market is a pay-as-clear market with more complex bid structures, including minimum acceptance ratios, bid types, and profiled volumes.
+    The market clearing algorithm accepts the following additional arguments via the `param_dict` in the market configuration:
 
-    The class supports two types of network representations:
+    - `solver` (str): Specifies the solver to be used for the optimization problem. Default is 'appsi_highs'.
+    - `log_flows` (bool): Indicates whether to log the power flows on the lines. Default is False.
+    - `pricing_mechanism` (str): Defines the pricing mechanism to be used. Default is 'pay_as_clear', with an alternative option of 'pay_as_bid'.
+    - `zones_identifier` (str): The key in the bus data that identifies the zone each bus belongs to. Used for zonal representation.
+
+    Example market configuration:
+    ```
+    market_mechanism: complex_clearing
+    param_dict:
+        solver: apps_highs
+        log_flows: true
+        pricing_mechanism: pay_as_clear
+        zones_identifier: zone_id
+    ```
 
-    1. **Zonal Representation**: The network is divided into zones, and the incidence matrix represents the connections between these zones.
+    Network Representations:
 
-    2. **Nodal Representation**: Each bus in the network is treated as a node, and the incidence matrix represents the connections between these nodes.
+    The class supports two types of network representations:
 
-    Zonal Representation:
-        If a `zones_identifier` is provided in the market configuration param_dict, the buses are grouped into zones based on this identifier.
-        The incidence matrix is then constructed to represent the power connections between these zones. The total transfer
-        capacity between zones is determined by the sum of the capacities of the lines connecting the zones.
-        - `zones_identifier` (str): The key in the bus data that identifies the zone to which each bus belongs.
+    1. Zonal Representation: The network is divided into zones, and the incidence matrix represents the connections between these zones.
+       - If a `zones_identifier` is provided, buses are grouped into zones based on this identifier. The incidence matrix is then constructed to represent the power connections between these zones. The total transfer capacity between zones is determined by the sum of the capacities of the lines connecting the zones.
 
-    Nodal Representation:
-        If no `zones_identifier` is provided, each bus is treated as a separate node.
-        The incidence matrix is constructed to represent the power connections between these nodes.
+    2. Nodal Representation: If no `zones_identifier` is provided, each bus is treated as a separate node, and the incidence matrix represents the connections between these nodes.
 
     Attributes:
-        marketconfig (MarketConfig): The market configuration.
-        incidence_matrix (pd.DataFrame): The incidence matrix representing the network connections.
-        nodes (list): List of nodes or zones in the network.
+    - `marketconfig` (MarketConfig): The market configuration.
+    - `incidence_matrix` (pd.DataFrame): The incidence matrix representing the power network connections.
+    - `nodes` (list): List of nodes or zones in the network, depending on the selected representation.
 
     Args:
-        marketconfig (MarketConfig): The market configuration.
+    - `marketconfig` (MarketConfig): The market configuration object containing all parameters for the market clearing process.
     """
 
     required_fields = ["bid_type"]
@@ -320,6 +330,11 @@
                 self.incidence_matrix = create_incidence_matrix(self.lines, buses)
                 self.nodes = buses.index.values
 
+        self.log_flows = self.marketconfig.param_dict.get("log_flows", False)
+        self.pricing_mechanism = self.marketconfig.param_dict.get(
+            "pricing_mechanism", "pay_as_clear"
+        )
+
     def define_solver(self, solver: str):
         # Get the solver from the market configuration
         if solver == "highs":
@@ -423,6 +438,7 @@
             accepted_orders (Orderbook): The accepted orders.
             rejected_orders (Orderbook): The rejected orders.
             meta (list[dict]): The market clearing results.
+            flows (dict): The power flows on the lines.
 
         Notes:
             First the market clearing is solved using the cost minimization with the pyomo model market_clearing_opt.
@@ -523,15 +539,14 @@
             if all(order_surplus >= 0 for order_surplus in orders_surplus):
                 break
 
-        log_flows = True
-
         accepted_orders, rejected_orders, meta, flows = extract_results(
             model=instance,
             orders=orderbook,
             rejected_orders=rejected_orders,
             market_products=market_products,
             market_clearing_prices=market_clearing_prices,
-            log_flows=log_flows,
+            pricing_mechanism=self.pricing_mechanism,
+            log_flows=self.log_flows,
         )
 
         self.all_orders = []
@@ -622,6 +637,7 @@
     rejected_orders: Orderbook,
     market_products: list[MarketProduct],
     market_clearing_prices: dict,
+    pricing_mechanism: str = "pay_as_clear",
     log_flows: bool = False,
 ):
     """
@@ -638,6 +654,9 @@
         tuple[Orderbook, Orderbook, list[dict]]: The accepted orders, rejected orders, and meta information
 
     """
+    if pricing_mechanism not in ["pay_as_clear", "pay_as_bid"]:
+        raise ValueError(f"Invalid pricing mechanism {pricing_mechanism}")
+
     accepted_orders: Orderbook = []
     meta = []
 
@@ -651,9 +670,12 @@
 
             # set the accepted volume and price for each simple bid
             order["accepted_volume"] = acceptance * order["volume"]
-            order["accepted_price"] = market_clearing_prices[order["node"]][
-                order["start_time"]
-            ]
+            if pricing_mechanism == "pay_as_clear":
+                order["accepted_price"] = market_clearing_prices[order["node"]][
+                    order["start_time"]
+                ]
+            elif pricing_mechanism == "pay_as_bid":
+                order["accepted_price"] = order["price"]
 
             # calculate the total cleared supply and demand volume
             if order["accepted_volume"] > 0:
@@ -672,9 +694,12 @@
             # set the accepted volume and price for each block bid
             for start_time, volume in order["volume"].items():
                 order["accepted_volume"][start_time] = acceptance * volume
-                order["accepted_price"][start_time] = market_clearing_prices[
-                    order["node"]
-                ][start_time]
+                if pricing_mechanism == "pay_as_clear":
+                    order["accepted_price"][start_time] = market_clearing_prices[
+                        order["node"]
+                    ][start_time]
+                elif pricing_mechanism == "pay_as_bid":
+                    order["accepted_price"][start_time] = order["price"]
 
                 # calculate the total cleared supply and demand volume
                 if order["accepted_volume"][start_time] > 0:

assume/markets/clearing_algorithms/redispatch.py

@@ -214,8 +214,12 @@
         )
 
         # return orderbook_df back to orderbook format as list of dicts
-        accepted_orders = orderbook_df[orderbook_df["accepted_volume"] != 0].to_dict("records")
-        rejected_orders = orderbook_df[orderbook_df["accepted_volume"] == 0].to_dict("records")
+        accepted_orders = orderbook_df[orderbook_df["accepted_volume"] != 0].to_dict(
+            "records"
+        )
+        rejected_orders = orderbook_df[orderbook_df["accepted_volume"] == 0].to_dict(
+            "records"
+        )
         meta = []
 
         # calculate meta data such as total upwared and downward redispatch, total backup dispatch

assume/scenario/loader_amiris.py

@@ -255,7 +255,7 @@ def add_agent_to_world(
                         "price": load["ValueOfLostLoad"],
                     },
                     # demand_series might contain more values than index
-                    NaiveForecast(index, demand=demand_series[:len(index)]),
+                    NaiveForecast(index, demand=demand_series[: len(index)]),
                 )
 
         case "StorageTrader":

assume/scenario/loader_oeds.py

@@ -215,12 +215,12 @@ def load_oeds(
         "postgresql://readonly:[email protected]:5432/opendata",
     )
 
-    default_nuts_config = 'DE1, DEA, DEB, DEC, DED, DEE, DEF'
+    default_nuts_config = "DE1, DEA, DEB, DEC, DED, DEE, DEF"
     nuts_config = os.getenv("NUTS_CONFIG", default_nuts_config).split(",")
     nuts_config = [n.strip() for n in nuts_config]
     year = 2019
     start = datetime(year, 1, 1)
-    end = datetime(year, 1+1, 1) - timedelta(hours=1)
+    end = datetime(year, 1 + 1, 1) - timedelta(hours=1)
     marketdesign = [
         MarketConfig(
             "EOM",

assume/scenario/loader_pypsa.py

@@ -95,7 +95,7 @@ def load_pypsa(
                 "bidding_strategies": bidding_strategies[unit_type][generator.name],
                 "technology": "conventional",
                 "node": generator.node,
-                "efficiency": 1, # do not use generator.efficiency as it is respected in marginal_cost,
+                "efficiency": 1,  # do not use generator.efficiency as it is respected in marginal_cost,
                 "fuel_type": generator.carrier,
                 "ramp_up": ramp_up,
                 "ramp_down": ramp_down,
@@ -173,7 +173,7 @@ def load_pypsa(
     scenario = "world_pypsa"
     study_case = "ac_dc_meshed"
     # "pay_as_clear", "redispatch" or "nodal"
-    market_mechanism = "pay_as_clear_complex"
+    market_mechanism = "complex_clearing"
 
     match study_case:
         case "ac_dc_meshed":
@@ -186,7 +186,7 @@ def load_pypsa(
             logger.info(f"invalid studycase: {study_case}")
             network = pd.DataFrame()
 
-    study_case = f"{study_case}_{market_mechanism}" 
+    study_case = f"{study_case}_{market_mechanism}"
 
     start = network.snapshots[0]
     end = network.snapshots[-1]
@@ -206,7 +206,12 @@ def load_pypsa(
         marketdesign.append(
             MarketConfig(
                 "EOM",
-                rr.rrule(rr.HOURLY, interval=1, dtstart=start-timedelta(hours=0.5), until=end),
+                rr.rrule(
+                    rr.HOURLY,
+                    interval=1,
+                    dtstart=start - timedelta(hours=0.5),
+                    until=end,
+                ),
                 timedelta(hours=0.25),
                 "pay_as_clear",
                 [MarketProduct(timedelta(hours=1), 1, timedelta(hours=1.5))],
@@ -225,7 +230,9 @@ def load_pypsa(
 
     bidding_strategies = {
         "power_plant": defaultdict(lambda: default_strategies),
-        "demand": defaultdict(lambda: {mc.market_id: "naive_eom" for mc in marketdesign}),
+        "demand": defaultdict(
+            lambda: {mc.market_id: "naive_eom" for mc in marketdesign}
+        ),
         "storage": defaultdict(lambda: default_strategies),
     }
 

docker_configs/dashboard-definitions/ASSUME Comparison.json

@@ -2396,4 +2396,4 @@
   "uid": "vP8U8-q4k",
   "version": 5,
   "weekStart": ""
-}
+}

docker_configs/dashboard-definitions/ASSUME.json

@@ -4683,4 +4683,4 @@
   "uid": "mQ3Lvkr4k",
   "version": 3,
   "weekStart": ""
-}
+}

docker_configs/dashboard-definitions/ASSUME_nodal.json

@@ -1048,4 +1048,4 @@
   "uid": "nodalview",
   "version": 21,
   "weekStart": ""
-}
+}

examples/inputs/example_01a/config.yaml

@@ -92,7 +92,10 @@ dam_with_complex_clearing:
       maximum_bid_price: 3000
       minimum_bid_price: -500
       price_unit: EUR/MWh
-      market_mechanism: pay_as_clear_complex
+      market_mechanism: complex_clearing
+      param_dict:
+        solver: highs
+        pricing_mechanism: pay_as_clear
       additional_fields:
         - bid_type
         - min_acceptance_ratio

examples/inputs/example_01c/config.yaml

@@ -25,9 +25,10 @@ eom_only:
       price_unit: EUR/MWh
       additional_fields:
         - bid_type
-      market_mechanism: pay_as_clear_complex
+      market_mechanism: complex_clearing
       param_dict:
         solver: highs
+        pricing_mechanism: pay_as_clear
 
 eom_and_crm:
   start_date: 2019-01-01 00:00
@@ -55,9 +56,10 @@ eom_and_crm:
       price_unit: EUR/MWh
       additional_fields:
         - bid_type
-      market_mechanism: pay_as_clear_complex
+      market_mechanism: complex_clearing
       param_dict:
         solver: highs
+        pricing_mechanism: pay_as_clear
 
     CRM_pos:
       operator: CRM_operator
@@ -114,10 +116,11 @@ dam_with_complex_opt_clearing:
       maximum_bid_price: 3000
       minimum_bid_price: -500
       price_unit: EUR/MWh
-      market_mechanism: pay_as_clear_complex
+      market_mechanism: complex_clearing
+      param_dict:
+        solver: highs
+        pricing_mechanism: pay_as_clear
       additional_fields:
         - bid_type
         - min_acceptance_ratio
         - parent_bid_id
-      param_dict:
-        solver: highs

examples/inputs/example_01d/config.yaml

@@ -73,11 +73,12 @@ zonal_case:
       maximum_bid_price: 3000
       minimum_bid_price: -500
       price_unit: EUR/MWh
-      market_mechanism: pay_as_clear_complex
-      additional_fields:
-        - bid_type
-        - node
+      market_mechanism: complex_clearing
       param_dict:
         network_path: .
         solver: highs
         zones_identifier: zone_id
+        pricing_mechanism: pay_as_clear
+      additional_fields:
+        - bid_type
+        - node

examples/inputs/example_02d/config.yaml

@@ -46,13 +46,14 @@ dam:
       maximum_bid_price: 3000
       minimum_bid_price: -500
       price_unit: EUR/MWh
-      market_mechanism: pay_as_clear_complex
+      market_mechanism: complex_clearing
+      param_dict:
+        solver: highs
+        pricing_mechanism: pay_as_clear
       additional_fields:
         - bid_type
         - min_acceptance_ratio
         - parent_bid_id
-      param_dict:
-        solver: highs
 
 tiny:
   start_date: 2019-01-01 00:00