Merge pull request #539 from liampauling/task/538-smart-matching

#538 passive; prevent double counting of traded size

HISTORY.rst

@@ -3,6 +3,21 @@
 Release History
 ---------------
 
+1.21.0 (2022-01-06)
++++++++++++++++++++
+
+**Improvements**
+
+- #528 smart matching on passive orders
+
+**Bug Fixes**
+
+- #528 simulation processing on in flight requests
+
+**Libraries**
+
+- black upgraded to 21.12b0
+
 1.20.13 (2021-12-03)
 +++++++++++++++++++
 

docs/advanced.md

@@ -112,6 +112,10 @@ logger.setLevel(logging.INFO)
 
 Updated to True when backtesting or paper trading
 
+#### simulated_strategy_isolation
+
+Defaults to True to match orders per strategy, when False prevents double counting of passive liquidity on all orders regardless of strategy.
+
 #### instance_id
 
 Store server id or similar (e.g. AWS ec2 instanceId)

flumine/__version__.py

@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betfair trading framework"
 __url__ = "https://github.com/liampauling/flumine"
-__version__ = "1.20.13"
+__version__ = "1.21.0"
 __author__ = "Liam Pauling"
 __license__ = "MIT"

flumine/backtest/simulated.py

@@ -33,7 +33,7 @@ def __init__(self, order):
         self._piq = 0.0
         self._bsp_reconciled = False
 
-    def __call__(self, market_book: MarketBook, runner_analytics) -> None:
+    def __call__(self, market_book: MarketBook, traded: dict) -> None:
         # simulates order matching
         if self._bsp_reconciled is False and market_book.bsp_reconciled:
             if self.take_sp:
@@ -52,10 +52,8 @@ def __call__(self, market_book: MarketBook, runner_analytics) -> None:
                         return
 
             # todo estimated piq cancellations
-            if runner_analytics.traded:
-                self._process_traded(
-                    market_book.publish_time_epoch, runner_analytics.traded
-                )
+            if traded:
+                self._process_traded(market_book.publish_time_epoch, traded)
 
     def place(
         self, order_package, market_book: MarketBook, instruction: dict, bet_id: int
@@ -94,11 +92,10 @@ def place(
                 error_code="RUNNER_REMOVED",
             )
         if self.order.order_type.ORDER_TYPE == OrderTypes.LIMIT:
-            available_to_back = get_price(runner.ex.available_to_back, 0) or 1.01
-            available_to_lay = get_price(runner.ex.available_to_lay, 0) or 1000
             price = self.order.order_type.price
             size = self.order.order_type.size
             if self.order.side == "BACK":
+                available_to_back = get_price(runner.ex.available_to_back, 0) or 1.01
                 if (
                     not order_package.client.best_price_execution
                     and available_to_back > price
@@ -119,6 +116,7 @@ def place(
                     return self._create_place_response(bet_id)
                 available = runner.ex.available_to_lay
             else:
+                available_to_lay = get_price(runner.ex.available_to_lay, 0) or 1000
                 if (
                     not order_package.client.best_price_execution
                     and available_to_lay < price
@@ -331,30 +329,37 @@ def _process_traded(self, publish_time: int, traded: dict) -> None:
                     )
                 )
             if side == "BACK" and traded_price >= price:
-                self._calculate_process_traded(publish_time, traded_size)
+                matched = self._calculate_process_traded(publish_time, traded_size)
+                if matched:
+                    traded[traded_price] = traded_size - matched
             elif side == "LAY" and traded_price <= price:
-                self._calculate_process_traded(publish_time, traded_size)
-
-    def _calculate_process_traded(self, publish_time: int, traded_size: float) -> None:
-        traded_size = traded_size / 2
-        if self._piq - traded_size < 0:
-            size = traded_size - self._piq
+                matched = self._calculate_process_traded(publish_time, traded_size)
+                if matched:
+                    traded[traded_price] = traded_size - matched
+
+    def _calculate_process_traded(self, publish_time: int, traded_size: float) -> float:
+        _traded_size = traded_size / 2
+        if self._piq - _traded_size < 0:
+            size = _traded_size - self._piq
             size = round(min(self.size_remaining, size), 2)
             if size:
                 self._update_matched(
                     [
                         publish_time,
                         self.order.order_type.price,
                         size,
-                    ]  # todo takes the worst price, i.e what was asked
+                    ]
                 )
+            _matched = (self._piq + size) * 2
             self._piq = 0
+            return _matched
         else:
-            self._piq -= traded_size
+            self._piq -= _traded_size
             if logger.isEnabledFor(logging.DEBUG):
                 logger.debug(
                     "Simulated order {0} PIQ: {1}".format(self.order.id, self._piq)
                 )
+            return traded_size
 
     @property
     def take_sp(self) -> bool:

flumine/config.py

@@ -2,6 +2,7 @@
 import socket
 
 simulated = False
+simulated_strategy_isolation = True
 
 instance_id = None  # instance id (e.g. AWS ec2 instanceId)
 

flumine/markets/middleware.py

@@ -4,6 +4,7 @@
 
 from ..order.order import OrderStatus, OrderTypes
 from ..utils import wap
+from .. import config
 
 logger = logging.getLogger(__name__)
 
@@ -180,14 +181,81 @@ def _calculate_reduction_factor(price: float, adjustment_factor: float) -> float
         price_adjusted = round(price * (1 - (adjustment_factor / 100)), 2)
         return max(price_adjusted, 1.01)  # min: 1.01
 
-    @staticmethod
-    def _process_simulated_orders(market, market_analytics: dict) -> None:
-        for order in market.blotter.live_orders:
-            if order.status == OrderStatus.EXECUTABLE and order.simulated:
-                runner_analytics = market_analytics[
-                    (order.selection_id, order.handicap)
+    def _process_simulated_orders(self, market, market_analytics: dict) -> None:
+        """
+        #538 smart matching
+          - isolation per order
+            Potential double counting of passive liquidity, old logic no longer implemented
+          - isolation per strategy (default)
+            Prevent double counting of passive liquidity per strategy
+          - isolation per instance
+            Prevent double counting of passive liquidity on all orders regardless of strategy (interaction across strategies)
+        """
+        # isolation per strategy (default)
+        if config.simulated_strategy_isolation:
+            for strategy, orders in market.blotter._strategy_orders.items():
+                live_orders = [
+                    o
+                    for o in orders
+                    if o.status
+                    in [
+                        OrderStatus.EXECUTABLE,
+                        OrderStatus.CANCELLING,
+                        OrderStatus.UPDATING,
+                        OrderStatus.REPLACING,
+                    ]
+                    and o.simulated
                 ]
-                order.simulated(market.market_book, runner_analytics)
+                if live_orders:
+                    _lookup = {k: v.traded.copy() for k, v in market_analytics.items()}
+                    live_orders_sorted = self._sort_orders(live_orders)
+                    for order in live_orders_sorted:
+                        runner_traded = _lookup[(order.selection_id, order.handicap)]
+                        order.simulated(market.market_book, runner_traded)
+        else:  # isolation per instance
+            live_orders = list(market.blotter.live_orders)
+            if live_orders:
+                _lookup = {k: v.traded.copy() for k, v in market_analytics.items()}
+                live_orders_sorted = self._sort_orders(live_orders)
+                for order in live_orders_sorted:
+                    if (
+                        order.status
+                        in [
+                            OrderStatus.EXECUTABLE,
+                            OrderStatus.CANCELLING,
+                            OrderStatus.UPDATING,
+                            OrderStatus.REPLACING,
+                        ]
+                        and order.simulated
+                    ):
+                        runner_traded = _lookup[(order.selection_id, order.handicap)]
+                        order.simulated(market.market_book, runner_traded)
+
+    @staticmethod
+    def _sort_orders(orders: list) -> list:
+        # order by betId (default), side (Lay,Back) and then price
+        lay_orders = sorted(
+            [
+                o
+                for o in orders
+                if o.side == "LAY"
+                and o.order_type.ORDER_TYPE != OrderTypes.MARKET_ON_CLOSE
+            ],
+            key=lambda x: -x.order_type.price,
+        )
+        back_orders = sorted(
+            [
+                o
+                for o in orders
+                if o.side == "BACK"
+                and o.order_type.ORDER_TYPE != OrderTypes.MARKET_ON_CLOSE
+            ],
+            key=lambda x: x.order_type.price,
+        )
+        moc = [
+            o for o in orders if o.order_type.ORDER_TYPE == OrderTypes.MARKET_ON_CLOSE
+        ]
+        return lay_orders + back_orders + moc
 
     @staticmethod
     def _process_runner(

requirements-test.txt

@@ -1,5 +1,5 @@
 # Tests & Linting
-black==21.11b1
+black==21.12b0
 coverage
 
 # Documentation

tests/test_config.py

@@ -6,6 +6,7 @@
 class ConfigTest(unittest.TestCase):
     def test_init(self):
         self.assertFalse(config.simulated)
+        self.assertTrue(config.simulated_strategy_isolation)
         self.assertIsInstance(config.customer_strategy_ref, str)
         self.assertIsInstance(config.process_id, int)
         self.assertIsNone(config.current_time)

tests/test_middleware.py

@@ -270,12 +270,43 @@ def test__calculate_reduction_factor(self):
         self.assertEqual(self.middleware._calculate_reduction_factor(10, 75.18), 2.48)
         self.assertEqual(self.middleware._calculate_reduction_factor(1.01, 75.18), 1.01)
 
-    def test__process_simulated_orders(self):
+    @mock.patch("flumine.markets.middleware.config")
+    def test__process_simulated_orders_strategy_isolation(self, mock_config):
+        mock_config.simulated_strategy_isolation = True
         mock_market_book = mock.Mock()
         mock_market = mock.Mock()
         mock_order = mock.Mock(
-            selection_id=123, handicap=1, status=OrderStatus.EXECUTABLE
+            selection_id=123, handicap=1, status=OrderStatus.EXECUTABLE, side="LAY"
         )
+        mock_order.order_type.price = 1.02
+        mock_order.order_type.ORDER_TYPE = OrderTypes.LIMIT
+        mock_order_two = mock.Mock(
+            selection_id=123, handicap=1, status=OrderStatus.PENDING
+        )
+        mock_order_three = mock.Mock(
+            selection_id=123, handicap=1, status=OrderStatus.EXECUTABLE, simulated=False
+        )
+        mock_market.blotter._strategy_orders = {
+            "test": [mock_order, mock_order_two, mock_order_three]
+        }
+        mock_market_analytics = {
+            (mock_order.selection_id, mock_order.handicap): mock.Mock(traded={1: 2})
+        }
+        mock_market.market_book = mock_market_book
+        self.middleware._process_simulated_orders(mock_market, mock_market_analytics)
+        mock_order.simulated.assert_called_with(mock_market_book, {1: 2})
+        mock_order_two.simulated.assert_not_called()
+
+    @mock.patch("flumine.markets.middleware.config")
+    def test__process_simulated_orders(self, mock_config):
+        mock_config.simulated_strategy_isolation = False
+        mock_market_book = mock.Mock()
+        mock_market = mock.Mock()
+        mock_order = mock.Mock(
+            selection_id=123, handicap=1, status=OrderStatus.EXECUTABLE, side="LAY"
+        )
+        mock_order.order_type.price = 1.02
+        mock_order.order_type.ORDER_TYPE = OrderTypes.LIMIT
         mock_order_two = mock.Mock(
             selection_id=123, handicap=1, status=OrderStatus.PENDING
         )
@@ -287,12 +318,52 @@ def test__process_simulated_orders(self):
             mock_order_two,
             mock_order_three,
         ]
-        mock_market_analytics = {(mock_order.selection_id, mock_order.handicap): "test"}
+        mock_market_analytics = {
+            (mock_order.selection_id, mock_order.handicap): mock.Mock(traded={1: 2})
+        }
         mock_market.market_book = mock_market_book
         self.middleware._process_simulated_orders(mock_market, mock_market_analytics)
-        mock_order.simulated.assert_called_with(mock_market_book, "test")
+        mock_order.simulated.assert_called_with(mock_market_book, {1: 2})
         mock_order_two.simulated.assert_not_called()
 
+    def test__sort_orders(self):
+        order_one = mock.Mock(side="LAY", bet_id=1)
+        order_one.order_type.price = 1.01
+        order_two = mock.Mock(side="LAY", bet_id=2)
+        order_two.order_type.price = 1.02
+        order_three = mock.Mock(side="LAY", bet_id=3)
+        order_three.order_type.price = 1.01
+        order_four = mock.Mock(side="BACK", bet_id=4)
+        order_four.order_type.price = 1.2
+        order_five = mock.Mock(side="BACK", bet_id=5)
+        order_five.order_type.price = 1.2
+        order_six = mock.Mock(side="BACK", bet_id=6)
+        order_six.order_type.price = 1.19
+        order_seven = mock.Mock(side="BACK", bet_id=6)
+        order_seven.order_type.price = "ERROR"
+        order_seven.order_type.ORDER_TYPE = OrderTypes.MARKET_ON_CLOSE
+        orders = [
+            order_one,
+            order_two,
+            order_three,
+            order_four,
+            order_five,
+            order_six,
+            order_seven,
+        ]
+        self.assertEqual(
+            self.middleware._sort_orders(orders),
+            [
+                order_two,
+                order_one,
+                order_three,
+                order_six,
+                order_four,
+                order_five,
+                order_seven,
+            ],
+        )
+
     @mock.patch("flumine.markets.middleware.RunnerAnalytics")
     def test__process_runner(self, mock_runner_analytics):
         market_analytics = {}