From: Willy Tarreau Date: Thu, 25 Jun 2026 07:13:27 +0000 (+0200) Subject: MEDIUM: task: add a new tasklet class for real-time: TL_RT X-Git-Tag: v3.5-dev1~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e37a20d3309d3b3e723e0aef6dae671d97150736;p=thirdparty%2Fhaproxy.git MEDIUM: task: add a new tasklet class for real-time: TL_RT This adds new class TL_RT, which is processed before other queues for one (and only one) tasklet featuring the TASK_RT flag. This is meant to process real time wakeups under load with even less latency. We only process one entry to make sure it will not be abused for unimportant stuff, and if tune.sched.low-latency is set, we also avoid picking more tasks from the current run queues and looping after the first call to run_tasks_from_list(). Measurements under a load of 10k concurrent conns injection at 10 Gbps (~58k 20kB objects/s) on 4 threads and with task profiling enabled shows that the average wakeup latency for wakeups every 10ms dropped from 220 microseconds to 1.8 microsecond, and even ~550 nanoseconds when tune.sched.low-latency is set, or 400 times less. The doc was updated, including the schematics. --- diff --git a/doc/internals/api/scheduler.txt b/doc/internals/api/scheduler.txt index 2ff149bbe..25680bf67 100644 --- a/doc/internals/api/scheduler.txt +++ b/doc/internals/api/scheduler.txt @@ -272,17 +272,22 @@ application, both for tasks and tasklets: sense to reset this flag from the ->process() function itself. - - TASK_RT when set, indicates that the task has real-time - constraints (this is ignored for tasklets). Such a task - will bypass the priority ordering of the run queue in - order to guarantee a wakeup time as close as possible to - the scheduled one even under load. Only one such task - may be executed per round so this must be restricted to - a few timing-critical tasks only (those for which a one - millisecond skew is not acceptable). Such tasks are not - meant to be woken up by other threads than the one they - are supposed to run on, otherwise(their constraints may - not be honored. + - TASK_RT when set, indicates that the task or tasklet has strong + real-time constraints. A task with this flag will bypass + the priority ordering of the run queue in order to + guarantee a wakeup time as close as possible to the + scheduled one even under load. A task or tasklet with + this flag will be executed in its own queue before all + other queues are consulted. When tune.sched.low-latency + is set, no new tasks will be dequeued from the priority + queue if such a task is present in the run queue. Only + one such task or tasklet may be executed per round so + this must be restricted to a few timing-critical tasks + only (those for which a one millisecond skew is not + acceptable). Such tasks/tasklets are not meant to be + woken up by other threads than the one they are supposed + to run on, otherwise(their constraints may not be + honored. - TASK_HEAVY when set, indicates that this task does so heavy processing that it will become mandatory to give back diff --git a/doc/internals/sched.fig b/doc/internals/sched.fig index 4134420cb..9a470334c 100644 --- a/doc/internals/sched.fig +++ b/doc/internals/sched.fig @@ -1,4 +1,4 @@ -#FIG 3.2 Produced by xfig version 2.4 +#FIG 3.2 Produced by xfig version 3.2.8b Landscape Center Metric @@ -287,18 +287,6 @@ Single 0 309 #9ec3d7 0 310 #ffff96 0 311 #ff600a -5 1 0 2 0 7 50 -1 -1 0.000 0 0 1 0 11301.000 3060.000 11205 3825 10530 3060 11205 2295 - 0 0 1.00 60.00 120.00 -5 1 0 2 0 7 50 -1 -1 0.000 0 1 0 1 11289.000 3060.000 11385 3825 12060 3060 11385 2295 - 0 0 1.00 60.00 120.00 -5 1 0 2 0 7 50 -1 -1 0.000 0 1 0 1 11293.750 3060.000 10890 3105 11700 3060 10890 3015 - 2 1 1.00 60.00 120.00 -5 1 0 2 0 7 50 -1 -1 0.000 0 0 1 0 7611.000 3060.000 7515 3825 6840 3060 7515 2295 - 0 0 1.00 60.00 120.00 -5 1 0 2 0 7 50 -1 -1 0.000 0 1 0 1 7599.000 3060.000 7695 3825 8370 3060 7695 2295 - 0 0 1.00 60.00 120.00 -5 1 0 2 0 7 50 -1 -1 0.000 0 1 0 1 7603.750 3060.000 7200 3105 8010 3060 7200 3015 - 2 1 1.00 60.00 120.00 5 1 0 2 0 7 50 -1 -1 0.000 0 0 1 0 4956.000 3060.000 4860 3825 4185 3060 4860 2295 0 0 1.00 60.00 120.00 5 1 0 2 0 7 50 -1 -1 0.000 0 1 0 1 4944.000 3060.000 5040 3825 5715 3060 5040 2295 @@ -311,56 +299,6 @@ Single 0 0 1.00 60.00 120.00 5 1 0 2 0 7 50 -1 -1 0.000 0 1 0 1 1258.750 3060.000 855 3105 1665 3060 855 3015 2 1 1.00 60.00 120.00 -6 10606 2371 11985 3749 -2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 - 11768 3060 11970 3060 11967 3119 11959 3177 11946 3234 11929 3291 - 11907 3345 11879 3397 11704 3296 11723 3259 11738 3222 11751 3182 - 11760 3142 11765 3101 11768 3060 -2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 - 11704 3296 11879 3397 11848 3447 11812 3494 11772 3537 11729 3577 - 11682 3613 11633 3644 11531 3469 11566 3447 11599 3422 11628 3393 - 11657 3364 11682 3331 11704 3296 -2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 - 11531 3469 11633 3644 11580 3672 11526 3694 11469 3711 11412 3724 - 11354 3732 11295 3734 11295 3532 11336 3530 11377 3525 11417 3516 - 11457 3503 11494 3488 11531 3469 -2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 - 11295 3532 11295 3734 11236 3732 11178 3724 11121 3711 11064 3694 - 11010 3672 10958 3644 11059 3469 11096 3488 11133 3503 11173 3516 - 11213 3525 11254 3530 11295 3532 -2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 - 11059 3469 10958 3644 10908 3613 10861 3577 10818 3537 10778 3494 - 10742 3447 10711 3398 10886 3296 10908 3331 10933 3364 10962 3393 - 10991 3422 11024 3447 11059 3469 -2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 - 10886 3296 10711 3398 10683 3345 10661 3291 10644 3234 10631 3177 - 10623 3119 10621 3060 10823 3060 10825 3101 10830 3142 10839 3182 - 10852 3222 10867 3259 10886 3296 -2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 - 10823 3060 10621 3060 10623 3001 10631 2943 10644 2886 10661 2829 - 10683 2775 10711 2723 10886 2824 10867 2861 10852 2898 10839 2938 - 10830 2978 10825 3019 10823 3060 -2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 - 10886 2824 10711 2723 10742 2673 10778 2626 10818 2583 10861 2543 - 10908 2507 10958 2476 11059 2651 11024 2673 10991 2698 10962 2727 - 10933 2756 10908 2789 10886 2824 -2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 - 11059 2651 10958 2476 11010 2448 11064 2426 11121 2409 11178 2396 - 11236 2388 11295 2386 11295 2588 11254 2590 11213 2595 11173 2604 - 11133 2617 11096 2632 11059 2651 -2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 - 11295 2588 11295 2386 11354 2388 11412 2396 11469 2409 11526 2426 - 11580 2448 11632 2476 11531 2651 11494 2632 11457 2617 11417 2604 - 11377 2595 11336 2590 11295 2588 -2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 - 11531 2651 11632 2476 11682 2507 11729 2543 11772 2583 11812 2626 - 11848 2673 11879 2723 11704 2824 11682 2789 11657 2756 11628 2727 - 11599 2698 11566 2673 11531 2651 -2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 - 11704 2824 11879 2723 11907 2775 11929 2829 11946 2886 11959 2943 - 11967 3001 11969 3060 11767 3060 11765 3019 11760 2978 11751 2938 - 11738 2898 11723 2861 11704 2824 --6 6 4261 2371 5640 3749 2 3 0 2 0 13 50 -1 20 0.000 0 0 -1 0 0 15 5423 3060 5625 3060 5622 3119 5614 3177 5601 3234 5584 3291 @@ -415,56 +353,6 @@ Single 1 1 0 3 8 11 52 -1 20 0.000 1 0.0000 3105 5049 810 171 3105 5049 3915 5049 4 1 0 50 -1 6 10 0.0000 4 150 1125 3105 5130 Most Urgent\001 -6 -6 6916 2371 8295 3749 -2 3 0 2 0 31 50 -1 43 0.000 0 0 -1 0 0 15 - 8078 3060 8280 3060 8277 3119 8269 3177 8256 3234 8239 3291 - 8217 3345 8189 3397 8014 3296 8033 3259 8048 3222 8061 3182 - 8070 3142 8075 3101 8078 3060 -2 3 0 2 0 31 50 -1 43 0.000 0 0 -1 0 0 15 - 8014 3296 8189 3397 8158 3447 8122 3494 8082 3537 8039 3577 - 7992 3613 7943 3644 7841 3469 7876 3447 7909 3422 7938 3393 - 7967 3364 7992 3331 8014 3296 -2 3 0 2 0 31 50 -1 43 0.000 0 0 -1 0 0 15 - 7841 3469 7943 3644 7890 3672 7836 3694 7779 3711 7722 3724 - 7664 3732 7605 3734 7605 3532 7646 3530 7687 3525 7727 3516 - 7767 3503 7804 3488 7841 3469 -2 3 0 2 0 31 50 -1 43 0.000 0 0 -1 0 0 15 - 7605 3532 7605 3734 7546 3732 7488 3724 7431 3711 7374 3694 - 7320 3672 7268 3644 7369 3469 7406 3488 7443 3503 7483 3516 - 7523 3525 7564 3530 7605 3532 -2 3 0 2 0 31 50 -1 43 0.000 0 0 -1 0 0 15 - 7369 3469 7268 3644 7218 3613 7171 3577 7128 3537 7088 3494 - 7052 3447 7021 3398 7196 3296 7218 3331 7243 3364 7272 3393 - 7301 3422 7334 3447 7369 3469 -2 3 0 2 0 31 50 -1 43 0.000 0 0 -1 0 0 15 - 7196 3296 7021 3398 6993 3345 6971 3291 6954 3234 6941 3177 - 6933 3119 6931 3060 7133 3060 7135 3101 7140 3142 7149 3182 - 7162 3222 7177 3259 7196 3296 -2 3 0 2 0 31 50 -1 43 0.000 0 0 -1 0 0 15 - 7133 3060 6931 3060 6933 3001 6941 2943 6954 2886 6971 2829 - 6993 2775 7021 2723 7196 2824 7177 2861 7162 2898 7149 2938 - 7140 2978 7135 3019 7133 3060 -2 3 0 2 0 31 50 -1 43 0.000 0 0 -1 0 0 15 - 7196 2824 7021 2723 7052 2673 7088 2626 7128 2583 7171 2543 - 7218 2507 7268 2476 7369 2651 7334 2673 7301 2698 7272 2727 - 7243 2756 7218 2789 7196 2824 -2 3 0 2 0 31 50 -1 43 0.000 0 0 -1 0 0 15 - 7369 2651 7268 2476 7320 2448 7374 2426 7431 2409 7488 2396 - 7546 2388 7605 2386 7605 2588 7564 2590 7523 2595 7483 2604 - 7443 2617 7406 2632 7369 2651 -2 3 0 2 0 31 50 -1 43 0.000 0 0 -1 0 0 15 - 7605 2588 7605 2386 7664 2388 7722 2396 7779 2409 7836 2426 - 7890 2448 7942 2476 7841 2651 7804 2632 7767 2617 7727 2604 - 7687 2595 7646 2590 7605 2588 -2 3 0 2 0 31 50 -1 43 0.000 0 0 -1 0 0 15 - 7841 2651 7942 2476 7992 2507 8039 2543 8082 2583 8122 2626 - 8158 2673 8189 2723 8014 2824 7992 2789 7967 2756 7938 2727 - 7909 2698 7876 2673 7841 2651 -2 3 0 2 0 31 50 -1 43 0.000 0 0 -1 0 0 15 - 8014 2824 8189 2723 8217 2775 8239 2829 8256 2886 8269 2943 - 8277 3001 8279 3060 8077 3060 8075 3019 8070 2978 8061 2938 - 8048 2898 8033 2861 8014 2824 --6 6 571 2371 1950 3749 2 3 0 2 0 13 50 -1 43 0.000 0 0 -1 0 0 15 1733 3060 1935 3060 1932 3119 1924 3177 1911 3234 1894 3291 @@ -515,80 +403,139 @@ Single 1932 3001 1934 3060 1732 3060 1730 3019 1725 2978 1716 2938 1703 2898 1688 2861 1669 2824 -6 -6 1800 1845 2520 2385 -4 1 0 50 -1 6 10 0.0000 4 120 570 2160 1980 Global\001 -4 1 0 50 -1 6 10 0.0000 4 120 495 2160 2160 tasks\001 -4 1 0 50 -1 6 10 0.0000 4 150 705 2160 2340 (locked)\001 --6 6 3960 1935 4500 2250 4 1 0 50 -1 6 10 0.0000 4 120 495 4230 2250 tasks\001 4 1 0 50 -1 6 10 0.0000 4 120 465 4230 2070 Local\001 -6 -6 8190 1845 8910 2385 -4 1 0 50 -1 6 10 0.0000 4 150 705 8550 2340 (locked)\001 -4 1 0 50 -1 6 10 0.0000 4 120 585 8550 2160 timers\001 -4 1 0 50 -1 6 10 0.0000 4 120 570 8550 1980 Global\001 --6 -6 10215 1935 10845 2250 -4 1 0 50 -1 6 10 0.0000 4 120 585 10530 2250 timers\001 -4 1 0 50 -1 6 10 0.0000 4 120 465 10530 2070 Local\001 --6 6 2430 945 3735 1530 1 1 0 3 20 29 52 -1 20 0.000 1 0.0000 3083 1180 607 170 3083 1180 3690 1350 4 1 0 50 -1 6 10 0.0000 4 120 615 3105 1260 Local ?\001 4 0 0 50 -1 6 9 0.0000 4 105 315 3375 1530 Yes\001 4 2 0 50 -1 6 9 0.0000 4 105 225 2790 1530 No\001 -6 -6 8775 945 10080 1530 -1 1 0 3 20 29 52 -1 20 0.000 1 0.0000 9428 1180 607 170 9428 1180 10035 1350 -4 1 0 50 -1 6 10 0.0000 4 120 615 9450 1260 Local ?\001 -4 0 0 50 -1 6 9 0.0000 4 105 315 9720 1530 Yes\001 -4 2 0 50 -1 6 9 0.0000 4 105 225 9135 1530 No\001 +6 8370 2070 10440 4005 +5 1 0 2 0 7 50 -1 -1 0.000 0 0 1 0 9456.000 3195.000 9360 3960 8685 3195 9360 2430 + 0 0 1.00 60.00 120.00 +5 1 0 2 0 7 50 -1 -1 0.000 0 1 0 1 9444.000 3195.000 9540 3960 10215 3195 9540 2430 + 0 0 1.00 60.00 120.00 +5 1 0 2 0 7 50 -1 -1 0.000 0 1 0 1 9448.750 3195.000 9045 3240 9855 3195 9045 3150 + 2 1 1.00 60.00 120.00 +6 8761 2506 10140 3884 +2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 + 9923 3195 10125 3195 10122 3254 10114 3312 10101 3369 10084 3426 + 10062 3480 10034 3532 9859 3431 9878 3394 9893 3357 9906 3317 + 9915 3277 9920 3236 9923 3195 +2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 + 9859 3431 10034 3532 10003 3582 9967 3629 9927 3672 9884 3712 + 9837 3748 9788 3779 9686 3604 9721 3582 9754 3557 9783 3528 + 9812 3499 9837 3466 9859 3431 +2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 + 9686 3604 9788 3779 9735 3807 9681 3829 9624 3846 9567 3859 + 9509 3867 9450 3869 9450 3667 9491 3665 9532 3660 9572 3651 + 9612 3638 9649 3623 9686 3604 +2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 + 9450 3667 9450 3869 9391 3867 9333 3859 9276 3846 9219 3829 + 9165 3807 9113 3779 9214 3604 9251 3623 9288 3638 9328 3651 + 9368 3660 9409 3665 9450 3667 +2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 + 9214 3604 9113 3779 9063 3748 9016 3712 8973 3672 8933 3629 + 8897 3582 8866 3533 9041 3431 9063 3466 9088 3499 9117 3528 + 9146 3557 9179 3582 9214 3604 +2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 + 9041 3431 8866 3533 8838 3480 8816 3426 8799 3369 8786 3312 + 8778 3254 8776 3195 8978 3195 8980 3236 8985 3277 8994 3317 + 9007 3357 9022 3394 9041 3431 +2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 + 8978 3195 8776 3195 8778 3136 8786 3078 8799 3021 8816 2964 + 8838 2910 8866 2858 9041 2959 9022 2996 9007 3033 8994 3073 + 8985 3113 8980 3154 8978 3195 +2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 + 9041 2959 8866 2858 8897 2808 8933 2761 8973 2718 9016 2678 + 9063 2642 9113 2611 9214 2786 9179 2808 9146 2833 9117 2862 + 9088 2891 9063 2924 9041 2959 +2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 + 9214 2786 9113 2611 9165 2583 9219 2561 9276 2544 9333 2531 + 9391 2523 9450 2521 9450 2723 9409 2725 9368 2730 9328 2739 + 9288 2752 9251 2767 9214 2786 +2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 + 9450 2723 9450 2521 9509 2523 9567 2531 9624 2544 9681 2561 + 9735 2583 9787 2611 9686 2786 9649 2767 9612 2752 9572 2739 + 9532 2730 9491 2725 9450 2723 +2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 + 9686 2786 9787 2611 9837 2642 9884 2678 9927 2718 9967 2761 + 10003 2808 10034 2858 9859 2959 9837 2924 9812 2891 9783 2862 + 9754 2833 9721 2808 9686 2786 +2 3 0 2 0 31 50 -1 20 0.000 0 0 -1 0 0 15 + 9859 2959 10034 2858 10062 2910 10084 2964 10101 3021 10114 3078 + 10122 3136 10124 3195 9922 3195 9920 3154 9915 3113 9906 3073 + 9893 3033 9878 2996 9859 2959 +-6 +6 8370 2070 9000 2385 +4 1 0 50 -1 6 10 0.0000 4 120 585 8685 2385 timers\001 +4 1 0 50 -1 6 10 0.0000 4 120 465 8685 2205 Local\001 +-6 +4 1 0 50 -1 4 9 1.5708 4 150 315 10395 3195 past\001 +4 1 0 50 -1 4 9 1.5708 4 120 420 8595 3195 future\001 -6 -6 7200 6345 9810 6885 +6 6840 3915 8460 4590 +4 0 0 50 -1 6 12 0.0000 4 165 1320 6840 4095 Time-based\001 +4 0 0 50 -1 6 12 0.0000 4 195 1605 6840 4545 - 1 per thread\001 +4 0 0 50 -1 6 12 0.0000 4 195 1395 6840 4320 Wait queues\001 +-6 +6 6885 7200 9495 7740 2 1 0 4 279 -1 48 -1 -1 0.000 0 0 -1 0 0 4 - 7234 6398 9776 6398 9776 6838 7234 6838 + 6919 7253 9461 7253 9461 7693 6919 7693 2 3 0 0 -1 278 49 -1 20 0.000 0 0 -1 0 0 5 - 7234 6838 9776 6838 9776 6398 7234 6398 7234 6838 + 6919 7693 9461 7693 9461 7253 6919 7253 6919 7693 2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 9613 6398 9613 6838 + 9298 7253 9298 7693 2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 9438 6398 9438 6838 + 9123 7253 9123 7693 2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 9264 6398 9264 6838 -4 1 0 46 -1 4 16 0.0000 4 210 1620 8460 6705 TL_URGENT\001 + 8949 7253 8949 7693 +4 1 0 46 -1 4 16 0.0000 4 240 1425 8145 7560 TL_URGENT\001 +-6 +6 3825 8685 4230 9900 +1 1 0 3 20 29 52 -1 20 0.000 1 1.5708 4015 9292 607 170 4015 9292 4185 8685 +4 1 0 50 -1 6 10 1.5708 4 120 585 4095 9270 Class?\001 -6 -6 4140 7830 4545 9045 -1 1 0 3 20 29 52 -1 20 0.000 1 1.5708 4330 8437 607 170 4330 8437 4500 7830 -4 1 0 50 -1 6 10 1.5708 4 120 585 4410 8415 Class?\001 +6 6885 6435 9495 6975 +2 1 0 4 279 -1 48 -1 -1 0.000 0 0 -1 0 0 4 + 6919 6488 9461 6488 9461 6928 6919 6928 +2 3 0 0 -1 278 49 -1 20 0.000 0 0 -1 0 0 5 + 6919 6928 9461 6928 9461 6488 6919 6488 6919 6928 +2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 + 9298 6488 9298 6928 +2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 + 9123 6488 9123 6928 +2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 + 8949 6488 8949 6928 +4 1 0 46 -1 4 16 0.0000 4 240 660 8145 6795 TL_RT\001 +-6 +6 3420 6975 5760 7740 +2 1 0 4 45 -1 48 -1 -1 0.000 0 0 -1 0 0 4 + 3671 7253 5599 7253 5599 7693 3671 7693 +2 1 0 2 45 -1 47 -1 -1 0.000 0 0 -1 0 0 2 + 5428 7253 5428 7693 +2 1 0 2 45 -1 47 -1 -1 0.000 0 0 -1 0 0 2 + 5253 7253 5253 7693 +2 1 0 2 45 -1 47 -1 -1 0.000 0 0 -1 0 0 2 + 5079 7253 5079 7693 +2 3 0 0 277 276 49 -1 43 0.000 0 0 -1 0 0 5 + 3671 7693 5599 7693 5599 7253 3671 7253 3671 7693 +4 1 0 46 -1 4 16 0.0000 4 210 1020 4455 7560 SHARED\001 +4 1 0 50 -1 4 10 0.0000 4 165 2280 4590 7110 (accessed using atomic ops)\001 -6 1 1 0 3 8 11 52 -1 20 0.000 1 0.0000 9450 5049 540 171 9450 5049 9990 5049 -1 1 0 3 20 29 52 -1 20 0.000 1 1.5708 2440 7672 607 170 2440 7672 2610 7065 -1 1 0 3 8 11 52 -1 20 0.000 1 1.5708 10755 7695 810 171 10755 7695 10755 6885 -2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 4 - 2 1 1.00 90.00 180.00 - 7605 3870 7605 4185 9270 4545 9270 4905 -2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 4 - 2 1 1.00 90.00 180.00 - 11301 3870 11301 4185 9636 4545 9636 4905 -2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 4 - 2 1 1.00 90.00 180.00 - 9630 1395 9626 1591 11291 1800 11295 2295 -2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 4 - 2 1 1.00 90.00 180.00 - 9270 1395 9270 1575 7605 1800 7605 2295 -2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 2 - 2 1 1.00 90.00 180.00 - 9450 360 9450 1035 +1 1 0 3 20 29 52 -1 20 0.000 1 0.0000 3083 415 607 170 3083 415 3690 585 +1 1 0 3 20 29 52 -1 20 0.000 1 1.5708 2125 8527 607 170 2125 8527 2295 7920 +1 1 0 3 8 11 52 -1 20 0.000 1 1.5708 10441 8209 810 171 10441 8209 10441 7399 2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 4 2 1 1.00 90.00 180.00 1260 3870 1260 4185 2925 4545 2925 4905 2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 4 2 1 1.00 90.00 180.00 4956 3870 4956 4185 3291 4545 3291 4905 -2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 2 - 2 1 1.00 90.00 180.00 - 3105 360 3105 1035 2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 4 2 1 1.00 90.00 180.00 3285 1395 3285 1575 4950 1845 4950 2385 @@ -596,153 +543,164 @@ Single 9180 5535 9000 5805 2 1 0 5 13 7 54 -1 -1 0.000 1 0 -1 1 0 4 2 1 1.00 120.00 240.00 - 3105 5220 3105 5850 3105 7200 7200 7200 + 3105 5220 3105 5850 3105 8055 6885 8055 2 1 0 5 22 7 54 -1 -1 0.000 1 0 -1 1 0 5 2 1 1.00 120.00 240.00 - 9450 5220 9450 5670 6300 5670 6300 1215 3690 1170 + 9450 5220 9450 5670 6300 5670 6300 405 3690 405 2 1 0 3 13 7 54 -1 -1 0.000 1 0 -1 0 0 2 3195 5535 3015 5805 2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 4 2 1 1.00 90.00 180.00 2925 1395 2925 1575 1260 1845 1260 2385 -2 2 0 3 35 34 100 -1 20 0.000 1 0 -1 0 0 5 - 6570 720 12330 720 12330 5400 6570 5400 6570 720 +2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 2 + 2 1 1.00 90.00 180.00 + 9456 4005 9450 4860 +2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 2 + 2 1 1.00 90.00 180.00 + 9450 -405 9450 2430 +2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 2 + 2 1 1.00 90.00 180.00 + 3105 630 3105 1035 +2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 2 + 2 1 1.00 90.00 180.00 + 3105 -405 3105 270 2 2 0 3 33 32 100 -1 20 0.000 1 0 -1 0 0 5 - 270 720 6030 720 6030 5400 270 5400 270 720 + 0 90 6030 90 6030 5400 0 5400 0 90 2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 2 2 1 1.00 90.00 180.00 - 315 7650 2250 7650 -2 1 0 5 4 7 54 -1 -1 0.000 1 0 -1 1 0 2 - 2 1 1.00 120.00 240.00 - 10890 7695 12285 7695 + 0 8505 1935 8505 2 1 0 3 0 7 54 -1 -1 0.000 1 0 -1 1 0 3 2 1 1.00 90.00 180.00 - 4455 8775 4725 8910 7200 8910 + 4140 9630 4410 9765 6885 9765 2 1 0 4 279 -1 48 -1 -1 0.000 0 0 -1 0 0 4 - 7234 7118 9776 7118 9776 7558 7234 7558 + 6919 7973 9461 7973 9461 8413 6919 8413 2 3 0 0 -1 278 49 -1 20 0.000 0 0 -1 0 0 5 - 7234 7558 9776 7558 9776 7118 7234 7118 7234 7558 + 6919 8413 9461 8413 9461 7973 6919 7973 6919 8413 2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 9613 7118 9613 7558 + 9298 7973 9298 8413 2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 9438 7118 9438 7558 + 9123 7973 9123 8413 2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 9264 7118 9264 7558 + 8949 7973 8949 8413 2 3 0 0 -1 278 49 -1 20 0.000 0 0 -1 0 0 5 - 7234 8278 9776 8278 9776 7838 7234 7838 7234 8278 + 6919 9133 9461 9133 9461 8693 6919 8693 6919 9133 2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 9613 7838 9613 8278 + 9298 8693 9298 9133 2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 9438 7838 9438 8278 + 9123 8693 9123 9133 2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 9264 7838 9264 8278 + 8949 8693 8949 9133 2 1 0 4 279 -1 48 -1 -1 0.000 0 0 -1 0 0 4 - 7234 8558 9776 8558 9776 8998 7234 8998 + 6919 9413 9461 9413 9461 9853 6919 9853 2 3 0 0 -1 278 49 -1 20 0.000 0 0 -1 0 0 5 - 7234 8998 9776 8998 9776 8558 7234 8558 7234 8998 + 6919 9853 9461 9853 9461 9413 6919 9413 6919 9853 2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 9613 8558 9613 8998 + 9298 9413 9298 9853 2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 9438 8558 9438 8998 + 9123 9413 9123 9853 2 1 0 2 279 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 9264 8558 9264 8998 -2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 2 - 2 1 1.00 90.00 180.00 - 6075 6480 7200 6480 + 8949 9413 8949 9853 2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 3 2 1 1.00 90.00 180.00 - 2610 7830 3195 8415 4140 8415 -2 1 0 4 45 -1 48 -1 -1 0.000 0 0 -1 0 0 4 - 4166 6398 6094 6398 6094 6838 4166 6838 -2 1 0 2 45 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 5923 6398 5923 6838 -2 1 0 2 45 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 5748 6398 5748 6838 -2 1 0 2 45 -1 47 -1 -1 0.000 0 0 -1 0 0 2 - 5574 6398 5574 6838 -2 1 0 3 0 7 54 -1 -1 0.000 1 0 -1 1 0 4 - 2 1 1.00 90.00 180.00 - 2610 7515 2925 6660 3645 6660 4140 6660 -2 3 0 0 277 276 49 -1 43 0.000 0 0 -1 0 0 5 - 4166 6838 6094 6838 6094 6398 4166 6398 4166 6838 -2 1 0 3 0 7 54 -1 -1 0.000 1 0 -1 1 0 3 - 2 1 1.00 90.00 180.00 - 9765 8775 10350 8775 10665 8280 + 2295 8685 2880 9270 3825 9270 2 1 0 3 0 7 54 -1 -1 0.000 1 0 -1 1 0 3 2 1 1.00 90.00 180.00 - 9765 8055 10305 8055 10620 7875 + 9450 9630 10035 9630 10350 8820 2 1 0 3 0 7 54 -1 -1 0.000 1 0 -1 1 0 3 2 1 1.00 90.00 180.00 - 9806 6605 10350 6615 10665 7155 + 9450 8910 10035 8910 10305 8550 2 1 0 3 0 7 54 -1 -1 0.000 1 0 -1 1 0 3 2 1 1.00 90.00 180.00 - 9720 7335 10350 7335 10620 7560 + 9405 8190 9945 8190 10305 8190 2 1 1 5 4 7 57 -1 -1 12.000 1 0 -1 1 0 2 2 1 1.00 120.00 240.00 - 9900 6165 9900 9450 + 9585 6390 9585 10305 2 1 0 2 0 7 54 -1 -1 0.000 1 0 -1 0 0 2 - 10080 7245 9990 7425 + 9765 8100 9675 8280 2 1 0 2 0 7 54 -1 -1 0.000 1 0 -1 0 0 2 - 10080 7965 9990 8145 + 9765 8820 9675 9000 2 1 0 2 0 7 54 -1 -1 0.000 1 0 -1 0 0 2 - 10080 8685 9990 8865 + 9765 9540 9675 9720 2 1 0 3 0 7 54 -1 -1 0.000 1 0 -1 1 0 4 2 1 1.00 90.00 180.00 - 4500 8550 6255 8550 6705 8190 7200 8190 + 4185 9495 6120 9495 6480 9045 6885 9045 2 1 0 3 0 7 54 -1 -1 0.000 1 0 -1 1 0 5 2 1 1.00 90.00 180.00 - 4500 8280 4725 8100 6435 8100 6750 7470 7200 7470 + 4185 9225 4455 9135 6165 9135 6435 8325 6885 8325 +2 1 0 4 279 -1 48 -1 -1 0.000 0 0 -1 0 0 4 + 6919 8693 9461 8693 9461 9133 6919 9133 +2 1 0 2 0 7 54 -1 -1 0.000 1 0 -1 0 0 2 + 9765 7380 9675 7560 +2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 2 + 2 1 1.00 90.00 180.00 + 5580 7335 6885 7335 +2 1 0 3 0 7 54 -1 -1 0.000 1 0 -1 1 0 3 + 2 1 1.00 90.00 180.00 + 9491 7460 10080 7470 10305 7830 2 1 0 3 0 7 54 -1 -1 0.000 1 0 -1 1 0 5 2 1 1.00 90.00 180.00 - 4455 8055 4635 7740 6390 7740 6750 6750 7200 6750 -2 1 0 4 279 -1 48 -1 -1 0.000 0 0 -1 0 0 4 - 7234 7838 9776 7838 9776 8278 7234 8278 + 4140 9045 4410 8820 6075 8820 6435 7605 6885 7605 +2 1 0 3 0 7 54 -1 -1 0.000 1 0 -1 1 0 3 + 2 1 1.00 90.00 180.00 + 9491 6695 10125 6705 10350 7515 2 1 0 2 0 7 54 -1 -1 0.000 1 0 -1 0 0 2 - 10080 6525 9990 6705 + 9765 6615 9675 6795 +2 1 0 3 0 7 54 -1 -1 0.000 1 0 -1 1 0 5 + 2 1 1.00 90.00 180.00 + 4140 8820 4410 8460 5895 8460 6435 6705 6885 6705 2 2 0 3 43 47 100 -1 20 0.000 1 0 -1 0 0 5 - 1935 5985 11070 5985 11070 9585 1935 9585 1935 5985 -4 1 0 50 -1 4 9 1.5708 4 135 315 12240 3060 past\001 -4 1 0 50 -1 4 9 1.5708 4 120 465 10440 3060 future\001 -4 1 0 50 -1 4 9 1.5708 4 135 315 8550 3060 past\001 -4 1 0 50 -1 4 9 1.5708 4 120 465 6750 3060 future\001 + 1620 6120 10755 6120 10755 10485 1620 10485 1620 6120 +2 1 0 3 0 7 54 -1 -1 0.000 1 0 -1 1 0 4 + 2 1 1.00 90.00 180.00 + 2295 8370 2610 7515 3330 7515 3645 7515 +2 1 0 3 0 7 50 -1 -1 0.000 1 0 -1 1 0 4 + 2 1 1.00 90.00 180.00 + 2790 585 135 1350 135 6795 1980 8235 +2 2 0 3 35 34 100 -1 20 0.000 1 0 -1 0 0 5 + 6570 90 11700 90 11700 5400 6570 5400 6570 90 +2 1 0 5 4 7 54 -1 -1 0.000 1 0 -1 1 0 2 + 2 1 1.00 120.00 240.00 + 10575 8235 11970 8235 4 1 0 50 -1 6 10 0.0000 4 120 600 9450 5130 Oldest\001 -4 1 0 50 -1 4 9 1.5708 4 105 540 405 3060 newest\001 +4 1 0 50 -1 4 9 1.5708 4 120 525 405 3060 newest\001 4 1 0 50 -1 4 9 1.5708 4 120 450 2205 3060 oldest\001 -4 1 0 50 -1 4 9 1.5708 4 105 540 4095 3060 newest\001 +4 1 0 50 -1 4 9 1.5708 4 120 525 4095 3060 newest\001 4 1 0 50 -1 4 9 1.5708 4 120 450 5895 3060 oldest\001 -4 0 0 50 -1 14 10 0.0000 4 135 1470 9135 5850 runqueue-depth\001 -4 0 0 50 -1 14 10 0.0000 4 135 1470 3195 5715 runqueue-depth\001 -4 1 0 50 -1 6 12 0.0000 4 165 1320 9450 3600 Time-based\001 -4 1 0 50 -1 6 12 0.0000 4 195 1395 9450 3780 Wait queues\001 -4 0 0 50 -1 6 12 0.0000 4 195 1050 9000 4005 - 1 global\001 -4 0 0 50 -1 6 12 0.0000 4 195 1605 9000 4185 - 1 per thread\001 +4 0 0 50 -1 14 10 0.0000 4 135 1680 9135 5850 runqueue-depth\001 +4 0 0 50 -1 14 10 0.0000 4 135 1680 3195 5715 runqueue-depth\001 4 1 0 50 -1 6 12 0.0000 4 195 1650 3105 3600 Priority-based\001 4 1 0 50 -1 6 12 0.0000 4 180 1365 3105 3780 Run queues\001 -4 0 0 50 -1 6 12 0.0000 4 195 1050 2655 4005 - 1 global\001 4 0 0 50 -1 6 12 0.0000 4 195 1605 2655 4185 - 1 per thread\001 -4 0 0 50 -1 14 10 0.0000 4 135 1365 3240 585 task_wakeup()\001 -4 0 0 50 -1 14 10 0.0000 4 135 1575 9585 630 task_schedule()\001 -4 0 0 50 -1 14 10 0.0000 4 135 1260 9585 450 task_queue()\001 -4 0 0 50 -1 14 10 0.0000 4 135 1680 315 7560 tasklet_wakeup()\001 -4 2 0 50 -1 14 10 0.0000 4 135 1260 12285 7515 t->process()\001 -4 2 4 50 -1 6 12 0.0000 4 150 525 12285 7335 Run!\001 -4 1 0 46 -1 4 16 0.0000 4 210 1695 8460 7425 TL_NORMAL\001 -4 1 0 46 -1 4 16 0.0000 4 210 1200 8460 8145 TL_BULK\001 -4 1 0 46 -1 4 16 0.0000 4 210 1425 8460 8865 TL_HEAVY\001 -4 1 0 46 -1 4 16 0.0000 4 195 1095 4950 6705 SHARED\001 -4 0 0 50 -1 6 9 0.0000 4 105 345 10035 7515 37%\001 -4 0 0 50 -1 6 9 0.0000 4 105 210 10080 8955 =1\001 -4 1 0 50 -1 4 10 0.0000 4 150 2280 5085 6255 (accessed using atomic ops)\001 -4 0 0 50 -1 6 9 0.0000 4 105 345 10035 6795 50%\001 -4 0 0 50 -1 6 9 0.0000 4 105 345 10035 8235 13%\001 -4 2 0 50 -1 6 9 1.5708 4 105 315 2745 8100 Yes\001 -4 1 0 50 -1 6 10 1.5708 4 120 615 2520 7650 Local ?\001 -4 0 0 50 -1 6 9 1.5708 4 105 225 2700 7110 No\001 -4 0 0 50 -1 14 10 0.0000 4 135 1680 4725 8460 TASK_SELF_WAKING\001 -4 0 0 50 -1 14 10 0.0000 4 135 1050 4725 8820 TASK_HEAVY\001 -4 0 0 50 -1 4 10 0.0000 4 165 675 4725 8010 (default)\001 -4 0 0 50 -1 4 10 0.0000 4 150 1290 4725 7650 In I/O or signals\001 -4 1 0 50 -1 6 10 1.5708 4 150 1125 10815 7695 Most Urgent\001 -4 0 4 50 -1 6 10 0.0000 4 120 480 9990 6480 order\001 -4 0 4 50 -1 6 10 0.0000 4 120 420 9990 6300 Scan\001 -4 1 0 50 -1 6 12 0.0000 4 195 9075 6030 9450 5 class-based tasklet queues per thread (one accessible from remote threads)\001 +4 1 0 50 -1 6 10 0.0000 4 135 945 3105 495 TASK_RT ?\001 +4 0 0 50 -1 6 9 0.0000 4 105 225 3195 765 No\001 +4 2 0 50 -1 6 9 0.0000 4 105 315 2430 585 Yes\001 +4 0 0 50 -1 14 10 0.0000 4 135 1920 0 8415 tasklet_wakeup()\001 +4 1 0 46 -1 4 16 0.0000 4 240 1545 8145 8280 TL_NORMAL\001 +4 1 0 46 -1 4 16 0.0000 4 240 990 8145 9000 TL_BULK\001 +4 1 0 46 -1 4 16 0.0000 4 240 1260 8145 9720 TL_HEAVY\001 +4 0 0 50 -1 6 9 0.0000 4 105 345 9720 8370 37%\001 +4 0 0 50 -1 6 9 0.0000 4 105 210 9765 9810 =1\001 +4 0 0 50 -1 6 9 0.0000 4 105 345 9720 7650 50%\001 +4 0 0 50 -1 6 9 0.0000 4 105 345 9720 9090 13%\001 +4 2 0 50 -1 6 9 1.5708 4 105 315 2430 8955 Yes\001 +4 1 0 50 -1 6 10 1.5708 4 120 615 2205 8505 Local ?\001 +4 0 0 50 -1 6 9 1.5708 4 105 225 2385 7965 No\001 +4 0 0 50 -1 14 10 0.0000 4 135 1200 4410 9675 TASK_HEAVY\001 +4 1 0 50 -1 6 12 0.0000 4 195 9075 5715 10305 5 class-based tasklet queues per thread (one accessible from remote threads)\001 +4 0 4 50 -1 6 10 0.0000 4 120 420 9675 6345 Scan\001 +4 0 4 50 -1 6 10 0.0000 4 120 480 9675 6525 order\001 +4 0 0 50 -1 6 9 0.0000 4 105 210 9765 6885 =1\001 +4 0 0 50 -1 14 10 0.0000 4 135 1920 4275 9405 TASK_SELF_WAKING\001 +4 0 0 50 -1 4 10 0.0000 4 165 1185 4455 8730 In I/O or signals\001 +4 0 0 50 -1 14 10 0.0000 4 135 840 4455 8370 TASK_RT\001 +4 0 0 50 -1 4 10 0.0000 4 150 705 4455 9045 (default)\001 +4 0 0 50 -1 14 10 0.0000 4 135 1800 9585 -45 task_schedule()\001 +4 0 0 50 -1 14 10 0.0000 4 135 1560 3240 -45 task_wakeup()\001 +4 0 0 50 -1 14 10 0.0000 4 135 1440 9585 -225 task_queue()\001 +4 2 0 50 -1 14 10 0.0000 4 135 1440 11970 8055 t->process()\001 +4 2 4 50 -1 6 12 0.0000 4 150 525 11970 7875 Run!\001 +4 1 0 50 -1 6 10 1.5708 4 150 1125 10500 8235 Most Urgent\001 +4 1 0 50 -1 6 10 0.0000 4 120 495 2160 2160 tasks\001 +4 1 0 50 -1 6 10 0.0000 4 150 705 2160 2340 (locked)\001 +4 1 0 50 -1 6 10 0.0000 4 120 630 2160 1980 Shared\001 +4 0 0 50 -1 6 12 0.0000 4 180 1335 2655 4005 - 1 per tgrp\001 diff --git a/doc/internals/sched.pdf b/doc/internals/sched.pdf index d1ce3ded7..3ede60f78 100644 Binary files a/doc/internals/sched.pdf and b/doc/internals/sched.pdf differ diff --git a/doc/internals/sched.png b/doc/internals/sched.png index 65c97a1e3..f17edb708 100644 Binary files a/doc/internals/sched.png and b/doc/internals/sched.png differ diff --git a/doc/internals/sched.svg b/doc/internals/sched.svg index 0fa329a71..0e4adbe06 100644 --- a/doc/internals/sched.svg +++ b/doc/internals/sched.svg @@ -1,32 +1,32 @@ - - + + + width="1140pt" height="1036pt" + viewBox="-33 -438 12057 10956"> - - - - + + - + - - - + - + - - - + - + - - - + - + - - - + + + - + - - - + - + - - - + - + - - - + + + + + + + - + - - - + - + - - - + + + - + - - - + - - - - - - - + - - - + - + - - - + + + - + - - - + - + + + + + + + + + + + + + + + + + @@ -197,81 +224,21 @@ - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - @@ -323,8 +290,8 @@ Most Urgent - + @@ -332,13 +299,13 @@ - + - + @@ -346,13 +313,13 @@ - + - + @@ -360,13 +327,13 @@ - + - + @@ -374,13 +341,13 @@ - + - + @@ -388,13 +355,13 @@ - + - + @@ -402,13 +369,13 @@ - + - + @@ -416,13 +383,13 @@ - + - + @@ -430,13 +397,13 @@ - + - + @@ -444,13 +411,13 @@ - + - + @@ -458,13 +425,13 @@ - + - + @@ -472,13 +439,13 @@ - + - + @@ -486,549 +453,361 @@ - + + +tasks + +Local + +Local ? + +Yes + +No - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Global - -tasks - -(locked) +timers -tasks - -Local - -(locked) - -timers - -Global - -timers - -Local - -Local ? - -Yes - -No - -Local ? - -Yes - -No - - -Class? - - - - - - - - - - - - - - - - - - +Local - + - - - + + - + - - - + - + - - - - - - - - - - - - + + + +past + + +future + +Time-based + +- 1 per thread + +Wait queues + + +Class? + +(accessed using atomic ops) - - + + M 9063,3412 9083,3375 9120,3395 9052,3227 9032,3233z"/> - - - + - - - + - - - + - - - + - - - + - + + + + + + + + + - - - - + - - - - + - - - - + - - - - + - - - - + - - - - + - - - - + - - - - + - - - - + - - - - + - - - - - - - - - - - - - - - - - - - - - - + - -past - - -future - - -past - - -future - Oldest @@ -1047,158 +826,180 @@ runqueue-depth -Time-based - -Wait queues - -- 1 global - -- 1 per thread - Priority-based Run queues -- 1 global - - 1 per thread -task_wakeup() - -task_schedule() +TASK_RT ? -task_queue() +No -tasklet_wakeup() +Yes -t->process() +tasklet_wakeup() -Run! +37% -37% +=1 -=1 +50% -(accessed using atomic ops) +13% -50% - -13% - - + Yes - + Local ? - + No -TASK_SELF_WAKING +TASK_HEAVY + +5 class-based tasklet queues per thread (one accessible from remote threads) + +Scan + +order -TASK_HEAVY +=1 -(default) +TASK_SELF_WAKING -In I/O or signals +In I/O or signals - +TASK_RT + +(default) + +task_schedule() + +task_wakeup() + +task_queue() + +t->process() + +Run! + + Most Urgent -order +tasks -Scan +(locked) -5 class-based tasklet queues per thread (one accessible from remote threads) - - - - +Shared + +- 1 per tgrp - + - + - - + - - + + - + - + + + + - - - - + + + + - - - - - - + + + + + + - - - - - - + - + - + + +TL_URGENT -TL_URGENT +TL_RT -TL_NORMAL +SHARED -TL_BULK +TL_NORMAL -TL_HEAVY +TL_BULK -SHARED +TL_HEAVY diff --git a/include/haproxy/tinfo-t.h b/include/haproxy/tinfo-t.h index 1f1d92e8e..e5ca44535 100644 --- a/include/haproxy/tinfo-t.h +++ b/include/haproxy/tinfo-t.h @@ -47,10 +47,11 @@ struct thread_set { /* tasklet classes */ enum { - TL_URGENT = 0, /* urgent tasklets (I/O callbacks) */ - TL_NORMAL = 1, /* normal tasks */ - TL_BULK = 2, /* bulk task/tasklets, streaming I/Os */ - TL_HEAVY = 3, /* heavy computational tasklets (e.g. TLS handshakes) */ + TL_RT = 0, /* rare but time-sensitive wakeups (e.g. stats collection) */ + TL_URGENT = 1, /* urgent tasklets (I/O callbacks) */ + TL_NORMAL = 2, /* normal tasks */ + TL_BULK = 3, /* bulk task/tasklets, streaming I/Os */ + TL_HEAVY = 4, /* heavy computational tasklets (e.g. TLS handshakes) */ TL_CLASSES /* must be last */ }; diff --git a/src/task.c b/src/task.c index 055f26d22..0c7f5b8fd 100644 --- a/src/task.c +++ b/src/task.c @@ -147,6 +147,10 @@ void __tasklet_wakeup_here(struct tasklet *tl) LIST_APPEND(&th_ctx->tasklets[TL_HEAVY], &tl->list); th_ctx->tl_class_mask |= 1 << TL_HEAVY; } + else if (unlikely(tl->state & TASK_RT)) { + LIST_APPEND(&th_ctx->tasklets[TL_RT], &tl->list); + th_ctx->tl_class_mask |= 1 << TL_RT; + } else if (tl->state & TASK_SELF_WAKING) { LIST_APPEND(&th_ctx->tasklets[TL_BULK], &tl->list); th_ctx->tl_class_mask |= 1 << TL_BULK; @@ -501,7 +505,8 @@ unsigned int run_tasks_from_lists(unsigned int budgets[]) * budget to run them. Let's switch to it now. */ queue = (th_ctx->tl_class_mask & 1) ? 0 : - (th_ctx->tl_class_mask & 2) ? 1 : 2; + (th_ctx->tl_class_mask & 2) ? 1 : + (th_ctx->tl_class_mask & 4) ? 2 : 3; continue; } @@ -713,6 +718,7 @@ void process_runnable_tasks() struct eb32_node *grq; // next global run queue entry struct task *t; const unsigned int default_weights[TL_CLASSES] = { + [TL_RT] = 1, // never more than 1 RT task at once [TL_URGENT] = 64, // ~50% of CPU bandwidth for I/O [TL_NORMAL] = 60, // ~47% of CPU bandwidth for tasks [TL_BULK] = 4, // ~3% of CPU bandwidth for self-wakers @@ -724,6 +730,7 @@ void process_runnable_tasks() unsigned int queue; int max_processed; int lpicked, gpicked; + int rt_queued = 0; int heavy_queued = 0; int budget, done; @@ -751,6 +758,16 @@ void process_runnable_tasks() not_done_yet: max[TL_URGENT] = max[TL_NORMAL] = max[TL_BULK] = 0; + /* RT tasklets list may be processed at most once */ + if (!rt_queued) { + if ((tt->tl_class_mask & (1 << TL_RT))) { + max[TL_RT] = default_weights[TL_RT]; + rt_queued = 1; + } + else + max[TL_RT] = 0; + } + /* urgent tasklets list gets a default weight of ~50% */ if ((tt->tl_class_mask & (1 << TL_URGENT)) || !MT_LIST_ISEMPTY(&tt->shared_tasklet_list)) @@ -785,13 +802,17 @@ void process_runnable_tasks() * a first MT_LIST_ISEMPTY() to succeed for thread_has_task() and the * one above to finally fail. This is extremely rare and not a problem. */ - max_total = max[TL_URGENT] + max[TL_NORMAL] + max[TL_BULK] + max[TL_HEAVY]; + max_total = max[TL_RT] + max[TL_URGENT] + max[TL_NORMAL] + max[TL_BULK] + max[TL_HEAVY]; if (!max_total) goto leave; for (queue = 0; queue < TL_CLASSES; queue++) max[queue] = ((unsigned)max_processed * max[queue] + max_total - 1) / max_total; + /* The RT queue must never process more than one task at once */ + if (max[TL_RT] > 1) + max[TL_RT] = 1; + /* The heavy queue must never process more than very few tasks at once * anyway. We set the limit to 1 if running on low_latency scheduling, * given that we know that other values can have an impact on latency @@ -819,7 +840,7 @@ void process_runnable_tasks() /* Note: the grq lock is always held when grq is not null */ lpicked = gpicked = 0; budget = max[TL_NORMAL] - tt->tasks_in_list; - while (lpicked + gpicked < budget) { + while (lpicked + gpicked < budget && (!rt_queued || !(global.tune.options & GTUNE_SCHED_LOW_LATENCY))) { if (!eb_is_empty(&th_ctx->rqueue_shared) && !grq) { #ifdef USE_THREAD HA_SPIN_LOCK(TASK_RQ_LOCK, &th_ctx->rqsh_lock); @@ -899,7 +920,7 @@ void process_runnable_tasks() max_processed -= done; /* some tasks may have woken other ones up */ - if (done && max_processed > 0 && thread_has_tasks()) + if (done && max_processed > 0 && !rt_queued && thread_has_tasks()) goto not_done_yet; leave: