浅析linux组调度配额、带宽和分享多余带宽发红包

1. #ifdef CONFIG_CFS_BANDWIDTH
   
2. /*
   
3.  * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool
   
4.  * each time a cfs_rq requests quota.
   
5.  *
   
6.  * Note: in the case that the slice exceeds the runtime remaining (either due
   
7.  * to consumption or the quota being specified to be smaller than the slice)
   
8.  * we will always only issue the remaining available time.
   
9.  *
   
10.  * default: 5 msec, units: microseconds
    
11.   */
    
12. unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
    
13. #endif
    
14. 
    
15. /*
    
16.  * Replenish runtime according to assigned quota and update expiration time.
    
17.  * We use sched_clock_cpu directly instead of rq->clock to avoid adding
    
18.  * additional synchronization around rq->lock.
    
19.  *
    
20.  * requires cfs_b->lock
    
21.  */
    
22. void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
    
23. {
    
24.     u64 now;
    
25. 
    
26.     if (cfs_b->quota == RUNTIME_INF)
    
27.         return;
    
28. 
    
29.     now = sched_clock_cpu(smp_processor_id());
    
30.     cfs_b->runtime = cfs_b->quota;
    
31.     cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);
    
32.     // 组调度, 配额, quota
    
33.     // 也就是说cfs_b->period这段时间分配给该cfs_b组的runtime配额为cfs_b->quota
    
34.     // 不论该组消耗了多少配额, 都会在这里将配额重新填满, 触发接下来的消耗period[2014-11-26]
    
35.     // start_bandwidth_timer(&cfs_b->period_timer, cfs_b->period);
    
36. }
    
37. 
    
38. 
    
39. static inline u64 sched_cfs_bandwidth_slice(void)
    
40. {
    
41.     return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
    
42. }
    
43. 
    
44. /* returns 0 on failure to allocate runtime */
    
45. // 努力延长本cfs_rq的runtime_remaining时间寿命[gliethttp]
    
46. static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
    
47. {
    
48.     struct task_group *tg = cfs_rq->tg;
    
49.     struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
    
50.     u64 amount = 0, min_amount, expires;
    
51. 
    
52.     /* note: this is a positive sum as runtime_remaining <= 0 */
    
53.     // 5ms + 欠的账(超出配额的部分), 如注释所说cfs_rq->runtime_remaining是一个负数, 欠费了
    
54.     min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining;
    
55. 
    
56.     raw_spin_lock(&cfs_b->lock);
    
57.     if (cfs_b->quota == RUNTIME_INF)
    
58.         amount = min_amount; // 如果是infinite的,那么每次都获取5ms
    
59.     else {
    
60.         /*
    
61.          * If the bandwidth pool has become inactive, then at least one
    
62.          * period must have elapsed since the last consumption.
    
63.          * Refresh the global state and ensure bandwidth timer becomes
    
64.          * active.
    
65.          */
    
66.         if (!cfs_b->timer_active) {
    
67.             // 如果带宽timer刚好stop了, 那么重新填充带宽timer的runtime和runtime_expires
    
68.             // cfs_b->runtime = cfs_b->quota;
    
69.             // cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);
    
70.             // 进入下一轮以cfs_b->period为间隔的, 有效时长(单位ns纳秒)cfs_b->runtime的带宽period[gliethttp]
    
71.             __refill_cfs_bandwidth_runtime(cfs_b);
    
72.             __start_cfs_bandwidth(cfs_b);
    
73.         }
    
74. 
    
75.         if (cfs_b->runtime > 0) {
    
76.             amount = min(cfs_b->runtime, min_amount); // 如果剩余的cfs_b->runtime不足5ms(sysctl_sched_cfs_bandwidth_slice)
    
77.             // 更甚至与不足-cfs_rq->runtime_remaining, 那么amount将等于2者最小者.
    
78.             cfs_b->runtime -= amount; // 被瓜分出去amount
    
79.             cfs_b->idle = 0;
    
80.         }
    
81.     }
    
82.     expires = cfs_b->runtime_expires;
    
83.     raw_spin_unlock(&cfs_b->lock);
    
84. 
    
85.     cfs_rq->runtime_remaining += amount; // 重新为自己充电amount纳秒
    
86.     /*
    
87.      * we may have advanced our local expiration to account for allowed
    
88.      * spread between our sched_clock and the one on which runtime was
    
89.      * issued.
    
90.      */
    
91.     if ((s64)(expires - cfs_rq->runtime_expires) > 0)
    
92.         cfs_rq->runtime_expires = expires;
    
93. 
    
94.     return cfs_rq->runtime_remaining > 0;
    
95. }
    
96. 
    
97. // 发红包啦[20141128]! 每个处于trhottled的cfs_rq获得
    
98. // 我有些多余的remaining纳秒带宽, 将在expires时到期
    
99. // cfs_b你把这些带宽发给你手里饥渴的人吧[20141128]
    
100. static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,
     
101.         u64 remaining, u64 expires)
     
102. {
     
103.     struct cfs_rq *cfs_rq;
     
104.     u64 runtime = remaining;
     
105. 
     
106.     rcu_read_lock();
     
107.     list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
     
108.                 throttled_list) {
     
109.         struct rq *rq = rq_of(cfs_rq);
     
110. 
     
111.         raw_spin_lock(&rq->lock);
     
112.         if (!cfs_rq_throttled(cfs_rq))
     
113.             goto next;
     
114. 
     
115.         runtime = -cfs_rq->runtime_remaining + 1; // 发红包啦[20141128]! 每个处于trhottled获得
     
116.         // -cfs_rq->runtime_remaining, 填平到数值0, 同时再额外附送1ns
     
117.         // 其实对于当前cfs_rq只获得了可用的1ns,
     
118.         // 但是发红包的人,可把这个cfs_rq签得账-cfs_rq->runtime_remaining都给偿还上了
     
119.         // 调用这个函数发红包的人, 真是个runtime富裕的大好人[20141128]
     
120.         // 不要想太美啦, 世上哪有好事,便宜让你占呀,
     
121.         // 其实是do_sched_cfs_period_timer自己欠下的账, 自己在这个时机来还而已[gliethttp]
     
122.         if (runtime > remaining)
     
123.             runtime = remaining;
     
124.         remaining -= runtime; // 红包发出去了runtime, 计算我还剩多少ns红包
     
125. 
     
126.         cfs_rq->runtime_remaining += runtime; // 红包给你这个cfs_rq了[20141128]
     
127.         cfs_rq->runtime_expires = expires; // 你cfs_rq使用我的expires吧, 我和我富裕出来的红包生命将在expires时刻因过期而终结
     
128. 
     
129.         /* we check whether we're throttled above */
     
130.         if (cfs_rq->runtime_remaining > 0) // 把自己throttled吧
     
131.             unthrottle_cfs_rq(cfs_rq);
     
132. 
     
133. next:
     
134.         raw_spin_unlock(&rq->lock);
     
135. 
     
136.         if (!remaining)
     
137.             break;
     
138.     }
     
139.     rcu_read_unlock();
     
140. 
     
141.     return remaining;
     
142. }
     
143. 
     
144. /*
     
145.  * Responsible for refilling a task_group's bandwidth and unthrottling its
     
146.  * cfs_rqs as appropriate. If there has been no activity within the last
     
147.  * period the timer is deactivated until scheduling resumes; cfs_b->idle is
     
148.  * used to track this state.
     
149.  */
     
150. // 新的cfs_b->period_timer带宽周期到达,
     
151. // 带宽数值(单位ns纳秒): cfs_b->runtime (cfs_b->runtime = cfs_b->quota;)
     
152. // 带宽有效期(单位ns纳秒): cfs_b->runtime_expires (cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);)
     
153. static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
     
154. {
     
155.     ......
     
156.     /*
     
157.      * This check is repeated as we are holding onto the new bandwidth
     
158.      * while we unthrottle. This can potentially race with an unthrottled
     
159.      * group trying to acquire new bandwidth from the global pool.
     
160.      */
     
161.     while (throttled && runtime > 0) {
     
162.         raw_spin_unlock(&cfs_b->lock);
     
163.         /* we can't nest cfs_b->lock while distributing bandwidth */
     
164.         runtime = distribute_cfs_runtime(cfs_b, runtime,
     
165.                          runtime_expires); // 把自己欠的账还回去
     
166.         raw_spin_lock(&cfs_b->lock);
     
167. 
     
168.         throttled = !list_empty(&cfs_b->throttled_cfs_rq);
     
169.     }
     
170. 
     
171.     /* return (any) remaining runtime */
     
172.     cfs_b->runtime = runtime;
     
173.     ......
     
174. }