Subject: Re: weird problem

From: Paweł Staszewski <[email-masked]>
Date: Thu, 25 Jun 2009 18:33:21 +0200

oprofile output per cpu
CPU: Core 2, speed 3000.08 MHz (estimated)
Counted CPU_CLK_UNHALTED events (Clock cycles when not halted) with a 
unit mask of 0x00 (Unhalted core cycles) count 100000
Samples on CPU 0
Samples on CPU 1
Samples on CPU 2
Samples on CPU 3
Samples on CPU 4
Samples on CPU 5
Samples on CPU 6
Samples on CPU 7
Samples on CPU all
samples  %        samples  %        samples  %        samples  %        
samples  %        samples  %        samples  %        samples  %        
samples  %
    image name               app name                 symbol name
29520    21.7362  0              0  0              0  0              0  
0              0  0              0  9571     34.0254  0              0  
4701      0.22
55  oprofiled                oprofiled                (no symbols)
18441    13.5785  0              0  0              0  0              0  
0              0  0              0  0              0  0              0  
2666      0.12
79  vmlinux                  vmlinux                  rt_worker_func
18428    13.5690  132089    2.1756  135138    1.4238  21735    42.7703  
775       1.9819  893       5.3180  847       3.0111  1056      3.3130  
37949     1.82
01  vmlinux                  vmlinux                  mwait_idle
15797    11.6317  212       0.0035  2879      0.0303  17        0.0335  
2         0.0051  4         0.0238  14        0.0498  185       0.5804  
2895      0.13
89  vmlinux                  vmlinux                  free_block
9034      6.6519  1218      0.0201  2727      0.0287  0              0  
0              0  0              0  0              0  0              0  
2004      0.09
61  vmlinux                  vmlinux                  dst_destroy
7113      5.2375  632       0.0104  483       0.0051  6615     13.0170  
543       1.3886  461       2.7454  1951      6.9359  1274      3.9970  
4073      0.19
54  vmlinux                  vmlinux                  mutex_spin_on_owner
4382      3.2266  652       0.0107  1880      0.0198  2         0.0039  
0              0  0              0  3         0.0107  7         0.0220  
1050      0.05
04  vmlinux                  vmlinux                  __call_rcu
4337      3.1934  418       0.0069  830       0.0087  58        0.1141  
18        0.0460  8         0.0476  20        0.0711  39        0.1224  
917       0.04
40  vmlinux                  vmlinux                  
__rcu_process_callbacks
3991      2.9387  51       8.4e-04  325       0.0034  136       0.2676  
2         0.0051  1         0.0060  1034      3.6759  205       0.6432  
1110      0.05
32  vmlinux                  vmlinux                  
copy_user_generic_string
3292      2.4240  275838    4.5432  391639    4.1263  869       1.7100  
431       1.1022  350       2.0843  673       2.3925  1069      3.3538  
89939     4.31
37  vmlinux                  vmlinux                  _raw_spin_lock
1909      1.4056  79        0.0013  376       0.0040  0              0  
0              0  0              0  0              0  0              0  
348       0.01
67  vmlinux                  vmlinux                  dst_rcu_free
1461      1.0758  0              0  1163      0.0123  0              0  
0              0  0              0  0              0  0              0  0
 0  vim                      vim                      (no symbols)
1122      0.8262  11514     0.1896  6590      0.0694  1857      3.6542  
7638     19.5325  2138     12.7323  1213      4.3123  4025     12.6278  
9676      0.46
41  libc-2.8.so              libc-2.8.so              (no symbols)
1035      0.7621  9027      0.1487  22216     0.2341  4         0.0079  
0              0  0              0  5         0.0178  15        0.0471  
4310      0.20
67  vmlinux                  vmlinux                  kmem_cache_free
1016      0.7481  218       0.0036  465       0.0049  0              0  
0              0  0              0  0              0  0              0  
255       0.01
22  vmlinux                  vmlinux                  ipv4_dst_destroy
951       0.7002  4290      0.0707  5514      0.0581  3283      6.4603  
867       2.2172  673       4.0079  1096      3.8963  1316      4.1288  
2593      0.12
44  vmlinux                  vmlinux                  tg_shares_up
708       0.5213  104       0.0017  279       0.0029  0              0  
0              0  0              0  0              0  0              0  
159       0.00
76  vmlinux                  vmlinux                  call_rcu_bh
581       0.4278  21812     0.3593  38364     0.4042  68        0.1338  
23        0.0588  23        0.1370  65        0.2311  96        0.3012  
8000      0.38
37  vmlinux                  vmlinux                  _raw_spin_unlock
563       0.4145  1027      0.0169  2528      0.0266  0              0  
0              0  0              0  0              0  3         0.0094  
545       0.02
61  vmlinux                  vmlinux                  local_bh_enable_ip
509       0.3748  0              0  3        3.2e-05  0              0  
0              0  0              0  4         0.0142  3         0.0094  
76        0.00
36  vmlinux                  vmlinux                  can_dirty
430       0.3166  74        0.0012  148       0.0016  25        0.0492  
13        0.0332  12        0.0715  9         0.0320  12        0.0376  
96        0.00
46  vmlinux                  vmlinux                  rcu_process_callbacks
372       0.2739  0              0  0              0  0              0  
0              0  0              0  3         0.0107  2         0.0063  
48        0.00
23  vmlinux                  vmlinux                  do_journal_end
347       0.2555  9991      0.1646  16066     0.1693  143       0.2814  
66        0.1688  25        0.1489  46        0.1635  71        0.2228  
3514      0.16
85  vmlinux                  vmlinux                  rb_get_reader_page
346       0.2548  952       0.0157  566       0.0060  871       1.7140  
321       0.8209  174       1.0362  324       1.1518  269       0.8439  
655       0.03
14  vmlinux                  vmlinux                  find_busiest_group
275       0.2025  1194      0.0197  1228      0.0129  1019      2.0052  
203       0.5191  151       0.8992  290       1.0310  653       2.0487  
799       0.03
83  vmlinux                  vmlinux                  find_next_bit
238       0.1752  3053      0.0503  4711      0.0496  116       0.2283  
82        0.2097  54        0.3216  101       0.3591  101       0.3169  
1174      0.05
63  oprofile                 oprofile                 (no symbols)
221       0.1627  0              0  0              0  0              0  
0              0  0              0  0              0  0              0  
18       8.6e-
04  vmlinux                  vmlinux                  ahci_interrupt
212       0.1561  38335     0.6314  60370     0.6361  329       0.6474  
165       0.4220  136       0.8099  133       0.4728  205       0.6432  
12821     0.61
49  vmlinux                  vmlinux                  apic_timer_interrupt
210       0.1546  117       0.0019  1001      0.0105  2         0.0039  
0              0  0              0  55        0.1955  16        0.0502  
270       0.01
30  vmlinux                  vmlinux                  memmove
207       0.1524  0              0  7        7.4e-05  34        0.0669  
0              0  0              0  93        0.3306  62        0.1945  
214       0.01
03  vmlinux                  vmlinux                  search_by_key
204       0.1502  26603     0.4382  25086     0.2643  45        0.0886  
18        0.0460  19        0.1131  19        0.0675  25        0.0784  
6580      0.31
56  vmlinux                  vmlinux                  __do_softirq
170       0.1252  2618      0.0431  3493      0.0368  69        0.1358  
42        0.1074  51        0.3037  64        0.2275  87        0.2729  
856       0.04
11  vmlinux                  vmlinux                  __schedule
151       0.1112  7590      0.1250  8909      0.0939  273       0.5372  
86        0.2199  73        0.4347  99        0.3519  142       0.4455  
2114      0.10
14  vmlinux                  vmlinux                  sched_clock_cpu
142       0.1046  200       0.0033  165       0.0017  65        0.1279  
23        0.0588  20        0.1191  15        0.0533  36        0.1129  
94        0.00
45  vmlinux                  vmlinux                  raise_softirq
141       0.1038  8048      0.1326  11386     0.1200  2         0.0039  
0              0  0              0  2         0.0071  4         0.0125  
2589      0.12
42  vmlinux                  vmlinux                  __phys_addr
141       0.1038  23726     0.3908  24837     0.2617  89        0.1751  
58        0.1483  55        0.3275  66        0.2346  62        0.1945  
5398      0.25
89  vmlinux                  vmlinux                  
tick_nohz_stop_sched_tick
139       0.1023  43230     0.7120  68500     0.7217  105       0.2066  
66        0.1688  86        0.5121  100       0.3555  100       0.3137  
13725     0.65
83  vmlinux                  vmlinux                  getnstimeofday
136       0.1001  0              0  0              0  0              0  
0              0  0              0  0              0  0              0  
36        0.00
17  vmlinux                  vmlinux                  remove_journal_hash
133       0.0979  8624      0.1420  9518      0.1003  217       0.4270  
68        0.1739  64        0.3811  79        0.2808  106       0.3326  
2348      0.11
26  vmlinux                  vmlinux                  native_sched_clock
129       0.0950  270       0.0044  233       0.0025  355       0.6986  
93        0.2378  60        0.3573  102       0.3626  105       0.3294  
214       0.01
03  vmlinux                  vmlinux                  rebalance_domains
117       0.0861  0              0  36       3.8e-04  57        0.1122  
0              0  10        0.0596  66        0.2346  60        0.1882  
220       0.01
06  vmlinux                  vmlinux                  find_get_page
112       0.0825  287       0.0047  397       0.0042  121       0.2381  
89        0.2276  63        0.3752  85        0.3022  114       0.3577  
220       0.01
06  vmlinux                  vmlinux                  scheduler_tick
107       0.0788  13702     0.2257  16191     0.1706  102       0.2007  
28        0.0716  44        0.2620  41        0.1458  66        0.2071  
3890      0.18
66  vmlinux                  vmlinux                  tick_nohz_stop_idle
105       0.0773  4277      0.0704  7002      0.0738  53        0.1043  
26        0.0665  24        0.1429  24        0.0853  38        0.1192  
1561      0.07
49  vmlinux                  vmlinux                  ring_buffer_consume
103       0.0758  0              0  2        2.1e-05  5         0.0098  
0              0  0              0  20        0.0711  13        0.0408  
61        0.00
29  vmlinux                  vmlinux                  __find_get_block
103       0.0758  49251     0.8112  85999     0.9061  166       0.3267  
105       0.2685  97        0.5777  122       0.4337  138       0.4330  
17108     0.82
06  vmlinux                  vmlinux                  read_tsc
98        0.0722  17406     0.2867  40897     0.4309  15        0.0295  
2         0.0051  2         0.0119  62        0.2204  51        0.1600  
7913      0.37
95  vmlinux                  vmlinux                  kmem_cache_alloc
98        0.0722  14817     0.2440  21291     0.2243  41        0.0807  
49        0.1253  48        0.2859  62        0.2204  64        0.2008  
4582      0.21
98  vmlinux                  vmlinux                  ktime_get_ts
94        0.0692  0              0  1        1.1e-05  2         0.0039  
0              0  0              0  7         0.0249  6         0.0188  
16       7.7e-
04  vmlinux                  vmlinux                  __find_get_block_slow
92        0.0677  1161      0.0191  1411      0.0149  41        0.0807  
39        0.0997  40        0.2382  53        0.1884  49        0.1537  
408       0.01
96  vmlinux                  vmlinux                  
select_nohz_load_balancer
88        0.0648  11160     0.1838  54990     0.5794  69        0.1358  
43        0.1100  35        0.2084  661       2.3499  55        0.1726  
4513      0.21
65  vmlinux                  vmlinux                  
get_next_timer_interrupt
75        0.0552  0              0  0              0  0              0  
0              0  0              0  0              0  0              0  
6        2.9e-
04  vmlinux                  vmlinux                  make_request
69        0.0508  803       0.0132  1048      0.0110  35        0.0689  
26        0.0665  15        0.0893  24        0.0853  58        0.1820  
334       0.01
--More--

Paweł Staszewski pisze:
> I dont know where to put this message so i start here because server 
> that i see this funny thing is make only networking
> Routing (bgpd) / traffic management (iproute2)
>
> This starts on kernels >= 2.6.29
>
> On server i have almost constant traffic that is about 300Mbit/s RX 
> and 300Mbit/s TX
> Server have two interfaces:
> 04:00.0 Ethernet controller: Intel Corporation 82573E Gigabit Ethernet 
> Controller (Copper) (rev 03)
> 05:00.0 Ethernet controller: Intel Corporation 82573L Gigabit Ethernet 
> Controller
>
> iBGP peer < - > [ eth0 - forward + shaping - eth1 ] < - >  eBGP
>
>
> And what is weird is CPU Load
> it fluctuate from 10% to almost 80% - but traffic is  constant :
> bwm-ng v0.6 (probing every 10.000s), press 'h' for help
>  input: /proc/net/dev type: rate
>  |         iface                   Rx                   
> Tx                Total
>  
> ============================================================================== 
>
>               lo:           0.00  b/s            0.00  b/s            
> 0.00  b/s
>             eth0:         200.22 Mb/s          266.74 Mb/s          
> 466.96 Mb/s
>             eth1:         264.00 Mb/s          192.59 Mb/s          
> 456.59 Mb/s
>  
> ------------------------------------------------------------------------------ 
>
>            total:         464.22 Mb/s          459.34 Mb/s          
> 923.56 Mb/s
>
>
> bwm-ng v0.6 (probing every 10.000s), press 'h' for help
>  input: /proc/net/dev type: rate
>  /         iface                   Rx                   
> Tx                Total
>  
> ============================================================================== 
>
>               lo:            0.00 P/s             0.00 P/s             
> 0.00 P/s
>             eth0:        51314.95 P/s         49880.20 P/s        
> 101195.14 P/s
>             eth1:        43309.85 P/s         41885.79 P/s         
> 85195.64 P/s
>  
> ------------------------------------------------------------------------------ 
>
>            total:        94624.80 P/s         91765.99 P/s        
> 186390.79 P/s
>
>
> mpstat -P ALL 1
> 18:00:05     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:06     all    0.00    0.00    0.00    0.00    1.33   13.65    
> 0.00    0.00   85.02
> 18:00:06       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:06       1    0.00    0.00    0.00    0.00    3.77   39.62    
> 0.00    0.00   56.60
> 18:00:06       2    0.00    0.00    0.00    0.00    7.92   70.30    
> 0.00    0.00   21.78
> 18:00:06       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:06       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:06       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:06       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:06       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:06     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:07     all    0.12    0.00    0.12    0.00    1.19   13.35    
> 0.00    0.00   85.22
> 18:00:07       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:07       1    0.00    0.00    0.00    0.00    2.75   42.20    
> 0.00    0.00   55.05
> 18:00:07       2    0.00    0.00    0.99    0.00    5.94   65.35    
> 0.00    0.00   27.72
> 18:00:07       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:07       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:07       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:07       6    0.00    0.00    0.99    0.00    0.00    0.00    
> 0.00    0.00   99.01
> 18:00:07       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:07     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:08     all    0.00    0.00    0.12    0.00    0.85   13.42    
> 0.00    0.00   85.61
> 18:00:08       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:08       1    0.00    0.00    0.00    0.00    1.85   38.89    
> 0.00    0.00   59.26
> 18:00:08       2    0.00    0.00    0.00    0.00    5.94   68.32    
> 0.00    0.00   25.74
> 18:00:08       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:08       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:08       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:08       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:08       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:08     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:09     all    0.00    0.00    0.00    0.00    1.33   14.18    
> 0.00    0.00   84.48
> 18:00:09       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:09       1    0.00    0.00    0.99    0.00    2.97   50.50    
> 0.00    0.00   45.54
> 18:00:09       2    0.00    0.00    0.00    0.00    6.93   65.35    
> 0.00    0.00   27.72
> 18:00:09       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:09       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:09       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:09       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:09       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:09     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:10     all    0.00    0.00    0.24    0.00    1.20   11.81    
> 0.00    0.00   86.75
> 18:00:10       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:10       1    0.00    0.00    0.95    0.00    4.76   35.24    
> 0.00    0.00   59.05
> 18:00:10       2    0.00    0.00    0.00    0.00    4.95   59.41    
> 0.00    0.00   35.64
> 18:00:10       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:10       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:10       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:10       6    0.00    0.00    0.95    0.00    0.00    0.00    
> 0.00    0.00   99.05
> 18:00:10       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:10     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:11     all    0.00    0.00    0.24    0.00    0.36    5.97    
> 0.00    0.00   93.43
> 18:00:11       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:11       1    0.00    0.00    0.97    0.00    0.97   17.48    
> 0.00    0.00   80.58
> 18:00:11       2    0.00    0.00    0.00    0.00    2.88   31.73    
> 0.00    0.00   65.38
> 18:00:11       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:11       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:11       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:11       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:11       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:11     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:12     all    0.00    0.00    0.25    0.00    0.37    5.76    
> 0.00    0.00   93.63
> 18:00:12       0    0.00    0.00    0.00    1.06    0.00    0.00    
> 0.00    0.00   98.94
> 18:00:12       1    0.00    0.00    1.92    0.00    0.00   18.27    
> 0.00    0.00   79.81
> 18:00:12       2    0.00    0.00    0.96    0.00    1.92   26.92    
> 0.00    0.00   70.19
> 18:00:12       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:12       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:12       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:12       6    0.96    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00   99.04
> 18:00:12       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:12     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:13     all    0.12    0.00    0.25    0.00    0.12    2.34    
> 0.00    0.00   97.17
> 18:00:13       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:13       1    0.00    0.00    0.00    0.00    0.97    7.77    
> 0.00    0.00   91.26
> 18:00:13       2    0.00    0.00    0.98    0.00    0.98   10.78    
> 0.00    0.00   87.25
> 18:00:13       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:13       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:13       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:13       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:13       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:13     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:14     all    0.00    0.00    0.37    0.00    0.37    2.33    
> 0.00    0.00   96.94
> 18:00:14       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:14       1    0.00    0.00    1.92    0.00    0.96    7.69    
> 0.00    0.00   89.42
> 18:00:14       2    0.00    0.00    0.97    0.00    0.97   10.68    
> 0.00    0.00   87.38
> 18:00:14       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:14       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:14       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:14       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:14       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:14     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:15     all    0.00    0.00    0.61    0.00    0.25    2.45    
> 0.00    0.00   96.69
> 18:00:15       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:15       1    0.00    0.00    3.85    0.00    0.96    7.69    
> 0.00    0.00   87.50
> 18:00:15       2    0.00    0.00    0.96    0.00    1.92   12.50    
> 0.00    0.00   84.62
> 18:00:15       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:15       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:15       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:15       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:15       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:15     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:16     all    0.00    0.00    0.37    0.00    0.37    1.83    
> 0.00    0.00   97.43
> 18:00:16       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:16       1    0.00    0.00    1.96    0.00    0.98    4.90    
> 0.00    0.00   92.16
> 18:00:16       2    0.00    0.00    1.00    0.00    1.00   10.00    
> 0.00    0.00   88.00
> 18:00:16       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:16       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:16       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:16       6    0.98    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00   99.02
> 18:00:16       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:16     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:17     all    0.00    0.00    0.25    0.00    0.25    2.73    
> 0.00    0.00   96.78
> 18:00:17       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:17       1    0.00    0.00    1.96    0.00    0.00    8.82    
> 0.00    0.00   89.22
> 18:00:17       2    0.00    0.00    0.99    0.00    1.98   11.88    
> 0.00    0.00   85.15
> 18:00:17       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:17       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:17       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:17       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:17       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:17     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:18     all    0.12    0.00    0.12    0.00    0.95   11.51    
> 0.00    0.00   87.31
> 18:00:18       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:18       1    0.00    0.00    0.00    0.00    3.03   33.33    
> 0.00    0.00   63.64
> 18:00:18       2    0.00    0.00    0.00    0.00    5.00   64.00    
> 0.00    0.00   31.00
> 18:00:18       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:18       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:18       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:18       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:18       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:18     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:19     all    0.36    0.00    0.12    0.00    1.46   14.48    
> 0.00    0.00   83.58
> 18:00:19       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:19       1    0.00    0.00    0.00    0.00    4.00   46.00    
> 0.00    0.00   50.00
> 18:00:19       2    0.00    0.00    0.99    0.00    7.92   72.28    
> 0.00    0.00   18.81
> 18:00:19       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:19       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:19       5    2.86    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00   97.14
> 18:00:19       6    0.95    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00   99.05
> 18:00:19       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:19     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:20     all    0.00    0.00    0.00    0.00    1.58   14.86    
> 0.00    0.00   83.56
> 18:00:20       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:20       1    0.00    0.00    0.00    0.00    3.96   49.50    
> 0.00    0.00   46.53
> 18:00:20       2    0.00    0.00    0.00    0.00    9.00   73.00    
> 0.00    0.00   18.00
> 18:00:20       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:20       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:20       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:20       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:20       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:20     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:21     all    0.00    0.00    0.00    0.00    1.71   14.86    
> 0.00    0.00   83.43
> 18:00:21       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:21       1    0.00    0.00    0.00    0.00    5.05   51.52    
> 0.00    0.00   43.43
> 18:00:21       2    0.00    0.00    0.00    0.00   10.00   70.00    
> 0.00    0.00   20.00
> 18:00:21       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:21       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:21       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:21       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:21       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:21     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:22     all    0.00    0.00    0.24    0.00    1.70   13.97    
> 0.00    0.00   84.08
> 18:00:22       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:22       1    0.00    0.00    0.00    0.00    4.95   42.57    
> 0.00    0.00   52.48
> 18:00:22       2    0.00    0.00    1.01    0.00    8.08   72.73    
> 0.00    0.00   18.18
> 18:00:22       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:22       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:22       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:22       6    0.00    0.00    0.95    0.00    0.00    0.00    
> 0.00    0.00   99.05
> 18:00:22       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:22     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:23     all    0.12    0.00    0.00    0.00    1.58   14.13    
> 0.00    0.00   84.17
> 18:00:23       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:23       1    0.00    0.00    0.00    0.00    5.00   45.00    
> 0.00    0.00   50.00
> 18:00:23       2    0.99    0.00    0.00    0.00    8.91   70.30    
> 0.00    0.00   19.80
> 18:00:23       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:23       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:23       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:23       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:23       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:23     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:24     all    0.00    0.00    0.00    0.00    1.34   14.13    
> 0.00    0.00   84.53
> 18:00:24       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:24       1    0.00    0.00    0.00    0.00    4.04   42.42    
> 0.00    0.00   53.54
> 18:00:24       2    0.00    0.00    0.00    0.00    6.93   73.27    
> 0.00    0.00   19.80
> 18:00:24       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:24       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:24       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:24       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:24       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:24     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:25     all    0.12    0.00    0.12    0.00    1.45   14.67    
> 0.00    0.00   83.64
> 18:00:25       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:25       1    0.00    0.00    0.99    0.00    3.96   45.54    
> 0.00    0.00   49.50
> 18:00:25       2    0.00    0.00    0.00    0.00    8.00   75.00    
> 0.00    0.00   17.00
> 18:00:25       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:25       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:25       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:25       6    0.95    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00   99.05
> 18:00:25       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:25     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:26     all    0.00    0.00    0.00    0.00    1.22   14.60    
> 0.00    0.00   84.18
> 18:00:26       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:26       1    0.99    0.00    0.00    0.00    2.97   50.50    
> 0.00    0.00   45.54
> 18:00:26       2    0.00    0.00    0.00    0.00    7.07   68.69    
> 0.00    0.00   24.24
> 18:00:26       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:26       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:26       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:26       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:26       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:26     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:27     all    0.00    0.00    0.00    0.00    1.59   15.02    
> 0.00    0.00   83.39
> 18:00:27       0    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:27       1    0.00    0.00    0.00    0.00    6.00   52.00    
> 0.00    0.00   42.00
> 18:00:27       2    0.00    0.00    0.00    0.00    6.93   71.29    
> 0.00    0.00   21.78
> 18:00:27       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:27       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:27       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:27       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:27       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> 18:00:27     CPU    %usr   %nice    %sys %iowait    %irq   %soft  
> %steal  %guest   %idle
> 18:00:28     all    0.00    0.00    0.00    0.00    1.33   14.56    
> 0.00    0.00   84.10
> 18:00:28       0    0.00    0.00    0.00    0.00    0.00    0.99    
> 0.00    0.00   99.01
> 18:00:28       1    0.00    0.00    0.00    0.00    4.00   50.00    
> 0.00    0.00   46.00
> 18:00:28       2    0.00    0.00    0.00    0.00    6.06   69.70    
> 0.00    0.00   24.24
> 18:00:28       3    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:28       4    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:28       5    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:28       6    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
> 18:00:28       7    0.00    0.00    0.00    0.00    0.00    0.00    
> 0.00    0.00  100.00
>
> at the same time top shows:
>  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
>   28 root      15  -5     0    0    0 S    2  0.0   0:16.60 events/1
>   29 root      15  -5     0    0    0 S    2  0.0   0:15.62 events/2
>   10 root      15  -5     0    0    0 S    2  0.0   0:41.01 ksoftirqd/2
>    7 root      15  -5     0    0    0 S    1  0.0   0:31.17 ksoftirqd/1
>   30 root      15  -5     0    0    0 S    1  0.0   0:03.00 events/3
> 6545 root      20   0 11112 3244 1780 S    0  0.0   0:08.69 oprofiled
> 6845 root      20   0 19084 1384 1008 R    0  0.0   0:00.07 top
>    1 root      20   0  3812  612  520 S    0  0.0   0:01.00 init
>    2 root      15  -5     0    0    0 S    0  0.0   0:00.00 kthreadd
>    3 root      RT  -5     0    0    0 S    0  0.0   0:00.00 migration/0
>    4 root      15  -5     0    0    0 S    0  0.0   0:01.04 ksoftirqd/0
>    5 root      RT  -5     0    0    0 S    0  0.0   0:00.00 watchdog/0
>    6 root      RT  -5     0    0    0 S    0  0.0   0:00.00 migration/1
>    8 root      RT  -5     0    0    0 S    0  0.0   0:00.00 watchdog/1
>    9 root      RT  -5     0    0    0 S    0  0.0   0:00.00 migration/2
>   11 root      RT  -5     0    0    0 S    0  0.0   0:00.00 watchdog/2
>   12 root      RT  -5     0    0    0 S    0  0.0   0:00.00 migration/3
>   13 root      15  -5     0    0    0 S    0  0.0   0:00.00 ksoftirqd/3
>   14 root      RT  -5     0    0    0 S    0  0.0   0:00.00 watchdog/3
>   15 root      RT  -5     0    0    0 S    0  0.0   0:00.00 migration/4
>   16 root      15  -5     0    0    0 S    0  0.0   0:00.00 ksoftirqd/4
>   17 root      RT  -5     0    0    0 S    0  0.0   0:00.00 watchdog/4
>   18 root      RT  -5     0    0    0 S    0  0.0   0:00.00 migration/5
>   19 root      15  -5     0    0    0 S    0  0.0   0:00.00 ksoftirqd/5
>   20 root      RT  -5     0    0    0 S    0  0.0   0:00.00 watchdog/5
>   21 root      RT  -5     0    0    0 S    0  0.0   0:00.00 migration/6
>   22 root      15  -5     0    0    0 S    0  0.0   0:00.00 ksoftirqd/6
>   23 root      RT  -5     0    0    0 S    0  0.0   0:00.00 watchdog/6
>   24 root      RT  -5     0    0    0 S    0  0.0   0:00.00 migration/7
>   25 root      15  -5     0    0    0 S    0  0.0   0:00.00 ksoftirqd/7
>   26 root      RT  -5     0    0    0 S    0  0.0   0:00.00 watchdog/7
>   27 root      15  -5     0    0    0 S    0  0.0   0:00.00 events/0
>   31 root      15  -5     0    0    0 S    0  0.0   0:00.00 events/4
>   32 root      15  -5     0    0    0 S    0  0.0   0:00.01 events/5
>   33 root      15  -5     0    0    0 S    0  0.0   0:00.02 events/6
>   34 root      15  -5     0    0    0 S    0  0.0   0:00.01 events/7
>   35 root      15  -5     0    0    0 S    0  0.0   0:00.01 khelper
>   41 root      15  -5     0    0    0 S    0  0.0   0:00.00 async/mgr
>  159 root      15  -5     0    0    0 S    0  0.0   0:00.00 kblockd/0
>  160 root      15  -5     0    0    0 S    0  0.0   0:00.00 kblockd/1
>  161 root      15  -5     0    0    0 S    0  0.0   0:00.00 kblockd/2
>  162 root      15  -5     0    0    0 S    0  0.0   0:00.00 kblockd/3
>  163 root      15  -5     0    0    0 S    0  0.0   0:00.00 kblockd/4
>  164 root      15  -5     0    0    0 S    0  0.0   0:00.00 kblockd/5
>  165 root      15  -5     0    0    0 S    0  0.0   0:00.00 kblockd/6
>  166 root      15  -5     0    0    0 S    0  0.0   0:00.00 kblockd/7
>  167 root      15  -5     0    0    0 S    0  0.0   0:00.00 kacpid
>  168 root      15  -5     0    0    0 S    0  0.0   0:00.00 kacpi_notify
>  246 root      15  -5     0    0    0 S    0  0.0   0:00.00 ata/0
>  247 root      15  -5     0    0    0 S    0  0.0   0:00.00 ata/1
>
> and oprofile output:
> CPU: Core 2, speed 3000.08 MHz (estimated)
> Counted CPU_CLK_UNHALTED events (Clock cycles when not halted) with a 
> unit mask of 0x00 (Unhalted core cycles) count 100000
> samples  %        image name               app name                 
> symbol name
> 11138304 12.3598  vmlinux                  vmlinux                  
> rb_next
> 7714260   8.5602  vmlinux                  vmlinux                  
> e1000_clean
> 5380466   5.9705  vmlinux                  vmlinux                  
> e1000_intr_msi
> 4881445   5.4168  vmlinux                  vmlinux                  
> u32_classify
> 3977072   4.4132  vmlinux                  vmlinux                  
> _raw_spin_lock
> 3471611   3.8523  vmlinux                  vmlinux                  
> hfsc_dequeue
> 3103493   3.4438  vmlinux                  vmlinux                  
> hfsc_enqueue
> 2636824   2.9260  vmlinux                  vmlinux                  
> delay_tsc
> 2464293   2.7345  vmlinux                  vmlinux                  
> e1000_xmit_frame
> 2035779   2.2590  vmlinux                  vmlinux                  
> e1000_clean_tx_irq
> 1947231   2.1608  vmlinux                  vmlinux                  
> mwait_idle
> 1919540   2.1300  vmlinux                  vmlinux                  
> ipt_do_table
> 1908544   2.1178  vmlinux                  vmlinux                  
> e1000_clean_rx_irq
> 1865996   2.0706  vmlinux                  vmlinux                  
> update_vf
> 1829293   2.0299  vmlinux                  vmlinux                  
> rb_erase
> 1777847   1.9728  vmlinux                  vmlinux                  
> ip_route_input
> 1430773   1.5877  vmlinux                  vmlinux                  
> rb_insert_color
> 1329674   1.4755  vmlinux                  vmlinux                  
> __alloc_skb
> 1091678   1.2114  vmlinux                  vmlinux                  
> sfq_enqueue
> 985105    1.0931  vmlinux                  vmlinux                  
> rb_first
> 944928    1.0486  vmlinux                  vmlinux                  
> sfq_dequeue
> 756452    0.8394  vmlinux                  vmlinux                  
> read_tsc
> 728703    0.8086  vmlinux                  vmlinux                  
> nf_iterate
> 665551    0.7385  vmlinux                  vmlinux                  
> ip_forward
> 662616    0.7353  vmlinux                  vmlinux                  
> dev_queue_xmit
> 648919    0.7201  vmlinux                  vmlinux                  
> __qdisc_run
> 615592    0.6831  vmlinux                  vmlinux                  
> ip_rcv
> 589743    0.6544  vmlinux                  vmlinux                  
> apic_timer_interrupt
> 550581    0.6110  vmlinux                  vmlinux                  
> irq_entries_start
> 536172    0.5950  vmlinux                  vmlinux                  
> getnstimeofday
> 527467    0.5853  vmlinux                  vmlinux                  
> rb_last
> 508305    0.5640  vmlinux                  vmlinux                  
> skb_release_data
> 506763    0.5623  vmlinux                  vmlinux                  kfree
> 506269    0.5618  vmlinux                  vmlinux                  
> dst_release
> 464412    0.5153  vmlinux                  vmlinux                  
> __hrtimer_start_range_ns
> 456354    0.5064  vmlinux                  vmlinux                  
> e1000_alloc_rx_buffers
> 435339    0.4831  vmlinux                  vmlinux                  
> dev_hard_start_xmit
> 371439    0.4122  vmlinux                  vmlinux                  
> local_bh_enable
> 369266    0.4098  vmlinux                  vmlinux                  
> __rb_rotate_left
> 361627    0.4013  vmlinux                  vmlinux                  
> net_tx_action
> 357797    0.3970  vmlinux                  vmlinux                  
> rtsc_y2x
> 350595    0.3890  vmlinux                  vmlinux                  
> _raw_spin_unlock
> 349359    0.3877  vmlinux                  vmlinux                  
> kmem_cache_alloc
> 336255    0.3731  vmlinux                  vmlinux                  
> skb_dma_map
> 328809    0.3649  vmlinux                  vmlinux                  
> netif_receive_skb
> 315826    0.3505  vmlinux                  vmlinux                  
> fn_trie_lookup
> 313065    0.3474  vmlinux                  vmlinux                  
> __kmalloc
> 307684    0.3414  vmlinux                  vmlinux                  
> __do_softirq
> 298676    0.3314  vmlinux                  vmlinux                  
> skb_pull
> 292352    0.3244  vmlinux                  vmlinux                  
> __napi_complete
> 288878    0.3206  vmlinux                  vmlinux                  
> get_next_timer_interrupt
> 280449    0.3112  vmlinux                  vmlinux                  
> ip_finish_output
> 264482    0.2935  vmlinux                  vmlinux                  
> consume_skb
> 251622    0.2792  vmlinux                  vmlinux                  
> __remove_hrtimer
> 227178    0.2521  vmlinux                  vmlinux                  
> tc_classify_compat
> 226224    0.2510  vmlinux                  vmlinux                  
> handle_edge_irq
>
>
>
> proces that this machine is running is only quagga (zebra + bgpd + 
> oprofiled)
>
>
> Regards
> Paweł Staszewski
>
>
> -- 
> To unsubscribe from this list: send the line "unsubscribe netdev" in
> the body of a message to [email-masked]
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>
>