]>
git.ipfire.org Git - thirdparty/kernel/stable.git/blob - tools/power/x86/intel_pstate_tracer/intel_pstate_tracer.py
2fa3c5757bcb5b4f72cf85a6bf8a6cbad8efbf6f
2 # -*- coding: utf-8 -*-
4 """ This utility can be used to debug and tune the performance of the
5 intel_pstate driver. This utility can be used in two ways:
6 - If there is Linux trace file with pstate_sample events enabled, then
7 this utility can parse the trace file and generate performance plots.
8 - If user has not specified a trace file as input via command line parameters,
9 then this utility enables and collects trace data for a user specified interval
10 and generates performance plots.
16 (Most of the distributions have these required packages. They may be called
17 gnuplot-py, phython-gnuplot. )
19 HWP (Hardware P-States are disabled)
20 Kernel config for Linux trace is enabled
22 see print_help(): for Usage and Output details
25 from __future__
import print_function
26 from datetime
import datetime
38 __author__
= "Srinivas Pandruvada"
39 __copyright__
= " Copyright (c) 2017, Intel Corporation. "
40 __license__
= "GPL version 2"
45 # Define the csv file columns
65 global sample_num
, last_sec_cpu
, last_usec_cpu
, start_time
, testname
67 # 11 digits covers uptime to 115 days
68 getcontext().prec
= 11
71 last_sec_cpu
= [0] * MAX_CPUS
72 last_usec_cpu
= [0] * MAX_CPUS
75 print('intel_pstate_tracer.py:')
77 print(' If the trace file is available, then to simply parse and plot, use (sudo not required):')
78 print(' ./intel_pstate_tracer.py [-c cpus] -t <trace_file> -n <test_name>')
80 print(' ./intel_pstate_tracer.py [--cpu cpus] ---trace_file <trace_file> --name <test_name>')
81 print(' To generate trace file, parse and plot, use (sudo required):')
82 print(' sudo ./intel_pstate_tracer.py [-c cpus] -i <interval> -n <test_name> -m <kbytes>')
84 print(' sudo ./intel_pstate_tracer.py [--cpu cpus] --interval <interval> --name <test_name> --memory <kbytes>')
85 print(' Optional argument:')
86 print(' cpus: comma separated list of CPUs')
87 print(' kbytes: Kilo bytes of memory per CPU to allocate to the trace buffer. Default: 10240')
89 print(' If not already present, creates a "results/test_name" folder in the current working directory with:')
90 print(' cpu.csv - comma seperated values file with trace contents and some additional calculations.')
91 print(' cpu???.csv - comma seperated values file for CPU number ???.')
92 print(' *.png - a variety of PNG format plot files created from the trace contents and the additional calculations.')
94 print(' Avoid the use of _ (underscore) in test names, because in gnuplot it is a subscript directive.')
95 print(' Maximum number of CPUs is {0:d}. If there are more the script will abort with an error.'.format(MAX_CPUS
))
96 print(' Off-line CPUs cause the script to list some warnings, and create some empty files. Use the CPU mask feature for a clean run.')
97 print(' Empty y range warnings for autoscaled plots can occur and can be ignored.')
99 def plot_perf_busy_with_sample(cpu_index
):
100 """ Plot method to per cpu information """
102 file_name
= 'cpu{:0>3}.csv'.format(cpu_index
)
103 if os
.path
.exists(file_name
):
104 output_png
= "cpu%03d_perf_busy_vs_samples.png" % cpu_index
105 g_plot
= common_all_gnuplot_settings(output_png
)
106 g_plot('set yrange [0:40]')
107 g_plot('set y2range [0:200]')
108 g_plot('set y2tics 0, 10')
109 g_plot('set title "{} : cpu perf busy vs. sample : CPU {:0>3} : {:%F %H:%M}"'.format(testname
, cpu_index
, datetime
.now()))
111 g_plot('set xlabel "Samples"')
112 g_plot('set ylabel "P-State"')
113 g_plot('set y2label "Scaled Busy/performance/io-busy(%)"')
114 set_4_plot_linestyles(g_plot
)
115 g_plot('plot "' + file_name
+ '" using {:d}:{:d} with linespoints linestyle 1 axis x1y2 title "performance",\\'.format(C_SAMPLE
, C_CORE
))
116 g_plot('"' + file_name
+ '" using {:d}:{:d} with linespoints linestyle 2 axis x1y2 title "scaled-busy",\\'.format(C_SAMPLE
, C_SCALED
))
117 g_plot('"' + file_name
+ '" using {:d}:{:d} with linespoints linestyle 3 axis x1y2 title "io-boost",\\'.format(C_SAMPLE
, C_BOOST
))
118 g_plot('"' + file_name
+ '" using {:d}:{:d} with linespoints linestyle 4 axis x1y1 title "P-State"'.format(C_SAMPLE
, C_TO
))
120 def plot_perf_busy(cpu_index
):
121 """ Plot some per cpu information """
123 file_name
= 'cpu{:0>3}.csv'.format(cpu_index
)
124 if os
.path
.exists(file_name
):
125 output_png
= "cpu%03d_perf_busy.png" % cpu_index
126 g_plot
= common_all_gnuplot_settings(output_png
)
127 g_plot('set yrange [0:40]')
128 g_plot('set y2range [0:200]')
129 g_plot('set y2tics 0, 10')
130 g_plot('set title "{} : perf busy : CPU {:0>3} : {:%F %H:%M}"'.format(testname
, cpu_index
, datetime
.now()))
131 g_plot('set ylabel "P-State"')
132 g_plot('set y2label "Scaled Busy/performance/io-busy(%)"')
133 set_4_plot_linestyles(g_plot
)
134 g_plot('plot "' + file_name
+ '" using {:d}:{:d} with linespoints linestyle 1 axis x1y2 title "performance",\\'.format(C_ELAPSED
, C_CORE
))
135 g_plot('"' + file_name
+ '" using {:d}:{:d} with linespoints linestyle 2 axis x1y2 title "scaled-busy",\\'.format(C_ELAPSED
, C_SCALED
))
136 g_plot('"' + file_name
+ '" using {:d}:{:d} with linespoints linestyle 3 axis x1y2 title "io-boost",\\'.format(C_ELAPSED
, C_BOOST
))
137 g_plot('"' + file_name
+ '" using {:d}:{:d} with linespoints linestyle 4 axis x1y1 title "P-State"'.format(C_ELAPSED
, C_TO
))
139 def plot_durations(cpu_index
):
140 """ Plot per cpu durations """
142 file_name
= 'cpu{:0>3}.csv'.format(cpu_index
)
143 if os
.path
.exists(file_name
):
144 output_png
= "cpu%03d_durations.png" % cpu_index
145 g_plot
= common_all_gnuplot_settings(output_png
)
146 # Should autoscale be used here? Should seconds be used here?
147 g_plot('set yrange [0:5000]')
148 g_plot('set ytics 0, 500')
149 g_plot('set title "{} : durations : CPU {:0>3} : {:%F %H:%M}"'.format(testname
, cpu_index
, datetime
.now()))
150 g_plot('set ylabel "Timer Duration (MilliSeconds)"')
152 g_plot('set key off')
153 set_4_plot_linestyles(g_plot
)
154 g_plot('plot "' + file_name
+ '" using {:d}:{:d} with linespoints linestyle 1 axis x1y1'.format(C_ELAPSED
, C_DURATION
))
156 def plot_loads(cpu_index
):
157 """ Plot per cpu loads """
159 file_name
= 'cpu{:0>3}.csv'.format(cpu_index
)
160 if os
.path
.exists(file_name
):
161 output_png
= "cpu%03d_loads.png" % cpu_index
162 g_plot
= common_all_gnuplot_settings(output_png
)
163 g_plot('set yrange [0:100]')
164 g_plot('set ytics 0, 10')
165 g_plot('set title "{} : loads : CPU {:0>3} : {:%F %H:%M}"'.format(testname
, cpu_index
, datetime
.now()))
166 g_plot('set ylabel "CPU load (percent)"')
168 g_plot('set key off')
169 set_4_plot_linestyles(g_plot
)
170 g_plot('plot "' + file_name
+ '" using {:d}:{:d} with linespoints linestyle 1 axis x1y1'.format(C_ELAPSED
, C_LOAD
))
172 def plot_pstate_cpu_with_sample():
173 """ Plot all cpu information """
175 if os
.path
.exists('cpu.csv'):
176 output_png
= 'all_cpu_pstates_vs_samples.png'
177 g_plot
= common_all_gnuplot_settings(output_png
)
178 g_plot('set yrange [0:40]')
180 g_plot('set xlabel "Samples"')
181 g_plot('set ylabel "P-State"')
182 g_plot('set title "{} : cpu pstate vs. sample : {:%F %H:%M}"'.format(testname
, datetime
.now()))
183 title_list
= subprocess
.check_output('ls cpu???.csv | sed -e \'s/.csv//\'',shell
=True).replace('\n', ' ')
184 plot_str
= "plot for [i in title_list] i.'.csv' using {:d}:{:d} pt 7 ps 1 title i".format(C_SAMPLE
, C_TO
)
185 g_plot('title_list = "{}"'.format(title_list
))
188 def plot_pstate_cpu():
189 """ Plot all cpu information from csv files """
191 output_png
= 'all_cpu_pstates.png'
192 g_plot
= common_all_gnuplot_settings(output_png
)
193 g_plot('set yrange [0:40]')
194 g_plot('set ylabel "P-State"')
195 g_plot('set title "{} : cpu pstates : {:%F %H:%M}"'.format(testname
, datetime
.now()))
197 # the following command is really cool, but doesn't work with the CPU masking option because it aborts on the first missing file.
198 # plot_str = 'plot for [i=0:*] file=sprintf("cpu%03d.csv",i) title_s=sprintf("cpu%03d",i) file using 16:7 pt 7 ps 1 title title_s'
200 title_list
= subprocess
.check_output('ls cpu???.csv | sed -e \'s/.csv//\'',shell
=True).replace('\n', ' ')
201 plot_str
= "plot for [i in title_list] i.'.csv' using {:d}:{:d} pt 7 ps 1 title i".format(C_ELAPSED
, C_TO
)
202 g_plot('title_list = "{}"'.format(title_list
))
206 """ Plot all cpu loads """
208 output_png
= 'all_cpu_loads.png'
209 g_plot
= common_all_gnuplot_settings(output_png
)
210 g_plot('set yrange [0:100]')
211 g_plot('set ylabel "CPU load (percent)"')
212 g_plot('set title "{} : cpu loads : {:%F %H:%M}"'.format(testname
, datetime
.now()))
214 title_list
= subprocess
.check_output('ls cpu???.csv | sed -e \'s/.csv//\'',shell
=True).replace('\n', ' ')
215 plot_str
= "plot for [i in title_list] i.'.csv' using {:d}:{:d} pt 7 ps 1 title i".format(C_ELAPSED
, C_LOAD
)
216 g_plot('title_list = "{}"'.format(title_list
))
219 def plot_frequency_cpu():
220 """ Plot all cpu frequencies """
222 output_png
= 'all_cpu_frequencies.png'
223 g_plot
= common_all_gnuplot_settings(output_png
)
224 g_plot('set yrange [0:4]')
225 g_plot('set ylabel "CPU Frequency (GHz)"')
226 g_plot('set title "{} : cpu frequencies : {:%F %H:%M}"'.format(testname
, datetime
.now()))
228 title_list
= subprocess
.check_output('ls cpu???.csv | sed -e \'s/.csv//\'',shell
=True).replace('\n', ' ')
229 plot_str
= "plot for [i in title_list] i.'.csv' using {:d}:{:d} pt 7 ps 1 title i".format(C_ELAPSED
, C_FREQ
)
230 g_plot('title_list = "{}"'.format(title_list
))
233 def plot_duration_cpu():
234 """ Plot all cpu durations """
236 output_png
= 'all_cpu_durations.png'
237 g_plot
= common_all_gnuplot_settings(output_png
)
238 g_plot('set yrange [0:5000]')
239 g_plot('set ytics 0, 500')
240 g_plot('set ylabel "Timer Duration (MilliSeconds)"')
241 g_plot('set title "{} : cpu durations : {:%F %H:%M}"'.format(testname
, datetime
.now()))
243 title_list
= subprocess
.check_output('ls cpu???.csv | sed -e \'s/.csv//\'',shell
=True).replace('\n', ' ')
244 plot_str
= "plot for [i in title_list] i.'.csv' using {:d}:{:d} pt 7 ps 1 title i".format(C_ELAPSED
, C_DURATION
)
245 g_plot('title_list = "{}"'.format(title_list
))
248 def plot_scaled_cpu():
249 """ Plot all cpu scaled busy """
251 output_png
= 'all_cpu_scaled.png'
252 g_plot
= common_all_gnuplot_settings(output_png
)
253 # autoscale this one, no set y range
254 g_plot('set ylabel "Scaled Busy (Unitless)"')
255 g_plot('set title "{} : cpu scaled busy : {:%F %H:%M}"'.format(testname
, datetime
.now()))
257 title_list
= subprocess
.check_output('ls cpu???.csv | sed -e \'s/.csv//\'',shell
=True).replace('\n', ' ')
258 plot_str
= "plot for [i in title_list] i.'.csv' using {:d}:{:d} pt 7 ps 1 title i".format(C_ELAPSED
, C_SCALED
)
259 g_plot('title_list = "{}"'.format(title_list
))
262 def plot_boost_cpu():
263 """ Plot all cpu IO Boosts """
265 output_png
= 'all_cpu_boost.png'
266 g_plot
= common_all_gnuplot_settings(output_png
)
267 g_plot('set yrange [0:100]')
268 g_plot('set ylabel "CPU IO Boost (percent)"')
269 g_plot('set title "{} : cpu io boost : {:%F %H:%M}"'.format(testname
, datetime
.now()))
271 title_list
= subprocess
.check_output('ls cpu???.csv | sed -e \'s/.csv//\'',shell
=True).replace('\n', ' ')
272 plot_str
= "plot for [i in title_list] i.'.csv' using {:d}:{:d} pt 7 ps 1 title i".format(C_ELAPSED
, C_BOOST
)
273 g_plot('title_list = "{}"'.format(title_list
))
277 """ Plot all cpu tsc ghz """
279 output_png
= 'all_cpu_ghz.png'
280 g_plot
= common_all_gnuplot_settings(output_png
)
281 # autoscale this one, no set y range
282 g_plot('set ylabel "TSC Frequency (GHz)"')
283 g_plot('set title "{} : cpu TSC Frequencies (Sanity check calculation) : {:%F %H:%M}"'.format(testname
, datetime
.now()))
285 title_list
= subprocess
.check_output('ls cpu???.csv | sed -e \'s/.csv//\'',shell
=True).replace('\n', ' ')
286 plot_str
= "plot for [i in title_list] i.'.csv' using {:d}:{:d} pt 7 ps 1 title i".format(C_ELAPSED
, C_GHZ
)
287 g_plot('title_list = "{}"'.format(title_list
))
290 def common_all_gnuplot_settings(output_png
):
291 """ common gnuplot settings for multiple CPUs one one graph. """
293 g_plot
= common_gnuplot_settings()
294 g_plot('set output "' + output_png
+ '"')
297 def common_gnuplot_settings():
298 """ common gnuplot settings. """
300 g_plot
= Gnuplot
.Gnuplot(persist
=1)
301 # The following line is for rigor only. It seems to be assumed for .csv files
302 g_plot('set datafile separator \",\"')
303 g_plot('set ytics nomirror')
304 g_plot('set xtics nomirror')
305 g_plot('set xtics font ", 10"')
306 g_plot('set ytics font ", 10"')
307 g_plot('set tics out scale 1.0')
309 g_plot('set key out horiz')
310 g_plot('set key bot center')
311 g_plot('set key samplen 2 spacing .8 font ", 9"')
312 g_plot('set term png size 1200, 600')
313 g_plot('set title font ", 11"')
314 g_plot('set ylabel font ", 10"')
315 g_plot('set xlabel font ", 10"')
316 g_plot('set xlabel offset 0, 0.5')
317 g_plot('set xlabel "Elapsed Time (Seconds)"')
320 def set_4_plot_linestyles(g_plot
):
321 """ set the linestyles used for 4 plots in 1 graphs. """
323 g_plot('set style line 1 linetype 1 linecolor rgb "green" pointtype -1')
324 g_plot('set style line 2 linetype 1 linecolor rgb "red" pointtype -1')
325 g_plot('set style line 3 linetype 1 linecolor rgb "purple" pointtype -1')
326 g_plot('set style line 4 linetype 1 linecolor rgb "blue" pointtype -1')
328 def store_csv(cpu_int
, time_pre_dec
, time_post_dec
, core_busy
, scaled
, _from
, _to
, mperf
, aperf
, tsc
, freq_ghz
, io_boost
, common_comm
, load
, duration_ms
, sample_num
, elapsed_time
, tsc_ghz
):
329 """ Store master csv file information """
331 global graph_data_present
333 if cpu_mask
[cpu_int
] == 0:
337 f_handle
= open('cpu.csv', 'a')
338 string_buffer
= "CPU_%03u, %05u, %06u, %u, %u, %u, %u, %u, %u, %u, %.4f, %u, %.2f, %.3f, %u, %.3f, %.3f, %s\n" % (cpu_int
, int(time_pre_dec
), int(time_post_dec
), int(core_busy
), int(scaled
), int(_from
), int(_to
), int(mperf
), int(aperf
), int(tsc
), freq_ghz
, int(io_boost
), load
, duration_ms
, sample_num
, elapsed_time
, tsc_ghz
, common_comm
)
339 f_handle
.write(string_buffer
);
342 print('IO error cpu.csv')
345 graph_data_present
= True;
348 """ seperate the all csv file into per CPU csv files. """
350 global current_max_cpu
352 if os
.path
.exists('cpu.csv'):
353 for index
in range(0, current_max_cpu
+ 1):
354 if cpu_mask
[int(index
)] != 0:
355 os
.system('grep -m 1 common_cpu cpu.csv > cpu{:0>3}.csv'.format(index
))
356 os
.system('grep CPU_{:0>3} cpu.csv >> cpu{:0>3}.csv'.format(index
, index
))
358 def fix_ownership(path
):
359 """Change the owner of the file to SUDO_UID, if required"""
361 uid
= os
.environ
.get('SUDO_UID')
362 gid
= os
.environ
.get('SUDO_GID')
364 os
.chown(path
, int(uid
), int(gid
))
366 def cleanup_data_files():
367 """ clean up existing data files """
369 if os
.path
.exists('cpu.csv'):
371 f_handle
= open('cpu.csv', 'a')
372 f_handle
.write('common_cpu, common_secs, common_usecs, core_busy, scaled_busy, from, to, mperf, aperf, tsc, freq, boost, load, duration_ms, sample_num, elapsed_time, tsc_ghz, common_comm')
376 def clear_trace_file():
377 """ Clear trace file """
380 f_handle
= open('/sys/kernel/debug/tracing/trace', 'w')
383 print('IO error clearing trace file ')
390 open('/sys/kernel/debug/tracing/events/power/pstate_sample/enable'
393 print('IO error enabling trace ')
397 """ Disable trace """
400 open('/sys/kernel/debug/tracing/events/power/pstate_sample/enable'
403 print('IO error disabling trace ')
406 def set_trace_buffer_size():
407 """ Set trace buffer size """
410 with
open('/sys/kernel/debug/tracing/buffer_size_kb', 'w') as fp
:
413 print('IO error setting trace buffer size ')
416 def free_trace_buffer():
417 """ Free the trace buffer memory """
420 open('/sys/kernel/debug/tracing/buffer_size_kb'
423 print('IO error freeing trace buffer ')
426 def read_trace_data(filename
):
427 """ Read and parse trace data """
429 global current_max_cpu
430 global sample_num
, last_sec_cpu
, last_usec_cpu
, start_time
433 data
= open(filename
, 'r').read()
435 print('Error opening ', filename
)
438 for line
in data
.splitlines():
440 re
.search(r
'(^(.*?)\[)((\d+)[^\]])(.*?)(\d+)([.])(\d+)(.*?core_busy=)(\d+)(.*?scaled=)(\d+)(.*?from=)(\d+)(.*?to=)(\d+)(.*?mperf=)(\d+)(.*?aperf=)(\d+)(.*?tsc=)(\d+)(.*?freq=)(\d+)'
444 cpu
= search_obj
.group(3)
448 time_pre_dec
= search_obj
.group(6)
449 time_post_dec
= search_obj
.group(8)
450 core_busy
= search_obj
.group(10)
451 scaled
= search_obj
.group(12)
452 _from
= search_obj
.group(14)
453 _to
= search_obj
.group(16)
454 mperf
= search_obj
.group(18)
455 aperf
= search_obj
.group(20)
456 tsc
= search_obj
.group(22)
457 freq
= search_obj
.group(24)
458 common_comm
= search_obj
.group(2).replace(' ', '')
460 # Not all kernel versions have io_boost field
462 search_obj
= re
.search(r
'.*?io_boost=(\d+)', line
)
464 io_boost
= search_obj
.group(1)
467 start_time
= Decimal(time_pre_dec
) + Decimal(time_post_dec
) / Decimal(1000000)
470 if last_sec_cpu
[cpu_int
] == 0 :
471 last_sec_cpu
[cpu_int
] = time_pre_dec
472 last_usec_cpu
[cpu_int
] = time_post_dec
474 duration_us
= (int(time_pre_dec
) - int(last_sec_cpu
[cpu_int
])) * 1000000 + (int(time_post_dec
) - int(last_usec_cpu
[cpu_int
]))
475 duration_ms
= Decimal(duration_us
) / Decimal(1000)
476 last_sec_cpu
[cpu_int
] = time_pre_dec
477 last_usec_cpu
[cpu_int
] = time_post_dec
478 elapsed_time
= Decimal(time_pre_dec
) + Decimal(time_post_dec
) / Decimal(1000000) - start_time
479 load
= Decimal(int(mperf
)*100)/ Decimal(tsc
)
480 freq_ghz
= Decimal(freq
)/Decimal(1000000)
481 # Sanity check calculation, typically anomalies indicate missed samples
482 # However, check for 0 (should never occur)
484 if duration_ms
!= Decimal(0) :
485 tsc_ghz
= Decimal(tsc
)/duration_ms
/Decimal(1000000)
486 store_csv(cpu_int
, time_pre_dec
, time_post_dec
, core_busy
, scaled
, _from
, _to
, mperf
, aperf
, tsc
, freq_ghz
, io_boost
, common_comm
, load
, duration_ms
, sample_num
, elapsed_time
, tsc_ghz
)
488 if cpu_int
> current_max_cpu
:
489 current_max_cpu
= cpu_int
490 # End of for each trace line loop
491 # Now seperate the main overall csv file into per CPU csv files.
494 def signal_handler(signal
, frame
):
495 print(' SIGINT: Forcing cleanup before exit.')
503 signal
.signal(signal
.SIGINT
, signal_handler
)
510 graph_data_present
= False;
515 cpu_mask
= zeros((MAX_CPUS
,), dtype
=int)
518 opts
, args
= getopt
.getopt(sys
.argv
[1:],"ht:i:c:n:m:",["help","trace_file=","interval=","cpu=","name=","memory="])
519 except getopt
.GetoptError
:
522 for opt
, arg
in opts
:
526 elif opt
in ("-t", "--trace_file"):
528 location
= os
.path
.realpath(os
.path
.join(os
.getcwd(), os
.path
.dirname(__file__
)))
529 filename
= os
.path
.join(location
, arg
)
530 elif opt
in ("-i", "--interval"):
533 elif opt
in ("-c", "--cpu"):
535 elif opt
in ("-n", "--name"):
538 elif opt
in ("-m", "--memory"):
541 if not (valid1
and valid2
):
546 for p
in re
.split("[,]", cpu_list
):
547 if int(p
) < MAX_CPUS
:
550 for i
in range (0, MAX_CPUS
):
553 if not os
.path
.exists('results'):
555 # The regular user needs to own the directory, not root.
556 fix_ownership('results')
559 if os
.path
.exists(testname
):
560 print('The test name directory already exists. Please provide a unique test name. Test re-run not supported, yet.')
563 # The regular user needs to own the directory, not root.
564 fix_ownership(testname
)
567 # Temporary (or perhaps not)
568 cur_version
= sys
.version_info
569 print('python version (should be >= 2.7):')
572 # Left as "cleanup" for potential future re-run ability.
576 filename
= "/sys/kernel/debug/tracing/trace"
578 set_trace_buffer_size()
580 print('Sleeping for ', interval
, 'seconds')
581 time
.sleep(int(interval
))
586 read_trace_data(filename
)
593 if graph_data_present
== False:
594 print('No valid data to plot')
597 for cpu_no
in range(0, current_max_cpu
+ 1):
598 plot_perf_busy_with_sample(cpu_no
)
599 plot_perf_busy(cpu_no
)
600 plot_durations(cpu_no
)
603 plot_pstate_cpu_with_sample()
612 # It is preferrable, but not necessary, that the regular user owns the files, not root.
613 for root
, dirs
, files
in os
.walk('.'):