]>
Commit | Line | Data |
---|---|---|
2987e32f AH |
1 | # export-to-postgresql.py: export perf data to a postgresql database |
2 | # Copyright (c) 2014, Intel Corporation. | |
3 | # | |
4 | # This program is free software; you can redistribute it and/or modify it | |
5 | # under the terms and conditions of the GNU General Public License, | |
6 | # version 2, as published by the Free Software Foundation. | |
7 | # | |
8 | # This program is distributed in the hope it will be useful, but WITHOUT | |
9 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
10 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
11 | # more details. | |
12 | ||
13 | import os | |
14 | import sys | |
15 | import struct | |
16 | import datetime | |
17 | ||
4b715d24 AH |
18 | # To use this script you will need to have installed package python-pyside which |
19 | # provides LGPL-licensed Python bindings for Qt. You will also need the package | |
20 | # libqt4-sql-psql for Qt postgresql support. | |
21 | # | |
22 | # The script assumes postgresql is running on the local machine and that the | |
23 | # user has postgresql permissions to create databases. Examples of installing | |
24 | # postgresql and adding such a user are: | |
25 | # | |
26 | # fedora: | |
27 | # | |
28 | # $ sudo yum install postgresql postgresql-server python-pyside qt-postgresql | |
29 | # $ sudo su - postgres -c initdb | |
30 | # $ sudo service postgresql start | |
31 | # $ sudo su - postgres | |
32 | # $ createuser <your user id here> | |
33 | # Shall the new role be a superuser? (y/n) y | |
34 | # | |
35 | # ubuntu: | |
36 | # | |
d6632dd5 | 37 | # $ sudo apt-get install postgresql python-pyside.qtsql libqt4-sql-psql |
4b715d24 | 38 | # $ sudo su - postgres |
d6632dd5 | 39 | # $ createuser -s <your user id here> |
4b715d24 AH |
40 | # |
41 | # An example of using this script with Intel PT: | |
42 | # | |
43 | # $ perf record -e intel_pt//u ls | |
44 | # $ perf script -s ~/libexec/perf-core/scripts/python/export-to-postgresql.py pt_example branches calls | |
45 | # 2015-05-29 12:49:23.464364 Creating database... | |
46 | # 2015-05-29 12:49:26.281717 Writing to intermediate files... | |
47 | # 2015-05-29 12:49:27.190383 Copying to database... | |
48 | # 2015-05-29 12:49:28.140451 Removing intermediate files... | |
49 | # 2015-05-29 12:49:28.147451 Adding primary keys | |
50 | # 2015-05-29 12:49:28.655683 Adding foreign keys | |
51 | # 2015-05-29 12:49:29.365350 Done | |
52 | # | |
53 | # To browse the database, psql can be used e.g. | |
54 | # | |
55 | # $ psql pt_example | |
56 | # pt_example=# select * from samples_view where id < 100; | |
57 | # pt_example=# \d+ | |
58 | # pt_example=# \d+ samples_view | |
59 | # pt_example=# \q | |
60 | # | |
61 | # An example of using the database is provided by the script | |
69e6e410 | 62 | # call-graph-from-sql.py. Refer to that script for details. |
35ca01c1 AH |
63 | # |
64 | # Tables: | |
65 | # | |
66 | # The tables largely correspond to perf tools' data structures. They are largely self-explanatory. | |
67 | # | |
68 | # samples | |
69 | # | |
70 | # 'samples' is the main table. It represents what instruction was executing at a point in time | |
71 | # when something (a selected event) happened. The memory address is the instruction pointer or 'ip'. | |
72 | # | |
73 | # calls | |
74 | # | |
75 | # 'calls' represents function calls and is related to 'samples' by 'call_id' and 'return_id'. | |
76 | # 'calls' is only created when the 'calls' option to this script is specified. | |
77 | # | |
78 | # call_paths | |
79 | # | |
80 | # 'call_paths' represents all the call stacks. Each 'call' has an associated record in 'call_paths'. | |
81 | # 'calls_paths' is only created when the 'calls' option to this script is specified. | |
82 | # | |
83 | # branch_types | |
84 | # | |
85 | # 'branch_types' provides descriptions for each type of branch. | |
86 | # | |
87 | # comm_threads | |
88 | # | |
89 | # 'comm_threads' shows how 'comms' relates to 'threads'. | |
90 | # | |
91 | # comms | |
92 | # | |
93 | # 'comms' contains a record for each 'comm' - the name given to the executable that is running. | |
94 | # | |
95 | # dsos | |
96 | # | |
97 | # 'dsos' contains a record for each executable file or library. | |
98 | # | |
99 | # machines | |
100 | # | |
101 | # 'machines' can be used to distinguish virtual machines if virtualization is supported. | |
102 | # | |
103 | # selected_events | |
104 | # | |
105 | # 'selected_events' contains a record for each kind of event that has been sampled. | |
106 | # | |
107 | # symbols | |
108 | # | |
109 | # 'symbols' contains a record for each symbol. Only symbols that have samples are present. | |
110 | # | |
111 | # threads | |
112 | # | |
113 | # 'threads' contains a record for each thread. | |
114 | # | |
115 | # Views: | |
116 | # | |
117 | # Most of the tables have views for more friendly display. The views are: | |
118 | # | |
119 | # calls_view | |
120 | # call_paths_view | |
121 | # comm_threads_view | |
122 | # dsos_view | |
123 | # machines_view | |
124 | # samples_view | |
125 | # symbols_view | |
126 | # threads_view | |
127 | # | |
128 | # More examples of browsing the database with psql: | |
129 | # Note that some of the examples are not the most optimal SQL query. | |
130 | # Note that call information is only available if the script's 'calls' option has been used. | |
131 | # | |
132 | # Top 10 function calls (not aggregated by symbol): | |
133 | # | |
134 | # SELECT * FROM calls_view ORDER BY elapsed_time DESC LIMIT 10; | |
135 | # | |
136 | # Top 10 function calls (aggregated by symbol): | |
137 | # | |
138 | # SELECT symbol_id,(SELECT name FROM symbols WHERE id = symbol_id) AS symbol, | |
139 | # SUM(elapsed_time) AS tot_elapsed_time,SUM(branch_count) AS tot_branch_count | |
140 | # FROM calls_view GROUP BY symbol_id ORDER BY tot_elapsed_time DESC LIMIT 10; | |
141 | # | |
142 | # Note that the branch count gives a rough estimation of cpu usage, so functions | |
143 | # that took a long time but have a relatively low branch count must have spent time | |
144 | # waiting. | |
145 | # | |
146 | # Find symbols by pattern matching on part of the name (e.g. names containing 'alloc'): | |
147 | # | |
148 | # SELECT * FROM symbols_view WHERE name LIKE '%alloc%'; | |
149 | # | |
150 | # Top 10 function calls for a specific symbol (e.g. whose symbol_id is 187): | |
151 | # | |
152 | # SELECT * FROM calls_view WHERE symbol_id = 187 ORDER BY elapsed_time DESC LIMIT 10; | |
153 | # | |
154 | # Show function calls made by function in the same context (i.e. same call path) (e.g. one with call_path_id 254): | |
155 | # | |
156 | # SELECT * FROM calls_view WHERE parent_call_path_id = 254; | |
157 | # | |
158 | # Show branches made during a function call (e.g. where call_id is 29357 and return_id is 29370 and tid is 29670) | |
159 | # | |
160 | # SELECT * FROM samples_view WHERE id >= 29357 AND id <= 29370 AND tid = 29670 AND event LIKE 'branches%'; | |
161 | # | |
162 | # Show transactions: | |
163 | # | |
164 | # SELECT * FROM samples_view WHERE event = 'transactions'; | |
165 | # | |
166 | # Note transaction start has 'in_tx' true whereas, transaction end has 'in_tx' false. | |
167 | # Transaction aborts have branch_type_name 'transaction abort' | |
168 | # | |
169 | # Show transaction aborts: | |
170 | # | |
171 | # SELECT * FROM samples_view WHERE event = 'transactions' AND branch_type_name = 'transaction abort'; | |
172 | # | |
173 | # To print a call stack requires walking the call_paths table. For example this python script: | |
174 | # #!/usr/bin/python2 | |
175 | # | |
176 | # import sys | |
177 | # from PySide.QtSql import * | |
178 | # | |
179 | # if __name__ == '__main__': | |
180 | # if (len(sys.argv) < 3): | |
181 | # print >> sys.stderr, "Usage is: printcallstack.py <database name> <call_path_id>" | |
182 | # raise Exception("Too few arguments") | |
183 | # dbname = sys.argv[1] | |
184 | # call_path_id = sys.argv[2] | |
185 | # db = QSqlDatabase.addDatabase('QPSQL') | |
186 | # db.setDatabaseName(dbname) | |
187 | # if not db.open(): | |
188 | # raise Exception("Failed to open database " + dbname + " error: " + db.lastError().text()) | |
189 | # query = QSqlQuery(db) | |
190 | # print " id ip symbol_id symbol dso_id dso_short_name" | |
191 | # while call_path_id != 0 and call_path_id != 1: | |
192 | # ret = query.exec_('SELECT * FROM call_paths_view WHERE id = ' + str(call_path_id)) | |
193 | # if not ret: | |
194 | # raise Exception("Query failed: " + query.lastError().text()) | |
195 | # if not query.next(): | |
196 | # raise Exception("Query failed") | |
197 | # print "{0:>6} {1:>10} {2:>9} {3:<30} {4:>6} {5:<30}".format(query.value(0), query.value(1), query.value(2), query.value(3), query.value(4), query.value(5)) | |
198 | # call_path_id = query.value(6) | |
4b715d24 | 199 | |
2987e32f AH |
200 | from PySide.QtSql import * |
201 | ||
202 | # Need to access PostgreSQL C library directly to use COPY FROM STDIN | |
203 | from ctypes import * | |
204 | libpq = CDLL("libpq.so.5") | |
205 | PQconnectdb = libpq.PQconnectdb | |
206 | PQconnectdb.restype = c_void_p | |
82ac2740 | 207 | PQconnectdb.argtypes = [ c_char_p ] |
2987e32f | 208 | PQfinish = libpq.PQfinish |
82ac2740 | 209 | PQfinish.argtypes = [ c_void_p ] |
2987e32f | 210 | PQstatus = libpq.PQstatus |
82ac2740 AH |
211 | PQstatus.restype = c_int |
212 | PQstatus.argtypes = [ c_void_p ] | |
2987e32f AH |
213 | PQexec = libpq.PQexec |
214 | PQexec.restype = c_void_p | |
82ac2740 | 215 | PQexec.argtypes = [ c_void_p, c_char_p ] |
2987e32f | 216 | PQresultStatus = libpq.PQresultStatus |
82ac2740 AH |
217 | PQresultStatus.restype = c_int |
218 | PQresultStatus.argtypes = [ c_void_p ] | |
2987e32f | 219 | PQputCopyData = libpq.PQputCopyData |
82ac2740 | 220 | PQputCopyData.restype = c_int |
2987e32f AH |
221 | PQputCopyData.argtypes = [ c_void_p, c_void_p, c_int ] |
222 | PQputCopyEnd = libpq.PQputCopyEnd | |
82ac2740 | 223 | PQputCopyEnd.restype = c_int |
2987e32f AH |
224 | PQputCopyEnd.argtypes = [ c_void_p, c_void_p ] |
225 | ||
226 | sys.path.append(os.environ['PERF_EXEC_PATH'] + \ | |
227 | '/scripts/python/Perf-Trace-Util/lib/Perf/Trace') | |
228 | ||
229 | # These perf imports are not used at present | |
230 | #from perf_trace_context import * | |
231 | #from Core import * | |
232 | ||
233 | perf_db_export_mode = True | |
6a70307d | 234 | perf_db_export_calls = False |
3521f3bc CP |
235 | perf_db_export_callchains = False |
236 | ||
2987e32f AH |
237 | |
238 | def usage(): | |
3521f3bc | 239 | print >> sys.stderr, "Usage is: export-to-postgresql.py <database name> [<columns>] [<calls>] [<callchains>]" |
2987e32f | 240 | print >> sys.stderr, "where: columns 'all' or 'branches'" |
3521f3bc CP |
241 | print >> sys.stderr, " calls 'calls' => create calls and call_paths table" |
242 | print >> sys.stderr, " callchains 'callchains' => create call_paths table" | |
2987e32f AH |
243 | raise Exception("Too few arguments") |
244 | ||
245 | if (len(sys.argv) < 2): | |
246 | usage() | |
247 | ||
248 | dbname = sys.argv[1] | |
249 | ||
250 | if (len(sys.argv) >= 3): | |
251 | columns = sys.argv[2] | |
252 | else: | |
253 | columns = "all" | |
254 | ||
255 | if columns not in ("all", "branches"): | |
256 | usage() | |
257 | ||
258 | branches = (columns == "branches") | |
259 | ||
3521f3bc CP |
260 | for i in range(3,len(sys.argv)): |
261 | if (sys.argv[i] == "calls"): | |
6a70307d | 262 | perf_db_export_calls = True |
3521f3bc CP |
263 | elif (sys.argv[i] == "callchains"): |
264 | perf_db_export_callchains = True | |
6a70307d AH |
265 | else: |
266 | usage() | |
267 | ||
2987e32f AH |
268 | output_dir_name = os.getcwd() + "/" + dbname + "-perf-data" |
269 | os.mkdir(output_dir_name) | |
270 | ||
271 | def do_query(q, s): | |
272 | if (q.exec_(s)): | |
273 | return | |
274 | raise Exception("Query failed: " + q.lastError().text()) | |
275 | ||
276 | print datetime.datetime.today(), "Creating database..." | |
277 | ||
278 | db = QSqlDatabase.addDatabase('QPSQL') | |
279 | query = QSqlQuery(db) | |
280 | db.setDatabaseName('postgres') | |
281 | db.open() | |
282 | try: | |
283 | do_query(query, 'CREATE DATABASE ' + dbname) | |
284 | except: | |
285 | os.rmdir(output_dir_name) | |
286 | raise | |
287 | query.finish() | |
288 | query.clear() | |
289 | db.close() | |
290 | ||
291 | db.setDatabaseName(dbname) | |
292 | db.open() | |
293 | ||
294 | query = QSqlQuery(db) | |
295 | do_query(query, 'SET client_min_messages TO WARNING') | |
296 | ||
297 | do_query(query, 'CREATE TABLE selected_events (' | |
298 | 'id bigint NOT NULL,' | |
299 | 'name varchar(80))') | |
300 | do_query(query, 'CREATE TABLE machines (' | |
301 | 'id bigint NOT NULL,' | |
302 | 'pid integer,' | |
303 | 'root_dir varchar(4096))') | |
304 | do_query(query, 'CREATE TABLE threads (' | |
305 | 'id bigint NOT NULL,' | |
306 | 'machine_id bigint,' | |
307 | 'process_id bigint,' | |
308 | 'pid integer,' | |
309 | 'tid integer)') | |
310 | do_query(query, 'CREATE TABLE comms (' | |
311 | 'id bigint NOT NULL,' | |
312 | 'comm varchar(16))') | |
313 | do_query(query, 'CREATE TABLE comm_threads (' | |
314 | 'id bigint NOT NULL,' | |
315 | 'comm_id bigint,' | |
316 | 'thread_id bigint)') | |
317 | do_query(query, 'CREATE TABLE dsos (' | |
318 | 'id bigint NOT NULL,' | |
319 | 'machine_id bigint,' | |
320 | 'short_name varchar(256),' | |
321 | 'long_name varchar(4096),' | |
322 | 'build_id varchar(64))') | |
323 | do_query(query, 'CREATE TABLE symbols (' | |
324 | 'id bigint NOT NULL,' | |
325 | 'dso_id bigint,' | |
326 | 'sym_start bigint,' | |
327 | 'sym_end bigint,' | |
328 | 'binding integer,' | |
329 | 'name varchar(2048))') | |
c29414f5 AH |
330 | do_query(query, 'CREATE TABLE branch_types (' |
331 | 'id integer NOT NULL,' | |
332 | 'name varchar(80))') | |
333 | ||
2987e32f AH |
334 | if branches: |
335 | do_query(query, 'CREATE TABLE samples (' | |
336 | 'id bigint NOT NULL,' | |
337 | 'evsel_id bigint,' | |
338 | 'machine_id bigint,' | |
339 | 'thread_id bigint,' | |
340 | 'comm_id bigint,' | |
341 | 'dso_id bigint,' | |
342 | 'symbol_id bigint,' | |
343 | 'sym_offset bigint,' | |
344 | 'ip bigint,' | |
345 | 'time bigint,' | |
346 | 'cpu integer,' | |
347 | 'to_dso_id bigint,' | |
348 | 'to_symbol_id bigint,' | |
349 | 'to_sym_offset bigint,' | |
c29414f5 AH |
350 | 'to_ip bigint,' |
351 | 'branch_type integer,' | |
c8a82728 AH |
352 | 'in_tx boolean,' |
353 | 'call_path_id bigint)') | |
2987e32f AH |
354 | else: |
355 | do_query(query, 'CREATE TABLE samples (' | |
356 | 'id bigint NOT NULL,' | |
357 | 'evsel_id bigint,' | |
358 | 'machine_id bigint,' | |
359 | 'thread_id bigint,' | |
360 | 'comm_id bigint,' | |
361 | 'dso_id bigint,' | |
362 | 'symbol_id bigint,' | |
363 | 'sym_offset bigint,' | |
364 | 'ip bigint,' | |
365 | 'time bigint,' | |
366 | 'cpu integer,' | |
367 | 'to_dso_id bigint,' | |
368 | 'to_symbol_id bigint,' | |
369 | 'to_sym_offset bigint,' | |
370 | 'to_ip bigint,' | |
371 | 'period bigint,' | |
372 | 'weight bigint,' | |
373 | 'transaction bigint,' | |
c29414f5 AH |
374 | 'data_src bigint,' |
375 | 'branch_type integer,' | |
3521f3bc CP |
376 | 'in_tx boolean,' |
377 | 'call_path_id bigint)') | |
2987e32f | 378 | |
3521f3bc | 379 | if perf_db_export_calls or perf_db_export_callchains: |
6a70307d AH |
380 | do_query(query, 'CREATE TABLE call_paths (' |
381 | 'id bigint NOT NULL,' | |
382 | 'parent_id bigint,' | |
383 | 'symbol_id bigint,' | |
384 | 'ip bigint)') | |
3521f3bc | 385 | if perf_db_export_calls: |
6a70307d AH |
386 | do_query(query, 'CREATE TABLE calls (' |
387 | 'id bigint NOT NULL,' | |
388 | 'thread_id bigint,' | |
389 | 'comm_id bigint,' | |
390 | 'call_path_id bigint,' | |
391 | 'call_time bigint,' | |
392 | 'return_time bigint,' | |
393 | 'branch_count bigint,' | |
394 | 'call_id bigint,' | |
395 | 'return_id bigint,' | |
396 | 'parent_call_path_id bigint,' | |
397 | 'flags integer)') | |
398 | ||
35ca01c1 AH |
399 | do_query(query, 'CREATE VIEW machines_view AS ' |
400 | 'SELECT ' | |
401 | 'id,' | |
402 | 'pid,' | |
403 | 'root_dir,' | |
404 | 'CASE WHEN id=0 THEN \'unknown\' WHEN pid=-1 THEN \'host\' ELSE \'guest\' END AS host_or_guest' | |
405 | ' FROM machines') | |
406 | ||
407 | do_query(query, 'CREATE VIEW dsos_view AS ' | |
408 | 'SELECT ' | |
409 | 'id,' | |
410 | 'machine_id,' | |
411 | '(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest,' | |
412 | 'short_name,' | |
413 | 'long_name,' | |
414 | 'build_id' | |
415 | ' FROM dsos') | |
416 | ||
417 | do_query(query, 'CREATE VIEW symbols_view AS ' | |
418 | 'SELECT ' | |
419 | 'id,' | |
420 | 'name,' | |
421 | '(SELECT short_name FROM dsos WHERE id=dso_id) AS dso,' | |
422 | 'dso_id,' | |
423 | 'sym_start,' | |
424 | 'sym_end,' | |
425 | 'CASE WHEN binding=0 THEN \'local\' WHEN binding=1 THEN \'global\' ELSE \'weak\' END AS binding' | |
426 | ' FROM symbols') | |
427 | ||
428 | do_query(query, 'CREATE VIEW threads_view AS ' | |
429 | 'SELECT ' | |
430 | 'id,' | |
431 | 'machine_id,' | |
432 | '(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest,' | |
433 | 'process_id,' | |
434 | 'pid,' | |
435 | 'tid' | |
436 | ' FROM threads') | |
437 | ||
438 | do_query(query, 'CREATE VIEW comm_threads_view AS ' | |
439 | 'SELECT ' | |
440 | 'comm_id,' | |
441 | '(SELECT comm FROM comms WHERE id = comm_id) AS command,' | |
442 | 'thread_id,' | |
443 | '(SELECT pid FROM threads WHERE id = thread_id) AS pid,' | |
444 | '(SELECT tid FROM threads WHERE id = thread_id) AS tid' | |
445 | ' FROM comm_threads') | |
446 | ||
3521f3bc | 447 | if perf_db_export_calls or perf_db_export_callchains: |
35ca01c1 AH |
448 | do_query(query, 'CREATE VIEW call_paths_view AS ' |
449 | 'SELECT ' | |
450 | 'c.id,' | |
451 | 'to_hex(c.ip) AS ip,' | |
452 | 'c.symbol_id,' | |
453 | '(SELECT name FROM symbols WHERE id = c.symbol_id) AS symbol,' | |
454 | '(SELECT dso_id FROM symbols WHERE id = c.symbol_id) AS dso_id,' | |
455 | '(SELECT dso FROM symbols_view WHERE id = c.symbol_id) AS dso_short_name,' | |
456 | 'c.parent_id,' | |
457 | 'to_hex(p.ip) AS parent_ip,' | |
458 | 'p.symbol_id AS parent_symbol_id,' | |
459 | '(SELECT name FROM symbols WHERE id = p.symbol_id) AS parent_symbol,' | |
460 | '(SELECT dso_id FROM symbols WHERE id = p.symbol_id) AS parent_dso_id,' | |
461 | '(SELECT dso FROM symbols_view WHERE id = p.symbol_id) AS parent_dso_short_name' | |
462 | ' FROM call_paths c INNER JOIN call_paths p ON p.id = c.parent_id') | |
3521f3bc | 463 | if perf_db_export_calls: |
35ca01c1 AH |
464 | do_query(query, 'CREATE VIEW calls_view AS ' |
465 | 'SELECT ' | |
466 | 'calls.id,' | |
467 | 'thread_id,' | |
468 | '(SELECT pid FROM threads WHERE id = thread_id) AS pid,' | |
469 | '(SELECT tid FROM threads WHERE id = thread_id) AS tid,' | |
470 | '(SELECT comm FROM comms WHERE id = comm_id) AS command,' | |
471 | 'call_path_id,' | |
472 | 'to_hex(ip) AS ip,' | |
473 | 'symbol_id,' | |
474 | '(SELECT name FROM symbols WHERE id = symbol_id) AS symbol,' | |
475 | 'call_time,' | |
476 | 'return_time,' | |
477 | 'return_time - call_time AS elapsed_time,' | |
478 | 'branch_count,' | |
479 | 'call_id,' | |
480 | 'return_id,' | |
481 | 'CASE WHEN flags=1 THEN \'no call\' WHEN flags=2 THEN \'no return\' WHEN flags=3 THEN \'no call/return\' ELSE \'\' END AS flags,' | |
482 | 'parent_call_path_id' | |
483 | ' FROM calls INNER JOIN call_paths ON call_paths.id = call_path_id') | |
484 | ||
2987e32f AH |
485 | do_query(query, 'CREATE VIEW samples_view AS ' |
486 | 'SELECT ' | |
487 | 'id,' | |
488 | 'time,' | |
489 | 'cpu,' | |
490 | '(SELECT pid FROM threads WHERE id = thread_id) AS pid,' | |
491 | '(SELECT tid FROM threads WHERE id = thread_id) AS tid,' | |
492 | '(SELECT comm FROM comms WHERE id = comm_id) AS command,' | |
493 | '(SELECT name FROM selected_events WHERE id = evsel_id) AS event,' | |
494 | 'to_hex(ip) AS ip_hex,' | |
495 | '(SELECT name FROM symbols WHERE id = symbol_id) AS symbol,' | |
496 | 'sym_offset,' | |
497 | '(SELECT short_name FROM dsos WHERE id = dso_id) AS dso_short_name,' | |
498 | 'to_hex(to_ip) AS to_ip_hex,' | |
499 | '(SELECT name FROM symbols WHERE id = to_symbol_id) AS to_symbol,' | |
500 | 'to_sym_offset,' | |
c29414f5 AH |
501 | '(SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name,' |
502 | '(SELECT name FROM branch_types WHERE id = branch_type) AS branch_type_name,' | |
503 | 'in_tx' | |
2987e32f AH |
504 | ' FROM samples') |
505 | ||
506 | ||
507 | file_header = struct.pack("!11sii", "PGCOPY\n\377\r\n\0", 0, 0) | |
508 | file_trailer = "\377\377" | |
509 | ||
510 | def open_output_file(file_name): | |
511 | path_name = output_dir_name + "/" + file_name | |
512 | file = open(path_name, "w+") | |
513 | file.write(file_header) | |
514 | return file | |
515 | ||
516 | def close_output_file(file): | |
517 | file.write(file_trailer) | |
518 | file.close() | |
519 | ||
520 | def copy_output_file_direct(file, table_name): | |
521 | close_output_file(file) | |
522 | sql = "COPY " + table_name + " FROM '" + file.name + "' (FORMAT 'binary')" | |
523 | do_query(query, sql) | |
524 | ||
525 | # Use COPY FROM STDIN because security may prevent postgres from accessing the files directly | |
526 | def copy_output_file(file, table_name): | |
527 | conn = PQconnectdb("dbname = " + dbname) | |
528 | if (PQstatus(conn)): | |
529 | raise Exception("COPY FROM STDIN PQconnectdb failed") | |
530 | file.write(file_trailer) | |
531 | file.seek(0) | |
532 | sql = "COPY " + table_name + " FROM STDIN (FORMAT 'binary')" | |
533 | res = PQexec(conn, sql) | |
534 | if (PQresultStatus(res) != 4): | |
535 | raise Exception("COPY FROM STDIN PQexec failed") | |
536 | data = file.read(65536) | |
537 | while (len(data)): | |
538 | ret = PQputCopyData(conn, data, len(data)) | |
539 | if (ret != 1): | |
540 | raise Exception("COPY FROM STDIN PQputCopyData failed, error " + str(ret)) | |
541 | data = file.read(65536) | |
542 | ret = PQputCopyEnd(conn, None) | |
543 | if (ret != 1): | |
544 | raise Exception("COPY FROM STDIN PQputCopyEnd failed, error " + str(ret)) | |
545 | PQfinish(conn) | |
546 | ||
547 | def remove_output_file(file): | |
548 | name = file.name | |
549 | file.close() | |
550 | os.unlink(name) | |
551 | ||
552 | evsel_file = open_output_file("evsel_table.bin") | |
553 | machine_file = open_output_file("machine_table.bin") | |
554 | thread_file = open_output_file("thread_table.bin") | |
555 | comm_file = open_output_file("comm_table.bin") | |
556 | comm_thread_file = open_output_file("comm_thread_table.bin") | |
557 | dso_file = open_output_file("dso_table.bin") | |
558 | symbol_file = open_output_file("symbol_table.bin") | |
c29414f5 | 559 | branch_type_file = open_output_file("branch_type_table.bin") |
2987e32f | 560 | sample_file = open_output_file("sample_table.bin") |
3521f3bc | 561 | if perf_db_export_calls or perf_db_export_callchains: |
6a70307d | 562 | call_path_file = open_output_file("call_path_table.bin") |
3521f3bc | 563 | if perf_db_export_calls: |
6a70307d | 564 | call_file = open_output_file("call_table.bin") |
2987e32f AH |
565 | |
566 | def trace_begin(): | |
567 | print datetime.datetime.today(), "Writing to intermediate files..." | |
568 | # id == 0 means unknown. It is easier to create records for them than replace the zeroes with NULLs | |
569 | evsel_table(0, "unknown") | |
570 | machine_table(0, 0, "unknown") | |
571 | thread_table(0, 0, 0, -1, -1) | |
572 | comm_table(0, "unknown") | |
573 | dso_table(0, 0, "unknown", "unknown", "") | |
574 | symbol_table(0, 0, 0, 0, 0, "unknown") | |
3521f3bc CP |
575 | sample_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) |
576 | if perf_db_export_calls or perf_db_export_callchains: | |
6a70307d | 577 | call_path_table(0, 0, 0, 0) |
2987e32f AH |
578 | |
579 | unhandled_count = 0 | |
580 | ||
581 | def trace_end(): | |
582 | print datetime.datetime.today(), "Copying to database..." | |
583 | copy_output_file(evsel_file, "selected_events") | |
584 | copy_output_file(machine_file, "machines") | |
585 | copy_output_file(thread_file, "threads") | |
586 | copy_output_file(comm_file, "comms") | |
587 | copy_output_file(comm_thread_file, "comm_threads") | |
588 | copy_output_file(dso_file, "dsos") | |
589 | copy_output_file(symbol_file, "symbols") | |
c29414f5 | 590 | copy_output_file(branch_type_file, "branch_types") |
2987e32f | 591 | copy_output_file(sample_file, "samples") |
3521f3bc | 592 | if perf_db_export_calls or perf_db_export_callchains: |
6a70307d | 593 | copy_output_file(call_path_file, "call_paths") |
3521f3bc | 594 | if perf_db_export_calls: |
6a70307d | 595 | copy_output_file(call_file, "calls") |
2987e32f AH |
596 | |
597 | print datetime.datetime.today(), "Removing intermediate files..." | |
598 | remove_output_file(evsel_file) | |
599 | remove_output_file(machine_file) | |
600 | remove_output_file(thread_file) | |
601 | remove_output_file(comm_file) | |
602 | remove_output_file(comm_thread_file) | |
603 | remove_output_file(dso_file) | |
604 | remove_output_file(symbol_file) | |
c29414f5 | 605 | remove_output_file(branch_type_file) |
2987e32f | 606 | remove_output_file(sample_file) |
3521f3bc | 607 | if perf_db_export_calls or perf_db_export_callchains: |
6a70307d | 608 | remove_output_file(call_path_file) |
3521f3bc | 609 | if perf_db_export_calls: |
6a70307d | 610 | remove_output_file(call_file) |
2987e32f AH |
611 | os.rmdir(output_dir_name) |
612 | print datetime.datetime.today(), "Adding primary keys" | |
613 | do_query(query, 'ALTER TABLE selected_events ADD PRIMARY KEY (id)') | |
614 | do_query(query, 'ALTER TABLE machines ADD PRIMARY KEY (id)') | |
615 | do_query(query, 'ALTER TABLE threads ADD PRIMARY KEY (id)') | |
616 | do_query(query, 'ALTER TABLE comms ADD PRIMARY KEY (id)') | |
617 | do_query(query, 'ALTER TABLE comm_threads ADD PRIMARY KEY (id)') | |
618 | do_query(query, 'ALTER TABLE dsos ADD PRIMARY KEY (id)') | |
619 | do_query(query, 'ALTER TABLE symbols ADD PRIMARY KEY (id)') | |
c29414f5 | 620 | do_query(query, 'ALTER TABLE branch_types ADD PRIMARY KEY (id)') |
2987e32f | 621 | do_query(query, 'ALTER TABLE samples ADD PRIMARY KEY (id)') |
3521f3bc | 622 | if perf_db_export_calls or perf_db_export_callchains: |
6a70307d | 623 | do_query(query, 'ALTER TABLE call_paths ADD PRIMARY KEY (id)') |
3521f3bc | 624 | if perf_db_export_calls: |
6a70307d | 625 | do_query(query, 'ALTER TABLE calls ADD PRIMARY KEY (id)') |
2987e32f AH |
626 | |
627 | print datetime.datetime.today(), "Adding foreign keys" | |
628 | do_query(query, 'ALTER TABLE threads ' | |
629 | 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id),' | |
630 | 'ADD CONSTRAINT processfk FOREIGN KEY (process_id) REFERENCES threads (id)') | |
631 | do_query(query, 'ALTER TABLE comm_threads ' | |
632 | 'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),' | |
633 | 'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id)') | |
634 | do_query(query, 'ALTER TABLE dsos ' | |
635 | 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id)') | |
636 | do_query(query, 'ALTER TABLE symbols ' | |
637 | 'ADD CONSTRAINT dsofk FOREIGN KEY (dso_id) REFERENCES dsos (id)') | |
638 | do_query(query, 'ALTER TABLE samples ' | |
639 | 'ADD CONSTRAINT evselfk FOREIGN KEY (evsel_id) REFERENCES selected_events (id),' | |
640 | 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id),' | |
641 | 'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id),' | |
642 | 'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),' | |
643 | 'ADD CONSTRAINT dsofk FOREIGN KEY (dso_id) REFERENCES dsos (id),' | |
644 | 'ADD CONSTRAINT symbolfk FOREIGN KEY (symbol_id) REFERENCES symbols (id),' | |
645 | 'ADD CONSTRAINT todsofk FOREIGN KEY (to_dso_id) REFERENCES dsos (id),' | |
646 | 'ADD CONSTRAINT tosymbolfk FOREIGN KEY (to_symbol_id) REFERENCES symbols (id)') | |
3521f3bc | 647 | if perf_db_export_calls or perf_db_export_callchains: |
6a70307d AH |
648 | do_query(query, 'ALTER TABLE call_paths ' |
649 | 'ADD CONSTRAINT parentfk FOREIGN KEY (parent_id) REFERENCES call_paths (id),' | |
650 | 'ADD CONSTRAINT symbolfk FOREIGN KEY (symbol_id) REFERENCES symbols (id)') | |
3521f3bc | 651 | if perf_db_export_calls: |
6a70307d AH |
652 | do_query(query, 'ALTER TABLE calls ' |
653 | 'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id),' | |
654 | 'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),' | |
655 | 'ADD CONSTRAINT call_pathfk FOREIGN KEY (call_path_id) REFERENCES call_paths (id),' | |
656 | 'ADD CONSTRAINT callfk FOREIGN KEY (call_id) REFERENCES samples (id),' | |
657 | 'ADD CONSTRAINT returnfk FOREIGN KEY (return_id) REFERENCES samples (id),' | |
658 | 'ADD CONSTRAINT parent_call_pathfk FOREIGN KEY (parent_call_path_id) REFERENCES call_paths (id)') | |
659 | do_query(query, 'CREATE INDEX pcpid_idx ON calls (parent_call_path_id)') | |
2987e32f AH |
660 | |
661 | if (unhandled_count): | |
662 | print datetime.datetime.today(), "Warning: ", unhandled_count, " unhandled events" | |
663 | print datetime.datetime.today(), "Done" | |
664 | ||
665 | def trace_unhandled(event_name, context, event_fields_dict): | |
666 | global unhandled_count | |
667 | unhandled_count += 1 | |
668 | ||
669 | def sched__sched_switch(*x): | |
670 | pass | |
671 | ||
672 | def evsel_table(evsel_id, evsel_name, *x): | |
673 | n = len(evsel_name) | |
674 | fmt = "!hiqi" + str(n) + "s" | |
675 | value = struct.pack(fmt, 2, 8, evsel_id, n, evsel_name) | |
676 | evsel_file.write(value) | |
677 | ||
678 | def machine_table(machine_id, pid, root_dir, *x): | |
679 | n = len(root_dir) | |
680 | fmt = "!hiqiii" + str(n) + "s" | |
681 | value = struct.pack(fmt, 3, 8, machine_id, 4, pid, n, root_dir) | |
682 | machine_file.write(value) | |
683 | ||
684 | def thread_table(thread_id, machine_id, process_id, pid, tid, *x): | |
685 | value = struct.pack("!hiqiqiqiiii", 5, 8, thread_id, 8, machine_id, 8, process_id, 4, pid, 4, tid) | |
686 | thread_file.write(value) | |
687 | ||
688 | def comm_table(comm_id, comm_str, *x): | |
689 | n = len(comm_str) | |
690 | fmt = "!hiqi" + str(n) + "s" | |
691 | value = struct.pack(fmt, 2, 8, comm_id, n, comm_str) | |
692 | comm_file.write(value) | |
693 | ||
694 | def comm_thread_table(comm_thread_id, comm_id, thread_id, *x): | |
695 | fmt = "!hiqiqiq" | |
696 | value = struct.pack(fmt, 3, 8, comm_thread_id, 8, comm_id, 8, thread_id) | |
697 | comm_thread_file.write(value) | |
698 | ||
699 | def dso_table(dso_id, machine_id, short_name, long_name, build_id, *x): | |
700 | n1 = len(short_name) | |
701 | n2 = len(long_name) | |
702 | n3 = len(build_id) | |
703 | fmt = "!hiqiqi" + str(n1) + "si" + str(n2) + "si" + str(n3) + "s" | |
704 | value = struct.pack(fmt, 5, 8, dso_id, 8, machine_id, n1, short_name, n2, long_name, n3, build_id) | |
705 | dso_file.write(value) | |
706 | ||
707 | def symbol_table(symbol_id, dso_id, sym_start, sym_end, binding, symbol_name, *x): | |
708 | n = len(symbol_name) | |
709 | fmt = "!hiqiqiqiqiii" + str(n) + "s" | |
710 | value = struct.pack(fmt, 6, 8, symbol_id, 8, dso_id, 8, sym_start, 8, sym_end, 4, binding, n, symbol_name) | |
711 | symbol_file.write(value) | |
712 | ||
c29414f5 AH |
713 | def branch_type_table(branch_type, name, *x): |
714 | n = len(name) | |
715 | fmt = "!hiii" + str(n) + "s" | |
716 | value = struct.pack(fmt, 2, 4, branch_type, n, name) | |
717 | branch_type_file.write(value) | |
718 | ||
3521f3bc | 719 | def sample_table(sample_id, evsel_id, machine_id, thread_id, comm_id, dso_id, symbol_id, sym_offset, ip, time, cpu, to_dso_id, to_symbol_id, to_sym_offset, to_ip, period, weight, transaction, data_src, branch_type, in_tx, call_path_id, *x): |
2987e32f | 720 | if branches: |
3521f3bc | 721 | value = struct.pack("!hiqiqiqiqiqiqiqiqiqiqiiiqiqiqiqiiiBiq", 18, 8, sample_id, 8, evsel_id, 8, machine_id, 8, thread_id, 8, comm_id, 8, dso_id, 8, symbol_id, 8, sym_offset, 8, ip, 8, time, 4, cpu, 8, to_dso_id, 8, to_symbol_id, 8, to_sym_offset, 8, to_ip, 4, branch_type, 1, in_tx, 8, call_path_id) |
2987e32f | 722 | else: |
3521f3bc | 723 | value = struct.pack("!hiqiqiqiqiqiqiqiqiqiqiiiqiqiqiqiqiqiqiqiiiBiq", 22, 8, sample_id, 8, evsel_id, 8, machine_id, 8, thread_id, 8, comm_id, 8, dso_id, 8, symbol_id, 8, sym_offset, 8, ip, 8, time, 4, cpu, 8, to_dso_id, 8, to_symbol_id, 8, to_sym_offset, 8, to_ip, 8, period, 8, weight, 8, transaction, 8, data_src, 4, branch_type, 1, in_tx, 8, call_path_id) |
2987e32f | 724 | sample_file.write(value) |
6a70307d AH |
725 | |
726 | def call_path_table(cp_id, parent_id, symbol_id, ip, *x): | |
727 | fmt = "!hiqiqiqiq" | |
728 | value = struct.pack(fmt, 4, 8, cp_id, 8, parent_id, 8, symbol_id, 8, ip) | |
729 | call_path_file.write(value) | |
730 | ||
731 | def call_return_table(cr_id, thread_id, comm_id, call_path_id, call_time, return_time, branch_count, call_id, return_id, parent_call_path_id, flags, *x): | |
732 | fmt = "!hiqiqiqiqiqiqiqiqiqiqii" | |
733 | value = struct.pack(fmt, 11, 8, cr_id, 8, thread_id, 8, comm_id, 8, call_path_id, 8, call_time, 8, return_time, 8, branch_count, 8, call_id, 8, return_id, 8, parent_call_path_id, 4, flags) | |
734 | call_file.write(value) |