docs(perf): profile CUDA graph coverage and PLE costs on Spark
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
{"label": "graph", "time": 1789664773.0911076, "event": "REQUEST_START", "name": "warmup", "profile": false, "prompt_tokens_preflight": 51}
|
||||
{"label": "graph", "time": 1789664774.6022928, "event": "RESULT", "name": "warmup", "profile": false, "correct": true, "content": "\n\n323", "finish_reason": "stop", "usage": {"prompt_tokens": 51, "total_tokens": 83, "completion_tokens": 32, "completion_tokens_details": {"reasoning_tokens": 26}}, "max_tokens": 2048, "prompt_sha256": "4181e1a44a0bed10d00c74e822377bd75a6c6c9a1ea458f5170b8193e9fce959", "output_sha256": "4477e82dcd9f3f8da1dd56be073d72e4ba20121d6ce6fbc97ee54258fb4b2959", "ttft_s": 0.6364480190022732, "first_content_s": 1.4175457309975172, "elapsed_s": 1.5047177029991872, "decode_tps_approx": 39.6877362766987, "metrics_delta": {"vllm:prefix_cache_queries_total": 51.0, "vllm:prefix_cache_hits_total": 0.0}}
|
||||
{"label": "graph", "time": 1789664774.614013, "event": "REQUEST_START", "name": "short-0", "profile": false, "prompt_tokens_preflight": 65}
|
||||
{"label": "graph", "time": 1789664781.980255, "event": "RESULT", "name": "short-0", "profile": false, "correct": true, "content": "\n\n数据库索引类似于书籍的目录，它通过B+树等数据结构将数据按特定字段有序组织，使查询时能快速定位目标记录，避免全表扫描，从而大幅提升读取效率。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库都需要同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略，需根据实际读写比例合理设计。", "finish_reason": "stop", "usage": {"prompt_tokens": 65, "total_tokens": 279, "completion_tokens": 214, "completion_tokens_details": {"reasoning_tokens": 111}}, "max_tokens": 512, "prompt_sha256": "40e7d1df3e96ad760d661cfaff312f5d732cb2edd5eab555b4b5c43cea4512ca", "output_sha256": "126232af3bc1889efc9c0dad67f2d47fc60d3c4ff56e699a83f1576c3763c274", "ttft_s": 0.32573997599683935, "first_content_s": 3.8834300599992275, "elapsed_s": 7.360314821999054, "decode_tps_approx": 30.279974628892248, "metrics_delta": {"vllm:prefix_cache_queries_total": 65.0, "vllm:prefix_cache_hits_total": 0.0}}
|
||||
{"label": "graph", "time": 1789664781.9975877, "event": "REQUEST_START", "name": "prefill-8192-0", "profile": false, "prompt_tokens_preflight": 8173}
|
||||
{"label": "graph", "time": 1789664788.1946163, "event": "RESULT", "name": "prefill-8192-0", "profile": false, "correct": true, "content": "\n\n21573", "finish_reason": "stop", "usage": {"prompt_tokens": 8173, "total_tokens": 8247, "completion_tokens": 74, "completion_tokens_details": {"reasoning_tokens": 66}}, "max_tokens": 2048, "prompt_sha256": "f59f163e1c7bb8a79d08df830d94d6d2e98fec457e9de75e3c09f6fca7ab0813", "output_sha256": "c0d6109d7f5d6c6f5486e269f63ea0338562a90ab0ceeede5f228b020e701eea", "ttft_s": 3.921214998998039, "first_content_s": 6.03858627199952, "elapsed_s": 6.191493427999376, "decode_tps_approx": 33.338899955373805, "metrics_delta": {"vllm:prefix_cache_queries_total": 8173.0, "vllm:prefix_cache_hits_total": 0.0}}
|
||||
{"label": "graph", "time": 1789664788.2102215, "event": "REQUEST_START", "name": "reuse-8192-0", "profile": false, "prompt_tokens_preflight": 8173}
|
||||
{"label": "graph", "time": 1789664791.047786, "event": "RESULT", "name": "reuse-8192-0", "profile": false, "correct": true, "content": "\n\n21573", "finish_reason": "stop", "usage": {"prompt_tokens": 8173, "total_tokens": 8238, "completion_tokens": 65, "completion_tokens_details": {"reasoning_tokens": 57}}, "max_tokens": 2048, "prompt_sha256": "f59f163e1c7bb8a79d08df830d94d6d2e98fec457e9de75e3c09f6fca7ab0813", "output_sha256": "50530d055152bcaa11d1779a66b456a8ad5175a5343fb48f268c163f397d0089", "ttft_s": 0.8737782960015466, "first_content_s": 2.671151395996276, "elapsed_s": 2.8319971849996364, "decode_tps_approx": 34.055335354489465, "metrics_delta": {"vllm:prefix_cache_queries_total": 8173.0, "vllm:prefix_cache_hits_total": 6400.0}}
|
||||
{"label": "graph", "time": 1789664791.096218, "event": "REQUEST_START", "name": "prefill-32768-0", "profile": false, "prompt_tokens_preflight": 32745}
|
||||
{"label": "graph", "time": 1789664808.7936926, "event": "RESULT", "name": "prefill-32768-0", "profile": false, "correct": true, "content": "\n\n90812", "finish_reason": "stop", "usage": {"prompt_tokens": 32745, "total_tokens": 32815, "completion_tokens": 70, "completion_tokens_details": {"reasoning_tokens": 62}}, "max_tokens": 2048, "prompt_sha256": "605f7b492252f50eaf60e8d128dde3f2d2e1b50dd26f0b45db88dac00e36a5ca", "output_sha256": "7eec57291da7ee95ae6d9214f8ed940a405e64beb9f49a4693387362aac3e14c", "ttft_s": 15.511759164000978, "first_content_s": 17.52370634300314, "elapsed_s": 17.691939716001798, "decode_tps_approx": 32.87688689931954, "metrics_delta": {"vllm:prefix_cache_queries_total": 32745.0, "vllm:prefix_cache_hits_total": 0.0}}
|
||||
{"label": "graph", "time": 1789664808.8368387, "event": "REQUEST_START", "name": "reuse-32768-0", "profile": false, "prompt_tokens_preflight": 32745}
|
||||
{"label": "graph", "time": 1789664812.1356192, "event": "RESULT", "name": "reuse-32768-0", "profile": false, "correct": true, "content": "\n\n90812", "finish_reason": "stop", "usage": {"prompt_tokens": 32745, "total_tokens": 32814, "completion_tokens": 69, "completion_tokens_details": {"reasoning_tokens": 61}}, "max_tokens": 2048, "prompt_sha256": "605f7b492252f50eaf60e8d128dde3f2d2e1b50dd26f0b45db88dac00e36a5ca", "output_sha256": "8b503330d6b70b8845bab80540601fc48c8f840d54d07f38d80250c8167de89f", "ttft_s": 1.325252010996337, "first_content_s": 3.1243166819986072, "elapsed_s": 3.293290553992847, "decode_tps_approx": 36.2184980346298, "metrics_delta": {"vllm:prefix_cache_queries_total": 32745.0, "vllm:prefix_cache_hits_total": 30400.0}}
|
||||
{"label": "graph", "time": 1789664812.1429493, "event": "REQUEST_START", "name": "short-1", "profile": false, "prompt_tokens_preflight": 65}
|
||||
{"label": "graph", "time": 1789664819.2091334, "event": "RESULT", "name": "short-1", "profile": false, "correct": true, "content": "\n\n数据库索引类似于书籍的目录，它通过B+树等数据结构将数据按特定字段有序组织，使查询时能快速定位目标记录，避免全表扫描，从而大幅提升读取效率。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库都需要同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略，需根据实际读写比例合理设计。", "finish_reason": "stop", "usage": {"prompt_tokens": 65, "total_tokens": 279, "completion_tokens": 214, "completion_tokens_details": {"reasoning_tokens": 111}}, "max_tokens": 512, "prompt_sha256": "40e7d1df3e96ad760d661cfaff312f5d732cb2edd5eab555b4b5c43cea4512ca", "output_sha256": "126232af3bc1889efc9c0dad67f2d47fc60d3c4ff56e699a83f1576c3763c274", "ttft_s": 0.2600282569983392, "first_content_s": 3.7012813049950637, "elapsed_s": 7.060739258995454, "decode_tps_approx": 31.32107217692348, "metrics_delta": {"vllm:prefix_cache_queries_total": 65.0, "vllm:prefix_cache_hits_total": 0.0}}
|
||||
{"label": "graph", "time": 1789664819.2247505, "event": "REQUEST_START", "name": "prefill-8192-1", "profile": false, "prompt_tokens_preflight": 8173}
|
||||
{"label": "graph", "time": 1789664824.82708, "event": "RESULT", "name": "prefill-8192-1", "profile": false, "correct": true, "content": "\n\n21573", "finish_reason": "stop", "usage": {"prompt_tokens": 8173, "total_tokens": 8236, "completion_tokens": 63, "completion_tokens_details": {"reasoning_tokens": 55}}, "max_tokens": 2048, "prompt_sha256": "6c13deb2d4ddee54db95d7ebc8a1b87d24ed8a4112c33644c18b6b5d61ad7a16", "output_sha256": "09d17d110b06ec13d532382c41ea29955b7edfa8dd4bd34163a08fa746e56610", "ttft_s": 3.7701075279983343, "first_content_s": 5.436177673000202, "elapsed_s": 5.596617848997994, "decode_tps_approx": 35.516432222041566, "metrics_delta": {"vllm:prefix_cache_queries_total": 8173.0, "vllm:prefix_cache_hits_total": 0.0}}
|
||||
{"label": "graph", "time": 1789664824.8427408, "event": "REQUEST_START", "name": "reuse-8192-1", "profile": false, "prompt_tokens_preflight": 8173}
|
||||
{"label": "graph", "time": 1789664827.8556995, "event": "RESULT", "name": "reuse-8192-1", "profile": false, "correct": true, "content": "\n\n21573", "finish_reason": "stop", "usage": {"prompt_tokens": 8173, "total_tokens": 8247, "completion_tokens": 74, "completion_tokens_details": {"reasoning_tokens": 66}}, "max_tokens": 2048, "prompt_sha256": "6c13deb2d4ddee54db95d7ebc8a1b87d24ed8a4112c33644c18b6b5d61ad7a16", "output_sha256": "c0d6109d7f5d6c6f5486e269f63ea0338562a90ab0ceeede5f228b020e701eea", "ttft_s": 0.8690901810041396, "first_content_s": 2.8428866180038312, "elapsed_s": 3.0074559670028975, "decode_tps_approx": 35.510793077788634, "metrics_delta": {"vllm:prefix_cache_queries_total": 8173.0, "vllm:prefix_cache_hits_total": 6400.0}}
|
||||
{"label": "graph", "time": 1789664827.8988104, "event": "REQUEST_START", "name": "prefill-32768-1", "profile": false, "prompt_tokens_preflight": 32745}
|
||||
{"label": "graph", "time": 1789664845.1446679, "event": "RESULT", "name": "prefill-32768-1", "profile": false, "correct": true, "content": "\n\n90812", "finish_reason": "stop", "usage": {"prompt_tokens": 32745, "total_tokens": 32814, "completion_tokens": 69, "completion_tokens_details": {"reasoning_tokens": 61}}, "max_tokens": 2048, "prompt_sha256": "0a5e4ba33ff264d180b8a56d89184c0fefad59a96fdd348d655bd54fe5b5ed45", "output_sha256": "8b503330d6b70b8845bab80540601fc48c8f840d54d07f38d80250c8167de89f", "ttft_s": 15.357147006005107, "first_content_s": 17.08089536800253, "elapsed_s": 17.239325528003974, "decode_tps_approx": 37.728914829613814, "metrics_delta": {"vllm:prefix_cache_queries_total": 32745.0, "vllm:prefix_cache_hits_total": 0.0}}
|
||||
{"label": "graph", "time": 1789664845.1907535, "event": "REQUEST_START", "name": "reuse-32768-1", "profile": false, "prompt_tokens_preflight": 32745}
|
||||
{"label": "graph", "time": 1789664848.6512141, "event": "RESULT", "name": "reuse-32768-1", "profile": false, "correct": true, "content": "\n\n90812", "finish_reason": "stop", "usage": {"prompt_tokens": 32745, "total_tokens": 32815, "completion_tokens": 70, "completion_tokens_details": {"reasoning_tokens": 62}}, "max_tokens": 2048, "prompt_sha256": "0a5e4ba33ff264d180b8a56d89184c0fefad59a96fdd348d655bd54fe5b5ed45", "output_sha256": "7eec57291da7ee95ae6d9214f8ed940a405e64beb9f49a4693387362aac3e14c", "ttft_s": 1.3272808809997514, "first_content_s": 3.296925946000556, "elapsed_s": 3.4540864679947845, "decode_tps_approx": 33.71980234098872, "metrics_delta": {"vllm:prefix_cache_queries_total": 32745.0, "vllm:prefix_cache_hits_total": 30400.0}}
|
||||
{"label": "graph", "time": 1789664848.660781, "event": "REQUEST_START", "name": "short-2", "profile": false, "prompt_tokens_preflight": 65}
|
||||
{"label": "graph", "time": 1789664855.73135, "event": "RESULT", "name": "short-2", "profile": false, "correct": true, "content": "\n\n数据库索引类似于书籍的目录，它通过B+树等数据结构将数据按特定字段有序组织，使查询时能快速定位目标记录，避免全表扫描，从而大幅提升读取效率。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库都需要同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略，需根据实际读写比例合理设计。", "finish_reason": "stop", "usage": {"prompt_tokens": 65, "total_tokens": 279, "completion_tokens": 214, "completion_tokens_details": {"reasoning_tokens": 111}}, "max_tokens": 512, "prompt_sha256": "40e7d1df3e96ad760d661cfaff312f5d732cb2edd5eab555b4b5c43cea4512ca", "output_sha256": "126232af3bc1889efc9c0dad67f2d47fc60d3c4ff56e699a83f1576c3763c274", "ttft_s": 0.2623967160034226, "first_content_s": 3.7147180649990332, "elapsed_s": 7.0642776130043785, "decode_tps_approx": 31.315723764990278, "metrics_delta": {"vllm:prefix_cache_queries_total": 65.0, "vllm:prefix_cache_hits_total": 0.0}}
|
||||
{"label": "graph", "time": 1789664855.7496016, "event": "REQUEST_START", "name": "prefill-8192-2", "profile": false, "prompt_tokens_preflight": 8173}
|
||||
{"label": "graph", "time": 1789664861.4250884, "event": "RESULT", "name": "prefill-8192-2", "profile": false, "correct": true, "content": "\n\n21573", "finish_reason": "stop", "usage": {"prompt_tokens": 8173, "total_tokens": 8242, "completion_tokens": 69, "completion_tokens_details": {"reasoning_tokens": 61}}, "max_tokens": 2048, "prompt_sha256": "b49f6dbbf80f49ad860a60b0a7c58dfc2122e38bb8493ae155790f22305d8f7a", "output_sha256": "f72eba21a095aa0a43356b25cb120346b064d7f0a253a63cee7579d2c3422b82", "ttft_s": 3.768740819999948, "first_content_s": 5.512005377000605, "elapsed_s": 5.670499198000471, "decode_tps_approx": 37.32072215199369, "metrics_delta": {"vllm:prefix_cache_queries_total": 8173.0, "vllm:prefix_cache_hits_total": 0.0}}
|
||||
{"label": "graph", "time": 1789664861.4389272, "event": "REQUEST_START", "name": "reuse-8192-2", "profile": false, "prompt_tokens_preflight": 8173}
|
||||
{"label": "graph", "time": 1789664864.1561685, "event": "RESULT", "name": "reuse-8192-2", "profile": false, "correct": true, "content": "\n\n21573", "finish_reason": "stop", "usage": {"prompt_tokens": 8173, "total_tokens": 8236, "completion_tokens": 63, "completion_tokens_details": {"reasoning_tokens": 55}}, "max_tokens": 2048, "prompt_sha256": "b49f6dbbf80f49ad860a60b0a7c58dfc2122e38bb8493ae155790f22305d8f7a", "output_sha256": "09d17d110b06ec13d532382c41ea29955b7edfa8dd4bd34163a08fa746e56610", "ttft_s": 0.8816422219970264, "first_content_s": 2.543782171997009, "elapsed_s": 2.711807350999152, "decode_tps_approx": 35.522341967132114, "metrics_delta": {"vllm:prefix_cache_queries_total": 8173.0, "vllm:prefix_cache_hits_total": 6400.0}}
|
||||
{"label": "graph", "time": 1789664864.196416, "event": "REQUEST_START", "name": "prefill-32768-2", "profile": false, "prompt_tokens_preflight": 32745}
|
||||
{"label": "graph", "time": 1789664881.472875, "event": "RESULT", "name": "prefill-32768-2", "profile": false, "correct": true, "content": "\n\n90812", "finish_reason": "stop", "usage": {"prompt_tokens": 32745, "total_tokens": 32814, "completion_tokens": 69, "completion_tokens_details": {"reasoning_tokens": 61}}, "max_tokens": 2048, "prompt_sha256": "d8796cea8608150048ce4f253e451ba96da5416ccfea4a6560dc2f6bb2412004", "output_sha256": "8b503330d6b70b8845bab80540601fc48c8f840d54d07f38d80250c8167de89f", "ttft_s": 15.375977513001999, "first_content_s": 17.108021717001975, "elapsed_s": 17.268514374001825, "decode_tps_approx": 37.54862656190945, "metrics_delta": {"vllm:prefix_cache_queries_total": 32745.0, "vllm:prefix_cache_hits_total": 0.0}}
|
||||
{"label": "graph", "time": 1789664881.5214229, "event": "REQUEST_START", "name": "reuse-32768-2", "profile": false, "prompt_tokens_preflight": 32745}
|
||||
{"label": "graph", "time": 1789664884.7351754, "event": "RESULT", "name": "reuse-32768-2", "profile": false, "correct": true, "content": "\n\n90812", "finish_reason": "stop", "usage": {"prompt_tokens": 32745, "total_tokens": 32814, "completion_tokens": 69, "completion_tokens_details": {"reasoning_tokens": 61}}, "max_tokens": 2048, "prompt_sha256": "d8796cea8608150048ce4f253e451ba96da5416ccfea4a6560dc2f6bb2412004", "output_sha256": "8b503330d6b70b8845bab80540601fc48c8f840d54d07f38d80250c8167de89f", "ttft_s": 1.3270910260034725, "first_content_s": 3.045836498000426, "elapsed_s": 3.2060768180017476, "decode_tps_approx": 37.82214005937634, "metrics_delta": {"vllm:prefix_cache_queries_total": 32745.0, "vllm:prefix_cache_hits_total": 30400.0}}
|
||||
{"label": "graph", "time": 1789664884.7446704, "event": "REQUEST_START", "name": "trace-short", "profile": true, "prompt_tokens_preflight": 65}
|
||||
{"label": "graph", "time": 1789664894.3422408, "event": "RESULT", "name": "trace-short", "profile": true, "correct": true, "content": "\n\n数据库索引类似于书籍的目录，它通过B+树等数据结构将数据按特定字段有序组织，使查询时能快速定位目标记录，避免全表扫描，从而大幅提升读取效率。然而，索引对写入操作有负面影响：每次插入、更新或删除数据时，数据库都需要同步维护索引结构，这会增加额外的I/O和计算开销，导致写入性能下降。因此，索引是\"以空间换时间\"的典型策略，需根据实际读写比例合理设计。", "finish_reason": "stop", "usage": {"prompt_tokens": 65, "total_tokens": 279, "completion_tokens": 214, "completion_tokens_details": {"reasoning_tokens": 111}}, "max_tokens": 512, "prompt_sha256": "40e7d1df3e96ad760d661cfaff312f5d732cb2edd5eab555b4b5c43cea4512ca", "output_sha256": "126232af3bc1889efc9c0dad67f2d47fc60d3c4ff56e699a83f1576c3763c274", "ttft_s": 0.23192091599776177, "first_content_s": 3.732794447001652, "elapsed_s": 9.19204898099997, "decode_tps_approx": 30.944518883030874, "metrics_delta": {"vllm:prefix_cache_queries_total": 65.0, "vllm:prefix_cache_hits_total": 0.0}}
|
||||
{"label": "graph", "time": 1789664894.3764791, "event": "REQUEST_START", "name": "trace-prefill-32768", "profile": true, "prompt_tokens_preflight": 32742}
|
||||
{"label": "graph", "time": 1789664914.0692568, "event": "RESULT", "name": "trace-prefill-32768", "profile": true, "correct": true, "content": "\n\n90812", "finish_reason": "stop", "usage": {"prompt_tokens": 32742, "total_tokens": 32812, "completion_tokens": 70, "completion_tokens_details": {"reasoning_tokens": 62}}, "max_tokens": 2048, "prompt_sha256": "464e9da0976ca56ddece006a237605bca8b60d3dd0aab538abbc0aaa1696815c", "output_sha256": "7eec57291da7ee95ae6d9214f8ed940a405e64beb9f49a4693387362aac3e14c", "ttft_s": 15.522943222997128, "first_content_s": 17.421024231000047, "elapsed_s": 19.56250263000402, "decode_tps_approx": 34.818013845987785, "metrics_delta": {"vllm:prefix_cache_queries_total": 32742.0, "vllm:prefix_cache_hits_total": 0.0}}
|
||||
{"label": "graph", "time": 1789664914.1044817, "event": "REQUEST_START", "name": "trace-reuse-32768", "profile": true, "prompt_tokens_preflight": 32742}
|
||||
{"label": "graph", "time": 1789664920.2165241, "event": "RESULT", "name": "trace-reuse-32768", "profile": true, "correct": true, "content": "\n\n90812", "finish_reason": "stop", "usage": {"prompt_tokens": 32742, "total_tokens": 32816, "completion_tokens": 74, "completion_tokens_details": {"reasoning_tokens": 66}}, "max_tokens": 2048, "prompt_sha256": "464e9da0976ca56ddece006a237605bca8b60d3dd0aab538abbc0aaa1696815c", "output_sha256": "997501e1996b1c37af707622b1c4caf8dba93113e868f28210bd47ba700b5804", "ttft_s": 1.3475809320007102, "first_content_s": 3.358952419999696, "elapsed_s": 5.983986291001202, "decode_tps_approx": 34.79999075012843, "metrics_delta": {"vllm:prefix_cache_queries_total": 32742.0, "vllm:prefix_cache_hits_total": 30400.0}}
|
||||
{"label": "graph", "time": 1789664920.2165873, "event": "PROFILE_SUITE_PASS"}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,852 @@
|
||||
{
|
||||
"file": "trace.2.sqlite",
|
||||
"gpu_activity_span_s": 17.613957408,
|
||||
"gpu_busy_union_s": 17.173015684,
|
||||
"gpu_idle_within_span_s": 0.44094172400000176,
|
||||
"kernel_count": 111055,
|
||||
"graph_node_kernel_count": 49140,
|
||||
"api": {
|
||||
"cudaStreamSynchronize_v3020": [
|
||||
96,
|
||||
7.291123311999998
|
||||
],
|
||||
"cuLaunchKernelEx": [
|
||||
22540,
|
||||
2.9715556959999576
|
||||
],
|
||||
"cudaLaunchKernelExC_v11060": [
|
||||
11197,
|
||||
2.7721689919999997
|
||||
],
|
||||
"cudaLaunchKernel_v7000": [
|
||||
24666,
|
||||
2.074785632000026
|
||||
],
|
||||
"cuLaunchKernel": [
|
||||
3512,
|
||||
0.1402584800000001
|
||||
],
|
||||
"cudaGraphLaunch_v10000": [
|
||||
1431,
|
||||
0.13174172799999972
|
||||
],
|
||||
"cudaEventSynchronize_v3020": [
|
||||
48,
|
||||
0.042627568
|
||||
],
|
||||
"cudaMemcpyAsync_v3020": [
|
||||
2398,
|
||||
0.02786942400000001
|
||||
],
|
||||
"cudaMemsetAsync_v3020": [
|
||||
768,
|
||||
0.005161456000000002
|
||||
],
|
||||
"cuKernelGetName": [
|
||||
25422,
|
||||
0.002872591999999989
|
||||
],
|
||||
"cudaEventQuery_v3020": [
|
||||
1557,
|
||||
0.0023473760000000004
|
||||
],
|
||||
"cuTensorMapEncodeTiled": [
|
||||
12786,
|
||||
0.002259360000000043
|
||||
],
|
||||
"cudaEventRecordWithFlags_v11010": [
|
||||
1003,
|
||||
0.001927552000000001
|
||||
],
|
||||
"cudaGetDriverEntryPointByVersion_v12050": [
|
||||
12096,
|
||||
0.0017841920000000863
|
||||
],
|
||||
"cuKernelGetFunction": [
|
||||
3512,
|
||||
0.0013231039999999851
|
||||
],
|
||||
"cudaStreamIsCapturing_v10000": [
|
||||
2144,
|
||||
0.0006217759999999976
|
||||
],
|
||||
"cudaStreamWaitEvent_v3020": [
|
||||
144,
|
||||
0.00041838400000000016
|
||||
],
|
||||
"cudaEventCreateWithFlags_v3020": [
|
||||
96,
|
||||
0.00024513599999999997
|
||||
],
|
||||
"cudaEventDestroy_v3020": [
|
||||
96,
|
||||
0.00022665600000000007
|
||||
]
|
||||
},
|
||||
"nvtx": {
|
||||
"PLE_lookup_tokens=1600": {
|
||||
"count": 20,
|
||||
"sum_s": 5.607371088,
|
||||
"union_s": 5.607371088,
|
||||
"gpu_overlap_s": 5.336825187
|
||||
},
|
||||
"PLE_wait_for_GPU": {
|
||||
"count": 48,
|
||||
"sum_s": 7.291963728,
|
||||
"union_s": 7.291963728,
|
||||
"gpu_overlap_s": 7.233423306
|
||||
},
|
||||
"PLE_CPU_dedup": {
|
||||
"count": 48,
|
||||
"sum_s": 0.020901856,
|
||||
"union_s": 0.020901856,
|
||||
"gpu_overlap_s": 0.0
|
||||
},
|
||||
"PLE_CPU_gather": {
|
||||
"count": 48,
|
||||
"sum_s": 0.292111936,
|
||||
"union_s": 0.292111936,
|
||||
"gpu_overlap_s": 0.0
|
||||
},
|
||||
"PLE_lookup_tokens=742": {
|
||||
"count": 1,
|
||||
"sum_s": 0.29688864,
|
||||
"union_s": 0.29688864,
|
||||
"gpu_overlap_s": 0.282970556
|
||||
},
|
||||
"PLE_lookup_tokens=4": {
|
||||
"count": 27,
|
||||
"sum_s": 1.71834584,
|
||||
"union_s": 1.71834584,
|
||||
"gpu_overlap_s": 1.61634065
|
||||
}
|
||||
},
|
||||
"dispatch": {
|
||||
"CG_DISPATCH:actual=1600:mode=NONE:padded=1600": 40,
|
||||
"CG_DISPATCH:actual=1:mode=NONE:padded=1": 48,
|
||||
"CG_DISPATCH:actual=742:mode=NONE:padded=742": 2,
|
||||
"CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4": 27,
|
||||
"CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4": 27
|
||||
},
|
||||
"dispatch_timeline": [
|
||||
{
|
||||
"start_ns": 46743952,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 416451360,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 443082544,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 469310784,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 1124006496,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 1154683696,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 1179774000,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 1845801456,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 1877195728,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 1902001888,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 2578899920,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 2609916288,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 2635831856,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 3318901952,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 3349740016,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 3376088736,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 4054732640,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 4085130432,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 4110746784,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 4797770512,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 4829017936,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 4854209488,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 5551719760,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 5581307632,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 5607134000,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 6295334464,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 6325098224,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 6349994272,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 7042285248,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 7072015024,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 7097530960,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 7785043856,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 7816741056,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 7842952592,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 8537987536,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 8569536560,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 8595452192,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 9285771248,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 9317233072,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 9342219952,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 10060629616,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 10091559456,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 10116812048,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 10807242976,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 10837982160,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 10864521920,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 11562733424,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 11593737408,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 11619654832,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 12323127600,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 12354216560,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 12380982848,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 13081498400,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 13113506512,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 13138598000,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 13833930640,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 13865480864,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 13890978272,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 14599979088,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 14632423408,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 14657285504,
|
||||
"name": "CG_DISPATCH:actual=742:mode=NONE:padded=742"
|
||||
},
|
||||
{
|
||||
"start_ns": 15244687408,
|
||||
"name": "CG_DISPATCH:actual=742:mode=NONE:padded=742"
|
||||
},
|
||||
{
|
||||
"start_ns": 15261168608,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 15272860704,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15510425328,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15511343584,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 15516520016,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15591171712,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15592218224,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 15597515056,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15670668032,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15671484256,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 15675816752,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15748911968,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15749703280,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 15753801520,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15829628848,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15830463168,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 15834544656,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15911548208,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15912496096,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 15917334144,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15989715552,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 15990467984,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 15994717936,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16067760256,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16068616608,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 16072918640,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16151936192,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16153013376,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 16158295088,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16231124064,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16232113808,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 16237373136,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16308737952,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16309646016,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 16314212128,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16388651696,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16389407904,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 16393690448,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16470779408,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16471838880,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 16476836288,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16549986896,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16550865424,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 16555256672,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16632067984,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16632894512,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 16637253408,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16709708944,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16710541136,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 16714847184,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16789609392,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16790634928,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 16795643776,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16869435120,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16870399200,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 16875445984,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16952330656,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 16953311568,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 16958431376,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17031362304,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17032114688,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 17036354400,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17111019568,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17112056224,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 17117105808,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17188540768,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17189324992,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 17193788208,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17269626256,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17270693728,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 17276082128,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17347956512,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17348732480,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 17352868096,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17427032704,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17427739648,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 17431571536,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17508960384,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17509972320,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 17514870544,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17592660304,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 17593603200,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
}
|
||||
],
|
||||
"top_kernels": [
|
||||
[
|
||||
"device_kernel",
|
||||
[
|
||||
4608,
|
||||
5.806803458000012
|
||||
]
|
||||
],
|
||||
[
|
||||
"Kernel2",
|
||||
[
|
||||
16931,
|
||||
1.7378030570000023
|
||||
]
|
||||
],
|
||||
[
|
||||
"_qsa_sparse_paged_gqa_splitk_kernel",
|
||||
[
|
||||
672,
|
||||
1.4925525440000014
|
||||
]
|
||||
],
|
||||
[
|
||||
"nvjet_sm121_tst_mma_128x208x64_2_32x104x64_tmaAB_bz_TNNN",
|
||||
[
|
||||
1000,
|
||||
1.3766302719999985
|
||||
]
|
||||
],
|
||||
[
|
||||
"_hc_combine_norm_kernel",
|
||||
[
|
||||
4848,
|
||||
0.9208744449999874
|
||||
]
|
||||
],
|
||||
[
|
||||
"kernel",
|
||||
[
|
||||
2853,
|
||||
0.8909366829999992
|
||||
]
|
||||
],
|
||||
[
|
||||
"_hc_gate_mix_kernel",
|
||||
[
|
||||
4944,
|
||||
0.6871558350000507
|
||||
]
|
||||
],
|
||||
[
|
||||
"nvjet_sm121_tst_mma_128x240x64_2_32x120x64_tmaAB_bz_TNNN",
|
||||
[
|
||||
980,
|
||||
0.5473217280000005
|
||||
]
|
||||
],
|
||||
[
|
||||
"nvjet_sm121_tst_mma_128x128x64_3_32x64x64_tmaAB_bz_TNNN",
|
||||
[
|
||||
2000,
|
||||
0.4623586880000001
|
||||
]
|
||||
],
|
||||
[
|
||||
"finalizeMoeRoutingKernel",
|
||||
[
|
||||
2304,
|
||||
0.4111630399999976
|
||||
]
|
||||
],
|
||||
[
|
||||
"_causal_conv1d_fwd_kernel",
|
||||
[
|
||||
756,
|
||||
0.24460185600000015
|
||||
]
|
||||
],
|
||||
[
|
||||
"doActivationKernel",
|
||||
[
|
||||
2304,
|
||||
0.23128571500000175
|
||||
]
|
||||
],
|
||||
[
|
||||
"_fused_post_conv_kernel",
|
||||
[
|
||||
756,
|
||||
0.23053734400000014
|
||||
]
|
||||
],
|
||||
[
|
||||
"elementwise_kernel",
|
||||
[
|
||||
6192,
|
||||
0.22792899099999814
|
||||
]
|
||||
],
|
||||
[
|
||||
"kernel_cutlass_kernel_flashinfergdn_kernelsdelta_rule_dsldelta_rule_sm120_FullyFusedDeltaRuleSm120_object_at__tensorptrf32gmemalign16o1_tensorptrf32gmemalign16o1_CopyAtom_ThrID10_TVLayout_0",
|
||||
[
|
||||
756,
|
||||
0.19290607999999995
|
||||
]
|
||||
],
|
||||
[
|
||||
"vectorized_elementwise_kernel",
|
||||
[
|
||||
11370,
|
||||
0.18636042799999758
|
||||
]
|
||||
],
|
||||
[
|
||||
"layer_norm_fwd_kernel",
|
||||
[
|
||||
756,
|
||||
0.18163276799999997
|
||||
]
|
||||
],
|
||||
[
|
||||
"sm120_fp8_fp4_gemm_1d1d_impl",
|
||||
[
|
||||
192,
|
||||
0.17354873599999998
|
||||
]
|
||||
],
|
||||
[
|
||||
"expandInputRowsKernel",
|
||||
[
|
||||
2304,
|
||||
0.1723688820000004
|
||||
]
|
||||
],
|
||||
[
|
||||
"nvjet_sm121_tst_mma_128x192x64_2_32x96x64_tmaAB_bz_TNNN",
|
||||
[
|
||||
980,
|
||||
0.13571574399999997
|
||||
]
|
||||
]
|
||||
],
|
||||
"notes": "GPU busy is interval union, not utilization counter. CPU/API/NVTX times overlap GPU work and each other; do not sum them as exclusive costs."
|
||||
}
|
||||
@@ -0,0 +1,636 @@
|
||||
{
|
||||
"file": "trace.3.sqlite",
|
||||
"gpu_activity_span_s": 3.549492992,
|
||||
"gpu_busy_union_s": 3.382135698,
|
||||
"gpu_idle_within_span_s": 0.16735729399999988,
|
||||
"kernel_count": 66056,
|
||||
"graph_node_kernel_count": 50960,
|
||||
"api": {
|
||||
"cudaStreamSynchronize_v3020": [
|
||||
60,
|
||||
2.063916384
|
||||
],
|
||||
"cudaLaunchKernelExC_v11060": [
|
||||
1284,
|
||||
0.28009695999999934
|
||||
],
|
||||
"cuLaunchKernelEx": [
|
||||
5990,
|
||||
0.2070799999999992
|
||||
],
|
||||
"cudaLaunchKernel_v7000": [
|
||||
7285,
|
||||
0.1831771039999994
|
||||
],
|
||||
"cudaGraphLaunch_v10000": [
|
||||
1484,
|
||||
0.119649248
|
||||
],
|
||||
"cudaEventSynchronize_v3020": [
|
||||
30,
|
||||
0.040196352
|
||||
],
|
||||
"cuLaunchKernel": [
|
||||
537,
|
||||
0.01219283200000001
|
||||
],
|
||||
"cudaMemcpyAsync_v3020": [
|
||||
1043,
|
||||
0.010008064000000006
|
||||
],
|
||||
"cudaMemsetAsync_v3020": [
|
||||
480,
|
||||
0.0025478400000000017
|
||||
],
|
||||
"cudaEventQuery_v3020": [
|
||||
921,
|
||||
0.0011545439999999998
|
||||
],
|
||||
"cuKernelGetName": [
|
||||
7357,
|
||||
0.00088769599999998
|
||||
],
|
||||
"cudaEventRecordWithFlags_v11010": [
|
||||
605,
|
||||
0.0008087840000000007
|
||||
],
|
||||
"cudaStreamIsCapturing_v10000": [
|
||||
1274,
|
||||
0.00032420800000000107
|
||||
],
|
||||
"cuTensorMapEncodeTiled": [
|
||||
1472,
|
||||
0.0003082879999999999
|
||||
],
|
||||
"cuKernelGetFunction": [
|
||||
537,
|
||||
0.0002906560000000001
|
||||
],
|
||||
"cudaGetDriverEntryPointByVersion_v12050": [
|
||||
1152,
|
||||
0.00016971199999999957
|
||||
],
|
||||
"cudaStreamWaitEvent_v3020": [
|
||||
90,
|
||||
0.000155392
|
||||
],
|
||||
"cudaEventCreateWithFlags_v3020": [
|
||||
60,
|
||||
0.00014443199999999998
|
||||
],
|
||||
"cudaEventDestroy_v3020": [
|
||||
60,
|
||||
8.001600000000003e-05
|
||||
]
|
||||
},
|
||||
"nvtx": {
|
||||
"PLE_lookup_tokens=1600": {
|
||||
"count": 1,
|
||||
"sum_s": 0.019309872,
|
||||
"union_s": 0.019309872,
|
||||
"gpu_overlap_s": 0.010934608
|
||||
},
|
||||
"PLE_wait_for_GPU": {
|
||||
"count": 30,
|
||||
"sum_s": 2.064303616,
|
||||
"union_s": 2.064303616,
|
||||
"gpu_overlap_s": 2.035614378
|
||||
},
|
||||
"PLE_CPU_dedup": {
|
||||
"count": 30,
|
||||
"sum_s": 0.00226768,
|
||||
"union_s": 0.00226768,
|
||||
"gpu_overlap_s": 0.0
|
||||
},
|
||||
"PLE_CPU_gather": {
|
||||
"count": 30,
|
||||
"sum_s": 0.109426208,
|
||||
"union_s": 0.109426208,
|
||||
"gpu_overlap_s": 0.0
|
||||
},
|
||||
"PLE_lookup_tokens=742": {
|
||||
"count": 1,
|
||||
"sum_s": 0.293855216,
|
||||
"union_s": 0.293855216,
|
||||
"gpu_overlap_s": 0.27970459
|
||||
},
|
||||
"PLE_lookup_tokens=4": {
|
||||
"count": 28,
|
||||
"sum_s": 1.874469744,
|
||||
"union_s": 1.874469744,
|
||||
"gpu_overlap_s": 1.748828475
|
||||
}
|
||||
},
|
||||
"dispatch": {
|
||||
"CG_DISPATCH:actual=1600:mode=NONE:padded=1600": 2,
|
||||
"CG_DISPATCH:actual=1:mode=NONE:padded=1": 30,
|
||||
"CG_DISPATCH:actual=742:mode=NONE:padded=742": 2,
|
||||
"CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4": 28,
|
||||
"CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4": 28
|
||||
},
|
||||
"dispatch_timeline": [
|
||||
{
|
||||
"start_ns": 47667312,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 442471520,
|
||||
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
|
||||
},
|
||||
{
|
||||
"start_ns": 473807600,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 498595616,
|
||||
"name": "CG_DISPATCH:actual=742:mode=NONE:padded=742"
|
||||
},
|
||||
{
|
||||
"start_ns": 1085215856,
|
||||
"name": "CG_DISPATCH:actual=742:mode=NONE:padded=742"
|
||||
},
|
||||
{
|
||||
"start_ns": 1101603472,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 1113435424,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1335575776,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1336548688,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 1340993280,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1412886784,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1413839472,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 1418416416,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1496932144,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1497849376,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 1502437168,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1574235248,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1574893632,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 1578152608,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1659975184,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1660978304,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 1666040464,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1739695136,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1740582992,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 1745368080,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1819362544,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1820259120,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 1825002016,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1898444080,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1899320480,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 1903978080,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1973992688,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 1974642976,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 1977460352,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2058854000,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2059781888,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 2064550128,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2140294400,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2141166416,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 2145782752,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2215803760,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2216437328,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 2219274592,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2295308880,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2295905872,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 2298529488,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2373456000,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2374001712,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 2376400016,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2451495056,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2452018624,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 2454446720,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2534005728,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2534498160,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 2536805632,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2617391424,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2618055024,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 2620944928,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2698279104,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2699135648,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 2704056256,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2781484592,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2782408128,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 2787209600,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2872938352,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2873856272,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 2878412464,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2955704400,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 2956657904,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 2961494272,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3046465232,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3047422016,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 3052108288,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3123689648,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3124649664,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 3129308048,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3199193712,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3199819264,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 3202578560,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3285661472,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3286559472,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 3291397856,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3363509456,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3364085808,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 3366681840,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3447715488,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3448689632,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
},
|
||||
{
|
||||
"start_ns": 3453541344,
|
||||
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3529683168,
|
||||
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
|
||||
},
|
||||
{
|
||||
"start_ns": 3530662560,
|
||||
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
|
||||
}
|
||||
],
|
||||
"top_kernels": [
|
||||
[
|
||||
"Kernel2",
|
||||
[
|
||||
14453,
|
||||
1.2634071650000012
|
||||
]
|
||||
],
|
||||
[
|
||||
"device_kernel",
|
||||
[
|
||||
2880,
|
||||
0.9274175719999982
|
||||
]
|
||||
],
|
||||
[
|
||||
"kernel",
|
||||
[
|
||||
1772,
|
||||
0.4575362240000001
|
||||
]
|
||||
],
|
||||
[
|
||||
"_qsa_sparse_paged_gqa_splitk_kernel",
|
||||
[
|
||||
420,
|
||||
0.13202313600000007
|
||||
]
|
||||
],
|
||||
[
|
||||
"_hc_combine_norm_kernel",
|
||||
[
|
||||
3030,
|
||||
0.07136511900000105
|
||||
]
|
||||
],
|
||||
[
|
||||
"nvjet_sm121_tst_mma_128x208x64_2_32x104x64_tmaAB_bz_TNNN",
|
||||
[
|
||||
50,
|
||||
0.06789817599999999
|
||||
]
|
||||
],
|
||||
[
|
||||
"sm120_fp8_fp4_gemm_1d1d_impl",
|
||||
[
|
||||
120,
|
||||
0.05055315199999998
|
||||
]
|
||||
],
|
||||
[
|
||||
"_hc_gate_mix_kernel",
|
||||
[
|
||||
3090,
|
||||
0.049511773999999786
|
||||
]
|
||||
],
|
||||
[
|
||||
"finalizeMoeRoutingKernel",
|
||||
[
|
||||
1440,
|
||||
0.04024281600000026
|
||||
]
|
||||
],
|
||||
[
|
||||
"nvjet_sm121_tst_mma_192x160x64_2_48x80x64_tmaAB_bz_TNNN",
|
||||
[
|
||||
51,
|
||||
0.035206783999999984
|
||||
]
|
||||
],
|
||||
[
|
||||
"gdn_decode_post_conv_mtp_kernel",
|
||||
[
|
||||
1008,
|
||||
0.027670688000000006
|
||||
]
|
||||
],
|
||||
[
|
||||
"nvjet_sm121_tst_mma_128x240x64_2_32x120x64_tmaAB_bz_TNNN",
|
||||
[
|
||||
49,
|
||||
0.027358240000000006
|
||||
]
|
||||
],
|
||||
[
|
||||
"elementwise_kernel",
|
||||
[
|
||||
2268,
|
||||
0.02362060799999982
|
||||
]
|
||||
],
|
||||
[
|
||||
"nvjet_sm121_tst_mma_128x128x64_3_32x64x64_tmaAB_bz_TNNN",
|
||||
[
|
||||
100,
|
||||
0.023191840000000002
|
||||
]
|
||||
],
|
||||
[
|
||||
"vectorized_elementwise_kernel",
|
||||
[
|
||||
6095,
|
||||
0.02246828099999996
|
||||
]
|
||||
],
|
||||
[
|
||||
"doActivationKernel",
|
||||
[
|
||||
1440,
|
||||
0.02160813899999983
|
||||
]
|
||||
],
|
||||
[
|
||||
"expandInputRowsKernel",
|
||||
[
|
||||
1440,
|
||||
0.020183961000000014
|
||||
]
|
||||
],
|
||||
[
|
||||
"_causal_conv1d_fwd_kernel",
|
||||
[
|
||||
72,
|
||||
0.017587615999999997
|
||||
]
|
||||
],
|
||||
[
|
||||
"nvjet_sm121_tst_mma_128x224x64_2_64x56x64_tmaAB_bz_TNNN",
|
||||
[
|
||||
50,
|
||||
0.016199904
|
||||
]
|
||||
],
|
||||
[
|
||||
"_fused_post_conv_kernel",
|
||||
[
|
||||
72,
|
||||
0.014694784
|
||||
]
|
||||
]
|
||||
],
|
||||
"notes": "GPU busy is interval union, not utilization counter. CPU/API/NVTX times overlap GPU work and each other; do not sum them as exclusive costs."
|
||||
}
|
||||
Reference in New Issue
Block a user