ruihan/qwen38-flash-next-dgx-spark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
perf/compile-prefix-cache
qwen38-flash-next-dgx-spark/docs/results/profile128k/eager/trace.2.summary.json
T

823 lines
19 KiB
JSON
Raw Permalink Blame History

{
"file": "trace.2.sqlite",
"gpu_activity_span_s": 17.45799776,
"gpu_busy_union_s": 16.999974145,
"gpu_idle_within_span_s": 0.4580236150000019,
"kernel_count": 107899,
"graph_node_kernel_count": 0,
"api": {
"cudaStreamSynchronize_v3020": [
92,
6.388727359999995
],
"cuLaunchKernelEx": [
30313,
2.942097295999861
],
"cudaLaunchKernelExC_v11060": [
27262,
2.8157289279998983
],
"cudaLaunchKernel_v7000": [
34403,
2.1073674239999285
],
"cuLaunchKernel": [
15921,
0.18051391999999974
],
"cudaMemcpyAsync_v3020": [
2359,
0.02657009600000002
],
"cudaEventSynchronize_v3020": [
46,
0.01590248
],
"cudaEventRecordWithFlags_v11010": [
3413,
0.006630880000000002
],
"cudaMemsetAsync_v3020": [
736,
0.0055193599999999975
],
"cuTensorMapEncodeTiled": [
27416,
0.004886224000000271
],
"cuKernelGetFunction": [
15921,
0.0038959359999997194
],
"cudaGetDriverEntryPointByVersion_v12050": [
26496,
0.0037329280000005572
],
"cuKernelGetName": [
35159,
0.0033410560000001516
],
"cudaStreamWaitEvent_v3020": [
2588,
0.0021758880000000004
],
"cudaEventQuery_v3020": [
1494,
0.0019392639999999957
],
"cudaStreamIsCapturing_v10000": [
6960,
0.0014392959999999955
],
"cudaEventCreateWithFlags_v3020": [
92,
0.000234256
],
"cudaEventDestroy_v3020": [
92,
0.00018534400000000007
]
},
"nvtx": {
"PLE_lookup_tokens=1600": {
"count": 20,
"sum_s": 5.58014968,
"union_s": 5.58014968,
"gpu_overlap_s": 5.318215141
},
"PLE_wait_for_GPU": {
"count": 46,
"sum_s": 6.389229296,
"union_s": 6.389229296,
"gpu_overlap_s": 6.341070402
},
"PLE_CPU_dedup": {
"count": 46,
"sum_s": 0.01914024,
"union_s": 0.01914024,
"gpu_overlap_s": 0.0
},
"PLE_CPU_gather": {
"count": 46,
"sum_s": 0.275209776,
"union_s": 0.275209776,
"gpu_overlap_s": 0.0
},
"PLE_lookup_tokens=742": {
"count": 1,
"sum_s": 0.29896912,
"union_s": 0.29896912,
"gpu_overlap_s": 0.278459202
},
"PLE_lookup_tokens=3": {
"count": 25,
"sum_s": 0.820145456,
"union_s": 0.820145456,
"gpu_overlap_s": 0.746778411
}
},
"dispatch": {
"CG_DISPATCH:actual=1600:mode=NONE:padded=1600": 40,
"CG_DISPATCH:actual=1:mode=NONE:padded=1": 46,
"CG_DISPATCH:actual=742:mode=NONE:padded=742": 2,
"CG_DISPATCH:actual=3:mode=NONE:padded=3": 50
},
"dispatch_timeline": [
{
"start_ns": 45828672,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 415343408,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 441920176,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 467464272,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 1117406208,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 1146479600,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1170640896,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 1839789152,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 1870939456,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1896334032,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 2568046544,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 2598479856,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2624039184,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 3305750304,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 3335311984,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 3361130160,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 4046256096,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 4076056912,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 4101620656,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 4786128624,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 4816276720,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 4842056960,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 5522623936,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 5552300720,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 5576921440,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 6259346144,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 6290320240,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 6315131568,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 7002476032,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 7032926192,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 7058327584,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 7753655264,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 7785387856,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 7811813408,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 8504434640,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 8535191696,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 8561506848,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 9253677696,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 9285233312,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 9311217936,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 10009614064,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 10040506016,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 10066581360,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 10757969120,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 10789570400,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 10815237984,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 11519444752,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 11550823200,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 11575788896,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 12278226144,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 12309497040,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 12335515504,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 13034051120,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 13066055552,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 13091963328,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 13790163952,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 13821175296,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 13846090352,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 14557613040,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 14589908560,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 14615422640,
"name": "CG_DISPATCH:actual=742:mode=NONE:padded=742"
},
{
"start_ns": 15203740016,
"name": "CG_DISPATCH:actual=742:mode=NONE:padded=742"
},
{
"start_ns": 15220760336,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15234841296,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15491074784,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15493643856,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15497135184,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15564983824,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15566982016,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15570203408,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15644290848,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15646296832,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15649329520,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15729121168,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15731019920,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15734338320,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15813356656,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15815902416,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15819264448,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15894042800,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15895960816,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15898832688,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15982235440,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 15984929392,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15988404656,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16064077552,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16066769024,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16070591120,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16140542304,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16142492784,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16145392256,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16230150416,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16232897584,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16236422000,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16305893904,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16307780160,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16310441504,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16394138896,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16396838944,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16400267168,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16476712080,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16479446496,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16482904144,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16559135840,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16561788800,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16565117088,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16632650032,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16634497504,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16637262864,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16722767040,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16725243152,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16728652272,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16807455696,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16810202752,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16813561920,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16894260000,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16898134688,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16903894256,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16982500864,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 16985299840,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16988503344,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 17054985488,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 17056901872,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 17059848448,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 17143682800,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 17146548688,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 17150240800,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 17226658560,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 17229352608,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 17233024096,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 17306935664,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 17308968096,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 17311992480,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 17388634448,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 17391419008,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 17394878992,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 17463051952,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 17464913120,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
}
],
"top_kernels": [
[
"device_kernel",
[
4416,
5.785133960999994
]
],
[
"Kernel2",
[
15921,
1.6569721249999942
]
],
[
"_qsa_sparse_paged_gqa_splitk_kernel",
[
644,
1.481612926999999
]
],
[
"nvjet_sm121_tst_mma_128x208x64_2_32x104x64_tmaAB_bz_TNNN",
[
1000,
1.3622323199999982
]
],
[
"_hc_combine_norm_kernel",
[
4646,
0.8981812170000064
]
],
[
"kernel",
[
1510,
0.8489803810000001
]
],
[
"_hc_gate_mix_kernel",
[
4738,
0.6790835519999859
]
],
[
"nvjet_sm121_tst_mma_128x240x64_2_32x120x64_tmaAB_bz_TNNN",
[
980,
0.5484992310000006
]
],
[
"nvjet_sm121_tst_mma_128x128x64_3_32x64x64_tmaAB_bz_TNNN",
[
2000,
0.484882112
]
],
[
"finalizeMoeRoutingKernel",
[
2208,
0.4100567680000026
]
],
[
"_causal_conv1d_fwd_kernel",
[
756,
0.2423829440000001
]
],
[
"_fused_post_conv_kernel",
[
756,
0.23299334399999985
]
],
[
"doActivationKernel",
[
2208,
0.2290338470000007
]
],
[
"elementwise_kernel",
[
6060,
0.22898364799999965
]
],
[
"kernel_cutlass_kernel_flashinfergdn_kernelsdelta_rule_dsldelta_rule_sm120_FullyFusedDeltaRuleSm120_object_at__tensorptrf32gmemalign16o1_tensorptrf32gmemalign16o1_CopyAtom_ThrID10_TVLayout_0",
[
756,
0.19385532799999977
]
],
[
"vectorized_elementwise_kernel",
[
10976,
0.18249009600000335
]
],
[
"expandInputRowsKernel",
[
2208,
0.17563680399999979
]
],
[
"layer_norm_fwd_kernel",
[
756,
0.17546492700000002
]
],
[
"sm120_fp8_fp4_gemm_1d1d_impl",
[
184,
0.16484399999999996
]
],
[
"nvjet_sm121_tst_mma_128x192x64_2_32x96x64_tmaAB_bz_TNNN",
[
980,
0.13451315200000008
]
]
],
"notes": "GPU busy is interval union, not utilization counter. CPU/API/NVTX times overlap GPU work and each other; do not sum them as exclusive costs."
}
Reference in New Issue View Git Blame Copy Permalink