ruihan/qwen38-flash-next-dgx-spark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
perf/compile-prefix-cache
qwen38-flash-next-dgx-spark/docs/results/profile128k/eager/trace.3.summary.json
T

583 lines
12 KiB
JSON
Raw Permalink Blame History

{
"file": "trace.3.sqlite",
"gpu_activity_span_s": 3.245924224,
"gpu_busy_union_s": 3.090907949,
"gpu_idle_within_span_s": 0.15501627499999993,
"kernel_count": 58496,
"graph_node_kernel_count": 0,
"api": {
"cudaStreamSynchronize_v3020": [
52,
1.0780331839999997
],
"cudaLaunchKernelExC_v11060": [
16696,
0.3360265120000019
],
"cuLaunchKernelEx": [
13138,
0.22730212800000002
],
"cudaLaunchKernel_v7000": [
16229,
0.21477230400000039
],
"cuLaunchKernel": [
12433,
0.05239439999999998
],
"cudaMemcpyAsync_v3020": [
939,
0.00983091199999999
],
"cudaEventSynchronize_v3020": [
26,
0.0089596
],
"cudaEventRecordWithFlags_v11010": [
2877,
0.004890783999999996
],
"cuKernelGetFunction": [
12433,
0.0027621279999998907
],
"cuTensorMapEncodeTiled": [
15496,
0.002756256000000075
],
"cudaMemsetAsync_v3020": [
416,
0.0023939039999999988
],
"cudaGetDriverEntryPointByVersion_v12050": [
14976,
0.001993504000000201
],
"cudaStreamWaitEvent_v3020": [
2430,
0.0018610560000000033
],
"cuKernelGetName": [
16301,
0.0014150560000001082
],
"cudaStreamIsCapturing_v10000": [
5810,
0.0011180959999999769
],
"cudaEventQuery_v3020": [
798,
0.001018544
],
"cudaEventCreateWithFlags_v3020": [
52,
0.000134192
],
"cudaEventDestroy_v3020": [
52,
0.00010068799999999997
]
},
"nvtx": {
"PLE_lookup_tokens=1600": {
"count": 1,
"sum_s": 0.018957376,
"union_s": 0.018957376,
"gpu_overlap_s": 0.011426528
},
"PLE_wait_for_GPU": {
"count": 26,
"sum_s": 1.078286144,
"union_s": 1.078286144,
"gpu_overlap_s": 1.058767637
},
"PLE_CPU_dedup": {
"count": 26,
"sum_s": 0.002243088,
"union_s": 0.002243088,
"gpu_overlap_s": 0.0
},
"PLE_CPU_gather": {
"count": 26,
"sum_s": 0.062812192,
"union_s": 0.062812192,
"gpu_overlap_s": 0.0
},
"PLE_lookup_tokens=742": {
"count": 1,
"sum_s": 0.292337536,
"union_s": 0.292337536,
"gpu_overlap_s": 0.279791319
},
"PLE_lookup_tokens=3": {
"count": 24,
"sum_s": 0.842259408,
"union_s": 0.842259408,
"gpu_overlap_s": 0.771610318
}
},
"dispatch": {
"CG_DISPATCH:actual=1600:mode=NONE:padded=1600": 2,
"CG_DISPATCH:actual=1:mode=NONE:padded=1": 26,
"CG_DISPATCH:actual=742:mode=NONE:padded=742": 2,
"CG_DISPATCH:actual=3:mode=NONE:padded=3": 48
},
"dispatch_timeline": [
{
"start_ns": 46439136,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 440098864,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 472554240,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 498029616,
"name": "CG_DISPATCH:actual=742:mode=NONE:padded=742"
},
{
"start_ns": 1080640368,
"name": "CG_DISPATCH:actual=742:mode=NONE:padded=742"
},
{
"start_ns": 1097925088,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1109689376,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1374741120,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1377837856,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1381724192,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1449909984,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1452632352,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1456060352,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1537442544,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1540106512,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1543362000,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1622817104,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1625489424,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1628952064,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1697539568,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1699380000,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1702138000,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1777756128,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1779549280,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1782099440,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1865886816,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1868546304,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1872017632,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1939645984,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 1941472672,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1944250112,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2021839712,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2023852576,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2026597104,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2103480432,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2105282848,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2107900064,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2193835440,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2196580896,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2199840944,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2282745504,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2285692352,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2289777504,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2362513824,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2365324800,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2368818832,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2436200000,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2438187600,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2441020928,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2516737184,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2518684848,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2521416768,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2596916928,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2598750864,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2601295488,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2678072112,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2679938368,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2682459200,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2756745440,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2758542912,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2761074960,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2838330208,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2840177632,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2842655760,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2918308832,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 2920073040,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2922520640,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 3006922960,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 3008938128,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 3011760224,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 3094064288,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 3096710352,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 3100043552,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 3174526160,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 3177165152,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 3180415120,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 3257416576,
"name": "CG_DISPATCH:actual=3:mode=NONE:padded=3"
},
{
"start_ns": 3260041984,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
}
],
"top_kernels": [
[
"Kernel2",
[
12433,
1.083779682999996
]
],
[
"device_kernel",
[
2496,
0.8852412400000004
]
],
[
"kernel",
[
360,
0.38886832
]
],
[
"_qsa_sparse_paged_gqa_splitk_kernel",
[
364,
0.12547817500000008
]
],
[
"_hc_combine_norm_kernel",
[
2626,
0.06954325899999982
]
],
[
"nvjet_sm121_tst_mma_128x208x64_2_32x104x64_tmaAB_bz_TNNN",
[
50,
0.068864416
]
],
[
"_hc_gate_mix_kernel",
[
2678,
0.04872185600000148
]
],
[
"finalizeMoeRoutingKernel",
[
1248,
0.03866553600000007
]
],
[
"sm120_fp8_fp4_gemm_1d1d_impl",
[
104,
0.038346688000000004
]
],
[
"nvjet_sm121_tst_mma_192x160x64_2_48x80x64_tmaAB_bz_TNNN",
[
51,
0.034701248
]
],
[
"nvjet_sm121_tst_mma_128x240x64_2_32x120x64_tmaAB_bz_TNNN",
[
49,
0.02691910399999999
]
],
[
"nvjet_sm121_tst_mma_128x128x64_3_32x64x64_tmaAB_bz_TNNN",
[
100,
0.024421632000000002
]
],
[
"gdn_decode_post_conv_mtp_kernel",
[
864,
0.023706272000000014
]
],
[
"expandInputRowsKernel",
[
1248,
0.022278243999999982
]
],
[
"doActivationKernel",
[
1248,
0.02122110900000002
]
],
[
"elementwise_kernel",
[
2004,
0.020444831999999843
]
],
[
"vectorized_elementwise_kernel",
[
5307,
0.018513366999999715
]
],
[
"_causal_conv1d_fwd_kernel",
[
72,
0.017434495999999997
]
],
[
"nvjet_sm121_tst_mma_128x224x64_2_64x56x64_tmaAB_bz_TNNN",
[
50,
0.016634912
]
],
[
"_fused_post_conv_kernel",
[
72,
0.015161888000000002
]
]
],
"notes": "GPU busy is interval union, not utilization counter. CPU/API/NVTX times overlap GPU work and each other; do not sum them as exclusive costs."
}
Reference in New Issue View Git Blame Copy Permalink