{ "file": "trace.3.sqlite", "gpu_activity_span_s": 3.245924224, "gpu_busy_union_s": 3.090907949, "gpu_idle_within_span_s": 0.15501627499999993, "kernel_count": 58496, "graph_node_kernel_count": 0, "api": { "cudaStreamSynchronize_v3020": [ 52, 1.0780331839999997 ], "cudaLaunchKernelExC_v11060": [ 16696, 0.3360265120000019 ], "cuLaunchKernelEx": [ 13138, 0.22730212800000002 ], "cudaLaunchKernel_v7000": [ 16229, 0.21477230400000039 ], "cuLaunchKernel": [ 12433, 0.05239439999999998 ], "cudaMemcpyAsync_v3020": [ 939, 0.00983091199999999 ], "cudaEventSynchronize_v3020": [ 26, 0.0089596 ], "cudaEventRecordWithFlags_v11010": [ 2877, 0.004890783999999996 ], "cuKernelGetFunction": [ 12433, 0.0027621279999998907 ], "cuTensorMapEncodeTiled": [ 15496, 0.002756256000000075 ], "cudaMemsetAsync_v3020": [ 416, 0.0023939039999999988 ], "cudaGetDriverEntryPointByVersion_v12050": [ 14976, 0.001993504000000201 ], "cudaStreamWaitEvent_v3020": [ 2430, 0.0018610560000000033 ], "cuKernelGetName": [ 16301, 0.0014150560000001082 ], "cudaStreamIsCapturing_v10000": [ 5810, 0.0011180959999999769 ], "cudaEventQuery_v3020": [ 798, 0.001018544 ], "cudaEventCreateWithFlags_v3020": [ 52, 0.000134192 ], "cudaEventDestroy_v3020": [ 52, 0.00010068799999999997 ] }, "nvtx": { "PLE_lookup_tokens=1600": { "count": 1, "sum_s": 0.018957376, "union_s": 0.018957376, "gpu_overlap_s": 0.011426528 }, "PLE_wait_for_GPU": { "count": 26, "sum_s": 1.078286144, "union_s": 1.078286144, "gpu_overlap_s": 1.058767637 }, "PLE_CPU_dedup": { "count": 26, "sum_s": 0.002243088, "union_s": 0.002243088, "gpu_overlap_s": 0.0 }, "PLE_CPU_gather": { "count": 26, "sum_s": 0.062812192, "union_s": 0.062812192, "gpu_overlap_s": 0.0 }, "PLE_lookup_tokens=742": { "count": 1, "sum_s": 0.292337536, "union_s": 0.292337536, "gpu_overlap_s": 0.279791319 }, "PLE_lookup_tokens=3": { "count": 24, "sum_s": 0.842259408, "union_s": 0.842259408, "gpu_overlap_s": 0.771610318 } }, "dispatch": { "CG_DISPATCH:actual=1600:mode=NONE:padded=1600": 2, "CG_DISPATCH:actual=1:mode=NONE:padded=1": 26, "CG_DISPATCH:actual=742:mode=NONE:padded=742": 2, "CG_DISPATCH:actual=3:mode=NONE:padded=3": 48 }, "dispatch_timeline": [ { "start_ns": 46439136, "name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600" }, { "start_ns": 440098864, "name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600" }, { "start_ns": 472554240, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 498029616, "name": "CG_DISPATCH:actual=742:mode=NONE:padded=742" }, { "start_ns": 1080640368, "name": "CG_DISPATCH:actual=742:mode=NONE:padded=742" }, { "start_ns": 1097925088, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 1109689376, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1374741120, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1377837856, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 1381724192, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1449909984, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1452632352, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 1456060352, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1537442544, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1540106512, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 1543362000, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1622817104, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1625489424, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 1628952064, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1697539568, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1699380000, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 1702138000, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1777756128, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1779549280, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 1782099440, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1865886816, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1868546304, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 1872017632, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1939645984, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 1941472672, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 1944250112, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2021839712, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2023852576, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 2026597104, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2103480432, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2105282848, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 2107900064, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2193835440, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2196580896, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 2199840944, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2282745504, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2285692352, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 2289777504, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2362513824, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2365324800, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 2368818832, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2436200000, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2438187600, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 2441020928, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2516737184, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2518684848, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 2521416768, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2596916928, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2598750864, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 2601295488, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2678072112, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2679938368, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 2682459200, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2756745440, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2758542912, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 2761074960, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2838330208, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2840177632, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 2842655760, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2918308832, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 2920073040, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 2922520640, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 3006922960, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 3008938128, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 3011760224, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 3094064288, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 3096710352, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 3100043552, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 3174526160, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 3177165152, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" }, { "start_ns": 3180415120, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 3257416576, "name": "CG_DISPATCH:actual=3:mode=NONE:padded=3" }, { "start_ns": 3260041984, "name": "CG_DISPATCH:actual=1:mode=NONE:padded=1" } ], "top_kernels": [ [ "Kernel2", [ 12433, 1.083779682999996 ] ], [ "device_kernel", [ 2496, 0.8852412400000004 ] ], [ "kernel", [ 360, 0.38886832 ] ], [ "_qsa_sparse_paged_gqa_splitk_kernel", [ 364, 0.12547817500000008 ] ], [ "_hc_combine_norm_kernel", [ 2626, 0.06954325899999982 ] ], [ "nvjet_sm121_tst_mma_128x208x64_2_32x104x64_tmaAB_bz_TNNN", [ 50, 0.068864416 ] ], [ "_hc_gate_mix_kernel", [ 2678, 0.04872185600000148 ] ], [ "finalizeMoeRoutingKernel", [ 1248, 0.03866553600000007 ] ], [ "sm120_fp8_fp4_gemm_1d1d_impl", [ 104, 0.038346688000000004 ] ], [ "nvjet_sm121_tst_mma_192x160x64_2_48x80x64_tmaAB_bz_TNNN", [ 51, 0.034701248 ] ], [ "nvjet_sm121_tst_mma_128x240x64_2_32x120x64_tmaAB_bz_TNNN", [ 49, 0.02691910399999999 ] ], [ "nvjet_sm121_tst_mma_128x128x64_3_32x64x64_tmaAB_bz_TNNN", [ 100, 0.024421632000000002 ] ], [ "gdn_decode_post_conv_mtp_kernel", [ 864, 0.023706272000000014 ] ], [ "expandInputRowsKernel", [ 1248, 0.022278243999999982 ] ], [ "doActivationKernel", [ 1248, 0.02122110900000002 ] ], [ "elementwise_kernel", [ 2004, 0.020444831999999843 ] ], [ "vectorized_elementwise_kernel", [ 5307, 0.018513366999999715 ] ], [ "_causal_conv1d_fwd_kernel", [ 72, 0.017434495999999997 ] ], [ "nvjet_sm121_tst_mma_128x224x64_2_64x56x64_tmaAB_bz_TNNN", [ 50, 0.016634912 ] ], [ "_fused_post_conv_kernel", [ 72, 0.015161888000000002 ] ] ], "notes": "GPU busy is interval union, not utilization counter. CPU/API/NVTX times overlap GPU work and each other; do not sum them as exclusive costs." }