ruihan/qwen38-flash-next-dgx-spark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
7f3464fb050439fafbc0a9f85a0dd23df7c4adec
qwen38-flash-next-dgx-spark/docs/results/profile128k/graph/trace.3.summary.json
T

636 lines
14 KiB
JSON
Raw Blame History

{
"file": "trace.3.sqlite",
"gpu_activity_span_s": 3.549492992,
"gpu_busy_union_s": 3.382135698,
"gpu_idle_within_span_s": 0.16735729399999988,
"kernel_count": 66056,
"graph_node_kernel_count": 50960,
"api": {
"cudaStreamSynchronize_v3020": [
60,
2.063916384
],
"cudaLaunchKernelExC_v11060": [
1284,
0.28009695999999934
],
"cuLaunchKernelEx": [
5990,
0.2070799999999992
],
"cudaLaunchKernel_v7000": [
7285,
0.1831771039999994
],
"cudaGraphLaunch_v10000": [
1484,
0.119649248
],
"cudaEventSynchronize_v3020": [
30,
0.040196352
],
"cuLaunchKernel": [
537,
0.01219283200000001
],
"cudaMemcpyAsync_v3020": [
1043,
0.010008064000000006
],
"cudaMemsetAsync_v3020": [
480,
0.0025478400000000017
],
"cudaEventQuery_v3020": [
921,
0.0011545439999999998
],
"cuKernelGetName": [
7357,
0.00088769599999998
],
"cudaEventRecordWithFlags_v11010": [
605,
0.0008087840000000007
],
"cudaStreamIsCapturing_v10000": [
1274,
0.00032420800000000107
],
"cuTensorMapEncodeTiled": [
1472,
0.0003082879999999999
],
"cuKernelGetFunction": [
537,
0.0002906560000000001
],
"cudaGetDriverEntryPointByVersion_v12050": [
1152,
0.00016971199999999957
],
"cudaStreamWaitEvent_v3020": [
90,
0.000155392
],
"cudaEventCreateWithFlags_v3020": [
60,
0.00014443199999999998
],
"cudaEventDestroy_v3020": [
60,
8.001600000000003e-05
]
},
"nvtx": {
"PLE_lookup_tokens=1600": {
"count": 1,
"sum_s": 0.019309872,
"union_s": 0.019309872,
"gpu_overlap_s": 0.010934608
},
"PLE_wait_for_GPU": {
"count": 30,
"sum_s": 2.064303616,
"union_s": 2.064303616,
"gpu_overlap_s": 2.035614378
},
"PLE_CPU_dedup": {
"count": 30,
"sum_s": 0.00226768,
"union_s": 0.00226768,
"gpu_overlap_s": 0.0
},
"PLE_CPU_gather": {
"count": 30,
"sum_s": 0.109426208,
"union_s": 0.109426208,
"gpu_overlap_s": 0.0
},
"PLE_lookup_tokens=742": {
"count": 1,
"sum_s": 0.293855216,
"union_s": 0.293855216,
"gpu_overlap_s": 0.27970459
},
"PLE_lookup_tokens=4": {
"count": 28,
"sum_s": 1.874469744,
"union_s": 1.874469744,
"gpu_overlap_s": 1.748828475
}
},
"dispatch": {
"CG_DISPATCH:actual=1600:mode=NONE:padded=1600": 2,
"CG_DISPATCH:actual=1:mode=NONE:padded=1": 30,
"CG_DISPATCH:actual=742:mode=NONE:padded=742": 2,
"CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4": 28,
"CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4": 28
},
"dispatch_timeline": [
{
"start_ns": 47667312,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 442471520,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 473807600,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 498595616,
"name": "CG_DISPATCH:actual=742:mode=NONE:padded=742"
},
{
"start_ns": 1085215856,
"name": "CG_DISPATCH:actual=742:mode=NONE:padded=742"
},
{
"start_ns": 1101603472,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1113435424,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1335575776,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1336548688,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1340993280,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1412886784,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1413839472,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1418416416,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1496932144,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1497849376,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1502437168,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1574235248,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1574893632,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1578152608,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1659975184,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1660978304,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1666040464,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1739695136,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1740582992,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1745368080,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1819362544,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1820259120,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1825002016,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1898444080,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1899320480,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1903978080,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1973992688,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 1974642976,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1977460352,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2058854000,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2059781888,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2064550128,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2140294400,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2141166416,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2145782752,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2215803760,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2216437328,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2219274592,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2295308880,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2295905872,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2298529488,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2373456000,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2374001712,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2376400016,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2451495056,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2452018624,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2454446720,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2534005728,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2534498160,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2536805632,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2617391424,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2618055024,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2620944928,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2698279104,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2699135648,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2704056256,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2781484592,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2782408128,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2787209600,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2872938352,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2873856272,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2878412464,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2955704400,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 2956657904,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2961494272,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3046465232,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3047422016,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 3052108288,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3123689648,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3124649664,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 3129308048,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3199193712,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3199819264,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 3202578560,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3285661472,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3286559472,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 3291397856,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3363509456,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3364085808,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 3366681840,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3447715488,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3448689632,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 3453541344,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3529683168,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 3530662560,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
}
],
"top_kernels": [
[
"Kernel2",
[
14453,
1.2634071650000012
]
],
[
"device_kernel",
[
2880,
0.9274175719999982
]
],
[
"kernel",
[
1772,
0.4575362240000001
]
],
[
"_qsa_sparse_paged_gqa_splitk_kernel",
[
420,
0.13202313600000007
]
],
[
"_hc_combine_norm_kernel",
[
3030,
0.07136511900000105
]
],
[
"nvjet_sm121_tst_mma_128x208x64_2_32x104x64_tmaAB_bz_TNNN",
[
50,
0.06789817599999999
]
],
[
"sm120_fp8_fp4_gemm_1d1d_impl",
[
120,
0.05055315199999998
]
],
[
"_hc_gate_mix_kernel",
[
3090,
0.049511773999999786
]
],
[
"finalizeMoeRoutingKernel",
[
1440,
0.04024281600000026
]
],
[
"nvjet_sm121_tst_mma_192x160x64_2_48x80x64_tmaAB_bz_TNNN",
[
51,
0.035206783999999984
]
],
[
"gdn_decode_post_conv_mtp_kernel",
[
1008,
0.027670688000000006
]
],
[
"nvjet_sm121_tst_mma_128x240x64_2_32x120x64_tmaAB_bz_TNNN",
[
49,
0.027358240000000006
]
],
[
"elementwise_kernel",
[
2268,
0.02362060799999982
]
],
[
"nvjet_sm121_tst_mma_128x128x64_3_32x64x64_tmaAB_bz_TNNN",
[
100,
0.023191840000000002
]
],
[
"vectorized_elementwise_kernel",
[
6095,
0.02246828099999996
]
],
[
"doActivationKernel",
[
1440,
0.02160813899999983
]
],
[
"expandInputRowsKernel",
[
1440,
0.020183961000000014
]
],
[
"_causal_conv1d_fwd_kernel",
[
72,
0.017587615999999997
]
],
[
"nvjet_sm121_tst_mma_128x224x64_2_64x56x64_tmaAB_bz_TNNN",
[
50,
0.016199904
]
],
[
"_fused_post_conv_kernel",
[
72,
0.014694784
]
]
],
"notes": "GPU busy is interval union, not utilization counter. CPU/API/NVTX times overlap GPU work and each other; do not sum them as exclusive costs."
}
Reference in New Issue View Git Blame Copy Permalink