ruihan/qwen38-flash-next-dgx-spark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
perf/compile-prefix-cache
qwen38-flash-next-dgx-spark/docs/results/profile128k/graph/trace.2.summary.json
T

852 lines
20 KiB
JSON
Raw Permalink Blame History

{
"file": "trace.2.sqlite",
"gpu_activity_span_s": 17.613957408,
"gpu_busy_union_s": 17.173015684,
"gpu_idle_within_span_s": 0.44094172400000176,
"kernel_count": 111055,
"graph_node_kernel_count": 49140,
"api": {
"cudaStreamSynchronize_v3020": [
96,
7.291123311999998
],
"cuLaunchKernelEx": [
22540,
2.9715556959999576
],
"cudaLaunchKernelExC_v11060": [
11197,
2.7721689919999997
],
"cudaLaunchKernel_v7000": [
24666,
2.074785632000026
],
"cuLaunchKernel": [
3512,
0.1402584800000001
],
"cudaGraphLaunch_v10000": [
1431,
0.13174172799999972
],
"cudaEventSynchronize_v3020": [
48,
0.042627568
],
"cudaMemcpyAsync_v3020": [
2398,
0.02786942400000001
],
"cudaMemsetAsync_v3020": [
768,
0.005161456000000002
],
"cuKernelGetName": [
25422,
0.002872591999999989
],
"cudaEventQuery_v3020": [
1557,
0.0023473760000000004
],
"cuTensorMapEncodeTiled": [
12786,
0.002259360000000043
],
"cudaEventRecordWithFlags_v11010": [
1003,
0.001927552000000001
],
"cudaGetDriverEntryPointByVersion_v12050": [
12096,
0.0017841920000000863
],
"cuKernelGetFunction": [
3512,
0.0013231039999999851
],
"cudaStreamIsCapturing_v10000": [
2144,
0.0006217759999999976
],
"cudaStreamWaitEvent_v3020": [
144,
0.00041838400000000016
],
"cudaEventCreateWithFlags_v3020": [
96,
0.00024513599999999997
],
"cudaEventDestroy_v3020": [
96,
0.00022665600000000007
]
},
"nvtx": {
"PLE_lookup_tokens=1600": {
"count": 20,
"sum_s": 5.607371088,
"union_s": 5.607371088,
"gpu_overlap_s": 5.336825187
},
"PLE_wait_for_GPU": {
"count": 48,
"sum_s": 7.291963728,
"union_s": 7.291963728,
"gpu_overlap_s": 7.233423306
},
"PLE_CPU_dedup": {
"count": 48,
"sum_s": 0.020901856,
"union_s": 0.020901856,
"gpu_overlap_s": 0.0
},
"PLE_CPU_gather": {
"count": 48,
"sum_s": 0.292111936,
"union_s": 0.292111936,
"gpu_overlap_s": 0.0
},
"PLE_lookup_tokens=742": {
"count": 1,
"sum_s": 0.29688864,
"union_s": 0.29688864,
"gpu_overlap_s": 0.282970556
},
"PLE_lookup_tokens=4": {
"count": 27,
"sum_s": 1.71834584,
"union_s": 1.71834584,
"gpu_overlap_s": 1.61634065
}
},
"dispatch": {
"CG_DISPATCH:actual=1600:mode=NONE:padded=1600": 40,
"CG_DISPATCH:actual=1:mode=NONE:padded=1": 48,
"CG_DISPATCH:actual=742:mode=NONE:padded=742": 2,
"CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4": 27,
"CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4": 27
},
"dispatch_timeline": [
{
"start_ns": 46743952,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 416451360,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 443082544,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 469310784,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 1124006496,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 1154683696,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1179774000,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 1845801456,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 1877195728,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 1902001888,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 2578899920,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 2609916288,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 2635831856,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 3318901952,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 3349740016,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 3376088736,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 4054732640,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 4085130432,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 4110746784,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 4797770512,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 4829017936,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 4854209488,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 5551719760,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 5581307632,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 5607134000,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 6295334464,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 6325098224,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 6349994272,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 7042285248,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 7072015024,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 7097530960,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 7785043856,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 7816741056,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 7842952592,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 8537987536,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 8569536560,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 8595452192,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 9285771248,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 9317233072,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 9342219952,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 10060629616,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 10091559456,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 10116812048,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 10807242976,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 10837982160,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 10864521920,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 11562733424,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 11593737408,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 11619654832,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 12323127600,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 12354216560,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 12380982848,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 13081498400,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 13113506512,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 13138598000,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 13833930640,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 13865480864,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 13890978272,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 14599979088,
"name": "CG_DISPATCH:actual=1600:mode=NONE:padded=1600"
},
{
"start_ns": 14632423408,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 14657285504,
"name": "CG_DISPATCH:actual=742:mode=NONE:padded=742"
},
{
"start_ns": 15244687408,
"name": "CG_DISPATCH:actual=742:mode=NONE:padded=742"
},
{
"start_ns": 15261168608,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15272860704,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15510425328,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15511343584,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15516520016,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15591171712,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15592218224,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15597515056,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15670668032,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15671484256,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15675816752,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15748911968,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15749703280,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15753801520,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15829628848,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15830463168,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15834544656,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15911548208,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15912496096,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15917334144,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15989715552,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 15990467984,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 15994717936,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16067760256,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16068616608,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16072918640,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16151936192,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16153013376,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16158295088,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16231124064,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16232113808,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16237373136,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16308737952,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16309646016,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16314212128,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16388651696,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16389407904,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16393690448,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16470779408,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16471838880,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16476836288,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16549986896,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16550865424,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16555256672,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16632067984,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16632894512,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16637253408,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16709708944,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16710541136,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16714847184,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16789609392,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16790634928,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16795643776,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16869435120,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16870399200,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16875445984,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16952330656,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 16953311568,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 16958431376,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17031362304,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17032114688,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 17036354400,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17111019568,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17112056224,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 17117105808,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17188540768,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17189324992,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 17193788208,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17269626256,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17270693728,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 17276082128,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17347956512,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17348732480,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 17352868096,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17427032704,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17427739648,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 17431571536,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17508960384,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17509972320,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
},
{
"start_ns": 17514870544,
"name": "CG_DISPATCH:actual=3:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17592660304,
"name": "CG_DISPATCH:actual=4:mode=PIECEWISE:padded=4"
},
{
"start_ns": 17593603200,
"name": "CG_DISPATCH:actual=1:mode=NONE:padded=1"
}
],
"top_kernels": [
[
"device_kernel",
[
4608,
5.806803458000012
]
],
[
"Kernel2",
[
16931,
1.7378030570000023
]
],
[
"_qsa_sparse_paged_gqa_splitk_kernel",
[
672,
1.4925525440000014
]
],
[
"nvjet_sm121_tst_mma_128x208x64_2_32x104x64_tmaAB_bz_TNNN",
[
1000,
1.3766302719999985
]
],
[
"_hc_combine_norm_kernel",
[
4848,
0.9208744449999874
]
],
[
"kernel",
[
2853,
0.8909366829999992
]
],
[
"_hc_gate_mix_kernel",
[
4944,
0.6871558350000507
]
],
[
"nvjet_sm121_tst_mma_128x240x64_2_32x120x64_tmaAB_bz_TNNN",
[
980,
0.5473217280000005
]
],
[
"nvjet_sm121_tst_mma_128x128x64_3_32x64x64_tmaAB_bz_TNNN",
[
2000,
0.4623586880000001
]
],
[
"finalizeMoeRoutingKernel",
[
2304,
0.4111630399999976
]
],
[
"_causal_conv1d_fwd_kernel",
[
756,
0.24460185600000015
]
],
[
"doActivationKernel",
[
2304,
0.23128571500000175
]
],
[
"_fused_post_conv_kernel",
[
756,
0.23053734400000014
]
],
[
"elementwise_kernel",
[
6192,
0.22792899099999814
]
],
[
"kernel_cutlass_kernel_flashinfergdn_kernelsdelta_rule_dsldelta_rule_sm120_FullyFusedDeltaRuleSm120_object_at__tensorptrf32gmemalign16o1_tensorptrf32gmemalign16o1_CopyAtom_ThrID10_TVLayout_0",
[
756,
0.19290607999999995
]
],
[
"vectorized_elementwise_kernel",
[
11370,
0.18636042799999758
]
],
[
"layer_norm_fwd_kernel",
[
756,
0.18163276799999997
]
],
[
"sm120_fp8_fp4_gemm_1d1d_impl",
[
192,
0.17354873599999998
]
],
[
"expandInputRowsKernel",
[
2304,
0.1723688820000004
]
],
[
"nvjet_sm121_tst_mma_128x192x64_2_32x96x64_tmaAB_bz_TNNN",
[
980,
0.13571574399999997
]
]
],
"notes": "GPU busy is interval union, not utilization counter. CPU/API/NVTX times overlap GPU work and each other; do not sum them as exclusive costs."
}
Reference in New Issue View Git Blame Copy Permalink